7e53d9360a
dev/vendor/MANIFEST.toml pins bcda-app@7decdcc (v3 handler + tests, request validation middleware, OperationOutcome writer, v3 Postman collection, gzip fixtures, sandbox config, LICENSE) and the bcda.cms.gov v3 data dictionary, v2 dictionary and official v3 sample ndjson. vendor_bcda_app.py --verify | --check | --sync (stdlib only); the bcda-vendor workflow runs --check weekly and files an issue on drift — nothing auto-commits, a payload change upstream is something a human reads before the flattener is re-pointed. gen_bcda_v3_crosswalk.py writes dev/seeds/bcda_v3_data_dictionary.csv (fhirPath, IDR sourceColumn, ccwMapping, cclfMapping per v3 field), bcda_v3_cclf_crosswalk.csv (293 CCLF elements -> v3 field) and bcda_v3_cclf_absent.csv (52 CCLF fields v3 does not carry); the CCLF sheet has label/name swapped on data rows and is normalised. Tests assert the committed seeds match the vendored workbook. refs #1139
906 lines
28 KiB
Python
906 lines
28 KiB
Python
"""Gitea Actions workflow emitter.
|
|
|
|
Thin adapter over the GitHub backend — Gitea Actions uses the same
|
|
YAML syntax but with:
|
|
- Output to .gitea/workflows/ (not .github/workflows/)
|
|
- Registry auth via PAT secret (GITEA_TOKEN lacks package scope)
|
|
- Platform registry (not ghcr.io)
|
|
- Full URL form for actions (for air-gapped installs)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from backends.github import (
|
|
_HEADER,
|
|
)
|
|
|
|
# ── Override action refs with full GitHub URLs ───────────────────
|
|
# Gitea Actions resolves short refs (e.g. actions/checkout@v4) against
|
|
# the local Gitea instance. Use full URLs to pull from GitHub.
|
|
|
|
|
|
def _checkout_step() -> str:
|
|
return """\
|
|
- name: Checkout
|
|
uses: https://github.com/actions/checkout@v4"""
|
|
|
|
|
|
def _setup_buildx_step() -> str:
|
|
return "" # Not needed — we use plain docker build/push
|
|
|
|
|
|
def _setup_uv_step(uv_version: str) -> str:
|
|
# astral-sh/setup-uv is a composite action that calls the Gitea API
|
|
# internally, which fails because 'astral-sh' isn't a local user.
|
|
# Install uv directly via the official installer script instead.
|
|
return """\
|
|
- name: Set up uv
|
|
run: curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
env:
|
|
UV_INSTALL_DIR: /usr/local/bin"""
|
|
|
|
|
|
def _build_push_step(
|
|
img: dict,
|
|
tags_expr: str,
|
|
registry: str,
|
|
owner_repo: str,
|
|
*,
|
|
no_cache: bool = False,
|
|
load_only: bool = False,
|
|
) -> str:
|
|
"""docker build (via socket) + crane push (daemonless, uses container DNS).
|
|
|
|
docker push goes through the host daemon which can't resolve 'gitea'.
|
|
crane pushes directly from the job container's network stack.
|
|
"""
|
|
name = img["name"]
|
|
tags = [t.strip() for t in tags_expr.split(",") if t.strip()]
|
|
local_tag = f"local/{name}:build"
|
|
cache_flag = " --no-cache" if no_cache else ""
|
|
lines = f"""\
|
|
- name: Build {name}
|
|
run: docker build{cache_flag} -f {img["dockerfile"]} -t {local_tag} {img["context"]}"""
|
|
if not load_only and tags:
|
|
# Save to tarball, then crane push each tag (daemonless)
|
|
push_cmds = "\n ".join(
|
|
f"crane push /tmp/{name}.tar {t} --insecure" for t in tags
|
|
)
|
|
lines += f"""
|
|
|
|
- name: Push {name}
|
|
run: |
|
|
docker save {local_tag} -o /tmp/{name}.tar
|
|
{push_cmds}"""
|
|
return lines
|
|
|
|
|
|
def _install_trivy_step() -> str:
|
|
"""Install trivy as a native binary — no Docker-in-Docker needed."""
|
|
return """\
|
|
- name: Install trivy
|
|
run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin"""
|
|
|
|
|
|
def _trivy_step(
|
|
img: dict,
|
|
tag: str,
|
|
registry: str,
|
|
owner_repo: str,
|
|
) -> str:
|
|
"""Scan local Docker image — no registry pull, no DNS resolution."""
|
|
name = img["name"]
|
|
local_tag = f"local/{name}:build"
|
|
sev = img.get("trivy_severity", "HIGH,CRITICAL")
|
|
ec = img.get("trivy_exit_code", 0)
|
|
return f"""\
|
|
- name: Scan {name}
|
|
run: trivy image --severity {sev} --exit-code {ec} --format json -o {name}-scan.json {local_tag}"""
|
|
|
|
|
|
# ── Failure reporting ─────────────────────────────────────────────
|
|
|
|
|
|
def _failure_step(workflow_name: str, job_name: str) -> str:
|
|
"""Emit an if:failure() step that files a Gitea issue."""
|
|
return f"""\
|
|
- name: File failure issue
|
|
if: failure()
|
|
env:
|
|
GITEA_TOKEN: ${{{{ secrets.DEPLOY_TOKEN }}}}
|
|
run: |
|
|
# Jobs that never install uv (image builds, infra checks) still
|
|
# need it here — otherwise the filer dies with "uv: command not
|
|
# found" and the failure is never filed.
|
|
command -v uv >/dev/null 2>&1 || curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
uv sync --no-dev --quiet 2>/dev/null || true
|
|
uv run python -m api.diag.ci \\
|
|
--workflow "{workflow_name}" --job "{job_name}" \\
|
|
--run "${{{{ github.run_number }}}}" \\
|
|
--sha "${{{{ github.sha }}}}" \\
|
|
--ref "${{{{ github.ref }}}}" || true"""
|
|
|
|
|
|
# ── Gitea-specific helpers ───────────────────────────────────────
|
|
|
|
|
|
def _docker_login_step(registry: str) -> str:
|
|
"""Install crane and authenticate — daemonless push tool."""
|
|
return f"""\
|
|
- name: Install crane
|
|
run: curl -sL https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz | tar xz -C /usr/local/bin crane
|
|
|
|
- name: Log in to registry
|
|
run: crane auth login {registry} -u "${{{{ secrets.REGISTRY_USER }}}}" -p "${{{{ secrets.REGISTRY_TOKEN }}}}"
|
|
env:
|
|
CRANE_INSECURE: "true\""""
|
|
|
|
|
|
# ── Workflow generators ──────────────────────────────────────────
|
|
|
|
|
|
def _gen_ci(
|
|
runner: str, uv_version: str, coverage_threshold: int = 99, **_kw: object
|
|
) -> tuple[str, str]:
|
|
content = f"""\
|
|
{_HEADER}
|
|
name: CI
|
|
|
|
on:
|
|
push:
|
|
branches: ["**"]
|
|
pull_request:
|
|
|
|
jobs:
|
|
lint:
|
|
runs-on: {runner}
|
|
steps:
|
|
{_checkout_step()}
|
|
|
|
{_setup_uv_step(uv_version)}
|
|
|
|
- name: Install dependencies
|
|
run: uv sync --dev
|
|
|
|
- name: Ruff check
|
|
run: uv run ruff check src/ tests/ --output-format=concise
|
|
|
|
- name: Ruff format
|
|
run: uv run ruff format --check src/ tests/
|
|
|
|
- name: Validate generated config
|
|
run: uv run python dev/scripts/gen_config.py --check
|
|
|
|
test:
|
|
runs-on: {runner}
|
|
steps:
|
|
{_checkout_step()}
|
|
|
|
{_setup_uv_step(uv_version)}
|
|
|
|
- name: Install dependencies
|
|
run: uv sync --dev
|
|
|
|
- name: Preinstall duckdb extensions
|
|
# tests/zot/test_duck.py runs INSTALL sqlite from every xdist
|
|
# worker; concurrent installs race the extension-file rename in
|
|
# ~/.duckdb ("Could not remove file ... sqlite_scanner", #515).
|
|
# Installing once up front makes the in-test INSTALL a no-op.
|
|
# ducklake: same race class, used by tests/aco DuckLakeContext.
|
|
run: uv run python -c "import duckdb; duckdb.connect().execute('INSTALL sqlite; INSTALL ducklake')"
|
|
|
|
- name: Pytest
|
|
# -n auto parallelizes across runner cores. Coverage combining
|
|
# is configured via [tool.coverage.run] parallel=true in
|
|
# pyproject.toml — without that the per-worker .coverage.* files
|
|
# don't merge reliably (see #388 for the prior revert).
|
|
run: uv run pytest tests/ -x --cov=src --cov-report=term-missing --cov-fail-under={coverage_threshold} -q -n auto
|
|
|
|
{_failure_step("CI", "test")}
|
|
|
|
notebooks-smoke:
|
|
runs-on: {runner}
|
|
steps:
|
|
{_checkout_step()}
|
|
|
|
{_setup_uv_step(uv_version)}
|
|
|
|
- name: Install dependencies
|
|
run: uv sync --dev
|
|
|
|
- name: Stub aco.duckdb
|
|
# The checkout has no data volume; conf.connect.duckdb() opens
|
|
# data/aco.duckdb read-only and fails if it doesn't exist. An
|
|
# empty-but-valid database is enough for the ci-smoke notebooks
|
|
# (schema browsing over information_schema returns zero rows).
|
|
run: |
|
|
[ -f data/aco.duckdb ] || uv run python -c \\
|
|
"import duckdb; duckdb.connect('data/aco.duckdb').close()"
|
|
|
|
- name: Run data-independent notebooks headless
|
|
# Executes the [ci_smoke] set from infra/marimo/nb-tests.toml via
|
|
# `marimo export session` and fails on any cell error. Runs on
|
|
# every push (not path-gated): notebooks import src/ modules, so
|
|
# src changes can break them too.
|
|
run: uv run python dev/scripts/nb_integration.py --set ci-smoke
|
|
|
|
{_failure_step("CI", "notebooks-smoke")}
|
|
"""
|
|
return (".gitea/workflows/ci.yml", content)
|
|
|
|
|
|
_CHANGES_JOB = "changes"
|
|
_CHANGES_STEP_ID = "filter"
|
|
_ALL_CHANGED = "__all__"
|
|
_NULL_SHA = "0" * 40
|
|
|
|
# Shell body of the change-detection step; the braces of the `match`
|
|
# function are doubled so they survive the f-string.
|
|
_CHANGES_SCRIPT = f"""\
|
|
# `github.event.head_commit.modified` only describes the tip commit
|
|
# and is empty for merge commits, so a PR merged with a merge commit
|
|
# never matched any path filter. Diff the pushed range instead:
|
|
# the branch tip before the push against the new one. When that
|
|
# previous tip is unknown or unfetchable (first push, history
|
|
# rewrite) build everything rather than nothing.
|
|
changed={_ALL_CHANGED}
|
|
if [ -n "$BEFORE" ] && [ "$BEFORE" != "{_NULL_SHA}" ] \\
|
|
&& git fetch --quiet --depth=1 origin "$BEFORE"; then
|
|
changed="$(git diff --name-only "$BEFORE" "$GITHUB_SHA")" || changed={_ALL_CHANGED}
|
|
fi
|
|
echo "changed since $BEFORE:"
|
|
printf '%s\\n' "$changed"
|
|
match() {{
|
|
name="$1"; shift
|
|
hit=false
|
|
if [ "$changed" = {_ALL_CHANGED} ]; then
|
|
hit=true
|
|
else
|
|
while IFS= read -r f; do
|
|
[ -n "$f" ] || continue
|
|
for g in "$@"; do
|
|
case "$f" in $g) hit=true ;; esac
|
|
done
|
|
done <<< "$changed"
|
|
fi
|
|
echo "$name=$hit" >> "$GITHUB_OUTPUT"
|
|
echo "$name: $hit"
|
|
}}"""
|
|
|
|
|
|
def _case_glob(pattern: str) -> str:
|
|
"""Translate a ``path_filter`` glob into a bash ``case`` pattern.
|
|
|
|
``*`` already crosses ``/`` in ``case``, so ``src/**`` becomes ``src/*``.
|
|
"""
|
|
while "**" in pattern:
|
|
pattern = pattern.replace("**", "*")
|
|
return pattern
|
|
|
|
|
|
def _changes_job(images: list[dict], runner: str) -> str:
|
|
"""Emit the job that turns the pushed diff into one boolean per image."""
|
|
outputs = "\n".join(
|
|
f" {img['name']}: "
|
|
f"${{{{ steps.{_CHANGES_STEP_ID}.outputs.{img['name']} }}}}"
|
|
for img in images
|
|
)
|
|
matches = []
|
|
for img in images:
|
|
globs = [_case_glob(p) for p in img.get("path_filter", [])]
|
|
if globs:
|
|
quoted = " ".join(f"'{g}'" for g in globs)
|
|
matches.append(f" match {img['name']} {quoted}")
|
|
else:
|
|
# No filter means the image follows every push.
|
|
matches.append(f' echo "{img["name"]}=true" >> "$GITHUB_OUTPUT"')
|
|
match_block = "\n".join(matches)
|
|
return f"""\
|
|
{_CHANGES_JOB}:
|
|
runs-on: {runner}
|
|
outputs:
|
|
{outputs}
|
|
steps:
|
|
{_checkout_step()}
|
|
|
|
- name: Detect changed paths
|
|
id: {_CHANGES_STEP_ID}
|
|
env:
|
|
BEFORE: ${{{{ github.event.before }}}}
|
|
run: |
|
|
{_CHANGES_SCRIPT}
|
|
{match_block}"""
|
|
|
|
|
|
def _gen_deploy(
|
|
images: list[dict],
|
|
scans: list[dict],
|
|
registry: str,
|
|
owner_repo: str,
|
|
runner: str,
|
|
uv_version: str,
|
|
**_kw: object,
|
|
) -> tuple[str, str]:
|
|
scan_set = {s["name"] for s in scans}
|
|
|
|
# Gitea Actions doesn't support `paths:` filters on push, so one job
|
|
# computes which images the pushed range touches and the per-image
|
|
# jobs gate on its outputs.
|
|
job_blocks = [_changes_job(images, runner)]
|
|
job_names = [_CHANGES_JOB]
|
|
for img in images:
|
|
name = img["name"]
|
|
job_names.append(name)
|
|
tags = (
|
|
f"{registry}/{owner_repo}/{name}:${{{{ env.SHORT_SHA }}}},"
|
|
f"{registry}/{owner_repo}/{name}:latest"
|
|
)
|
|
scan_step = ""
|
|
if name in scan_set:
|
|
scan_step = f"""
|
|
|
|
{_install_trivy_step()}
|
|
|
|
{_trivy_step(img, "${{{{ env.SHORT_SHA }}}}", registry, owner_repo)}"""
|
|
|
|
job_blocks.append(f"""\
|
|
{name}:
|
|
runs-on: {runner}
|
|
needs: [{_CHANGES_JOB}]
|
|
if: needs.{_CHANGES_JOB}.outputs.{name} == 'true'
|
|
steps:
|
|
{_checkout_step()}
|
|
|
|
{_docker_login_step(registry)}
|
|
{scan_step}
|
|
|
|
- name: Compute short SHA
|
|
run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
|
|
|
|
{_build_push_step(img, tags, registry, owner_repo)}
|
|
|
|
{_failure_step("Deploy", name)}""")
|
|
|
|
jobs_block = "\n\n".join(job_blocks)
|
|
needs_list = ", ".join(job_names)
|
|
|
|
content = f"""\
|
|
{_HEADER}
|
|
name: Deploy
|
|
|
|
on:
|
|
push:
|
|
branches: [main]
|
|
|
|
jobs:
|
|
{jobs_block}
|
|
|
|
report:
|
|
runs-on: {runner}
|
|
needs: [{needs_list}]
|
|
if: always() && !cancelled()
|
|
steps:
|
|
{_checkout_step()}
|
|
|
|
{_setup_uv_step(uv_version)}
|
|
|
|
- name: Report
|
|
# Only file an issue if at least one upstream build job failed.
|
|
# Without this gate api.diag.ci runs unconditionally and files
|
|
# a fake "failed" issue on every successful Deploy run.
|
|
if: contains(needs.*.result, 'failure')
|
|
env:
|
|
GITEA_TOKEN: ${{{{ secrets.DEPLOY_TOKEN }}}}
|
|
run: |
|
|
uv sync --no-dev --quiet 2>/dev/null || true
|
|
uv run python -m api.diag.ci \\
|
|
--workflow "Deploy" --job "report" \\
|
|
--run "${{{{ github.run_number }}}}" \\
|
|
--sha "${{{{ github.sha }}}}" \\
|
|
--ref "${{{{ github.ref }}}}" || true
|
|
"""
|
|
return (".gitea/workflows/deploy.yml", content)
|
|
|
|
|
|
def _gen_harden(
|
|
images: list[dict],
|
|
scans: list[dict],
|
|
registry: str,
|
|
owner_repo: str,
|
|
runner: str,
|
|
uv_version: str,
|
|
**_kw: object,
|
|
) -> tuple[str, str]:
|
|
build_steps = []
|
|
for img in images:
|
|
tags = (
|
|
f"{registry}/{owner_repo}/{img['name']}:hardened,"
|
|
f"{registry}/{owner_repo}/{img['name']}:latest"
|
|
)
|
|
build_steps.append(
|
|
_build_push_step(img, tags, registry, owner_repo, no_cache=True)
|
|
)
|
|
build_block = "\n\n".join(build_steps)
|
|
|
|
scan_steps = []
|
|
for img in scans:
|
|
scan_steps.append(_trivy_step(img, "hardened", registry, owner_repo))
|
|
scan_block = "\n\n".join(scan_steps)
|
|
|
|
vuln_files = " ".join(f"{i['name']}-scan.json" for i in scans)
|
|
|
|
content = f"""\
|
|
{_HEADER}
|
|
name: Harden
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
schedule:
|
|
- cron: "0 2 * * 0"
|
|
|
|
jobs:
|
|
build-scan-report:
|
|
runs-on: {runner}
|
|
steps:
|
|
{_checkout_step()}
|
|
|
|
{_docker_login_step(registry)}
|
|
|
|
{_install_trivy_step()}
|
|
|
|
{_setup_uv_step(uv_version)}
|
|
|
|
{build_block}
|
|
|
|
{scan_block}
|
|
|
|
- name: Close resolved or file new vuln issues
|
|
env:
|
|
GITEA_TOKEN: ${{{{ secrets.DEPLOY_TOKEN }}}}
|
|
run: |
|
|
uv sync --no-dev
|
|
for f in {vuln_files}; do
|
|
if [ -f "$f" ]; then
|
|
uv run python -m api.diag.vuln --close "$f" || \\
|
|
uv run python -m api.diag.vuln "$f" || true
|
|
fi
|
|
done
|
|
|
|
{_failure_step("Harden", "build-scan-report")}
|
|
"""
|
|
return (".gitea/workflows/harden.yml", content)
|
|
|
|
|
|
def _gen_rebuild_all(
|
|
images: list[dict],
|
|
scans: list[dict],
|
|
registry: str,
|
|
owner_repo: str,
|
|
runner: str,
|
|
uv_version: str,
|
|
**_kw: object,
|
|
) -> tuple[str, str]:
|
|
build_steps = []
|
|
for img in images:
|
|
tags = (
|
|
f"{registry}/{owner_repo}/{img['name']}:${{{{ env.SHORT_SHA }}}},"
|
|
f"{registry}/{owner_repo}/{img['name']}:latest"
|
|
)
|
|
build_steps.append(_build_push_step(img, tags, registry, owner_repo))
|
|
build_block = "\n\n".join(build_steps)
|
|
|
|
scan_steps = []
|
|
for img in scans:
|
|
scan_steps.append(
|
|
_trivy_step(img, "${{ env.SHORT_SHA }}", registry, owner_repo)
|
|
)
|
|
scan_block = "\n\n".join(scan_steps)
|
|
|
|
vuln_files = " ".join(f"{i['name']}-scan.json" for i in scans)
|
|
|
|
content = f"""\
|
|
{_HEADER}
|
|
name: Rebuild All
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
|
|
jobs:
|
|
build-scan-report:
|
|
runs-on: {runner}
|
|
steps:
|
|
{_checkout_step()}
|
|
|
|
{_docker_login_step(registry)}
|
|
|
|
{_install_trivy_step()}
|
|
|
|
{_setup_uv_step(uv_version)}
|
|
|
|
- name: Compute short SHA
|
|
run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
|
|
|
|
{build_block}
|
|
|
|
{scan_block}
|
|
|
|
- name: Report vulnerabilities
|
|
env:
|
|
GITEA_TOKEN: ${{{{ secrets.DEPLOY_TOKEN }}}}
|
|
run: |
|
|
uv sync --no-dev
|
|
for f in {vuln_files}; do
|
|
[ -f "$f" ] && uv run python -m api.diag.vuln "$f" || true
|
|
done
|
|
|
|
{_failure_step("Rebuild All", "build-scan-report")}
|
|
"""
|
|
return (".gitea/workflows/rebuild-all.yml", content)
|
|
|
|
|
|
def _gen_infra_ci(
|
|
images: list[dict],
|
|
runner: str,
|
|
**_kw: object,
|
|
) -> tuple[str, str]:
|
|
all_paths: list[str] = []
|
|
for img in images:
|
|
all_paths.extend(img.get("path_filter", []))
|
|
paths_block = "\n".join(f" - {p!r}" for p in all_paths)
|
|
|
|
jobs_parts = []
|
|
for img in images:
|
|
if not img.get("hadolint", True):
|
|
continue
|
|
name = img["name"]
|
|
# The notebooks image gets a frontend smoke gate: boot the freshly
|
|
# built image and load the editor in a headless browser, failing on
|
|
# any console/page error. Guards against runtime-broken frontend
|
|
# bundles that compile cleanly (the 2026-07-09 "d is not a
|
|
# constructor" incident shipped through a green build).
|
|
fe_smoke_step = ""
|
|
if name == "notebooks":
|
|
fe_smoke_step = f"""
|
|
|
|
- name: Frontend smoke gate
|
|
run: python3 dev/scripts/nb_fe_smoke.py --image local/{name}:build --network ci"""
|
|
jobs_parts.append(f"""\
|
|
{name}:
|
|
runs-on: {runner}
|
|
steps:
|
|
{_checkout_step()}
|
|
|
|
- name: Hadolint {name}
|
|
uses: https://github.com/hadolint/hadolint-action@v3.1.0
|
|
with:
|
|
dockerfile: {img["dockerfile"]}
|
|
|
|
{_setup_buildx_step()}
|
|
|
|
{_build_push_step(img, f"ci-test-{name}", "", "", load_only=True)}{fe_smoke_step}
|
|
|
|
{_failure_step("Infra CI", name)}""")
|
|
|
|
jobs_block = "\n\n".join(jobs_parts)
|
|
|
|
content = f"""\
|
|
{_HEADER}
|
|
name: Infra CI
|
|
|
|
on:
|
|
push:
|
|
paths:
|
|
{paths_block}
|
|
pull_request:
|
|
paths:
|
|
{paths_block}
|
|
|
|
jobs:
|
|
{jobs_block}
|
|
"""
|
|
return (".gitea/workflows/infra-ci.yml", content)
|
|
|
|
|
|
def _gen_notebooks_integration(
|
|
runner: str, uv_version: str, **_kw: object
|
|
) -> tuple[str, str]:
|
|
"""Nightly full-set notebook run inside the production container.
|
|
|
|
The full notebook set needs real data (aco.duckdb, ./data mounts) that
|
|
only the prod container has, so the scripts are docker-cp'd in and run
|
|
there. Failures are routed through nb_issue_filer (dedup + auto-close),
|
|
not exit codes — a red nightly run should page via the issue tracker,
|
|
not accumulate api.diag.ci duplicates. 03:30 sits before the 06:00
|
|
pkg-supply-chain run and outside interactive hours (duckdb lock, #508).
|
|
"""
|
|
content = f"""\
|
|
{_HEADER}
|
|
name: Notebooks Integration
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
schedule:
|
|
- cron: "30 3 * * *"
|
|
|
|
jobs:
|
|
notebooks-integration:
|
|
runs-on: {runner}
|
|
steps:
|
|
{_checkout_step()}
|
|
|
|
{_setup_uv_step(uv_version)}
|
|
|
|
- name: Run full notebook set in prod container
|
|
env:
|
|
GITEA_TOKEN: ${{{{ secrets.DEPLOY_TOKEN }}}}
|
|
run: |
|
|
set -euo pipefail
|
|
docker exec notebooks mkdir -p /tmp/nbtest
|
|
docker cp dev/scripts/nb_integration.py notebooks:/tmp/nbtest/
|
|
docker cp dev/scripts/nb_issue_filer.py notebooks:/tmp/nbtest/
|
|
docker cp infra/marimo/nb-tests.toml notebooks:/tmp/nbtest/
|
|
docker exec \\
|
|
-e GITEA_TOKEN \\
|
|
-e GITEA_API_BASE=http://git:3000/api/v1 \\
|
|
notebooks \\
|
|
uv run --project /home/kert/workspace python /tmp/nbtest/nb_integration.py \\
|
|
--set all --file-issues \\
|
|
--nb-dir /home/kert/notebooks \\
|
|
--config /tmp/nbtest/nb-tests.toml \\
|
|
--report /tmp/nbtest/report.json
|
|
docker cp notebooks:/tmp/nbtest/report.json nb-integration-report.json
|
|
cat nb-integration-report.json
|
|
|
|
- name: Frontend smoke against live service
|
|
run: python3 dev/scripts/nb_fe_smoke.py --url http://notebooks:2718 --network gateway
|
|
|
|
{_failure_step("Notebooks Integration", "notebooks-integration")}
|
|
"""
|
|
return (".gitea/workflows/notebooks-integration.yml", content)
|
|
|
|
|
|
def _gen_zotero_sync(runner: str, uv_version: str, **_kw: object) -> tuple[str, str]:
|
|
"""Nightly bib → Zotero routing for mail-ingested items.
|
|
|
|
mail-poller ingests CMS mail into bib continuously, but Zotero only
|
|
sees those items when `stack bib sync-zotero` pushes them — and that
|
|
had no scheduler, so the Cmsupdates collection silently froze for two
|
|
months (Apr 29 → Jul 10, 212-item backlog). sync writes directly into
|
|
zotero.sqlite, so the running Zotero app must be stopped for the
|
|
duration; the stop/start lives in workflow steps (visible, and
|
|
restart is if:always()) rather than the CLI's --hold-zotero, because
|
|
the sync runs docker-exec'd inside the api container, which has no
|
|
docker socket. 04:15 sits after the 03:30 notebooks-integration run
|
|
so the two never contend for zotero.sqlite.
|
|
"""
|
|
content = f"""\
|
|
{_HEADER}
|
|
name: Zotero Sync
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
schedule:
|
|
- cron: "15 4 * * *"
|
|
|
|
jobs:
|
|
zotero-sync:
|
|
runs-on: {runner}
|
|
steps:
|
|
{_checkout_step()}
|
|
|
|
{_setup_uv_step(uv_version)}
|
|
|
|
- name: Stop zotero (release SQLite write lock)
|
|
run: docker stop zotero
|
|
|
|
- name: Sync mail-ingested bib items into Zotero
|
|
# --merge-dupes: listserv re-sends are now aliased at ingest
|
|
# (Task 3), and any same-URL twins that still land in Zotero are
|
|
# merged (union tags/collections/notes/attachments, keep-earliest)
|
|
# rather than deleted.
|
|
run: |
|
|
docker exec api uv run --no-sync \\
|
|
stack bib sync-zotero --tag source:email --no-hold \\
|
|
--merge-dupes
|
|
|
|
- name: Sync Federal Register rule items into Zotero
|
|
# Rule items only ever reached Zotero via one-off manual syncs
|
|
# (P36 step 5); P37 skipped that step and the CY2027 NPRM never
|
|
# arrived (#626/#627). Tag-scoped: a full unscoped sync would
|
|
# hydrate ~180k bib items.
|
|
run: |
|
|
docker exec api uv run --no-sync \\
|
|
stack bib sync-zotero --tag source:federal-register --no-hold
|
|
|
|
- name: Restart zotero
|
|
if: always()
|
|
run: docker start zotero
|
|
|
|
{_failure_step("Zotero Sync", "zotero-sync")}
|
|
"""
|
|
return (".gitea/workflows/zotero-sync.yml", content)
|
|
|
|
|
|
def _gen_llm_golden(runner: str, uv_version: str, **_kw: object) -> tuple[str, str]:
|
|
"""Nightly golden longitudinal evaluation of the chat (P49 Task 7).
|
|
|
|
Mirrors ``_gen_notebooks_integration``: the golden set needs a live
|
|
chat over the real pgvector/DuckDB/Ollama stack, which only the
|
|
``llm`` container (``uvicorn llm.api:app``, port 8000, no published
|
|
port) has — so the runner and the golden set are docker-cp'd in and
|
|
run there against its own ``http://localhost:8000``. Regressions
|
|
are filed through ``nb_issue_filer`` (dedup + auto-close) under the
|
|
``llm`` label, not ``notebooks``. 03:40 sits after the 03:30
|
|
notebooks-integration run and before the 04:15 zotero-sync run.
|
|
"""
|
|
content = f"""\
|
|
{_HEADER}
|
|
name: LLM Golden
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
schedule:
|
|
- cron: "40 3 * * *"
|
|
|
|
jobs:
|
|
llm-golden:
|
|
runs-on: {runner}
|
|
steps:
|
|
{_checkout_step()}
|
|
|
|
{_setup_uv_step(uv_version)}
|
|
|
|
- name: Run golden longitudinal evaluation against the live chat
|
|
# golden_rc is captured (not left to `set -e`) so a failing golden
|
|
# run still docker-cps the report back and cats it — llm_golden.py
|
|
# itself exits 0 once --file-issues has filed the regressions
|
|
# (nb_issue_filer is the failure channel here), but this still
|
|
# guards the case where it exits nonzero before reaching that
|
|
# point (e.g. a crash while loading the golden set).
|
|
env:
|
|
GITEA_TOKEN: ${{{{ secrets.DEPLOY_TOKEN }}}}
|
|
run: |
|
|
set -euo pipefail
|
|
docker exec llm mkdir -p /tmp/golden
|
|
docker cp dev/scripts/llm_golden.py llm:/tmp/golden/
|
|
docker cp dev/scripts/nb_issue_filer.py llm:/tmp/golden/
|
|
docker cp tests/llm/golden_lineage.yaml llm:/tmp/golden/
|
|
docker exec \\
|
|
-e GITEA_TOKEN \\
|
|
-e GITEA_API_BASE=http://git:3000/api/v1 \\
|
|
-e NB_ISSUE_LABEL=llm \\
|
|
llm \\
|
|
uv run --no-sync --project /app python /tmp/golden/llm_golden.py run \\
|
|
--url http://localhost:8000 \\
|
|
--set /tmp/golden/golden_lineage.yaml \\
|
|
--report /tmp/golden/report.json \\
|
|
--file-issues --source nightly-llm-golden || golden_rc=$?
|
|
docker cp llm:/tmp/golden/report.json llm-golden-report.json || true
|
|
cat llm-golden-report.json || true
|
|
exit "${{golden_rc:-0}}"
|
|
|
|
{_failure_step("LLM Golden", "llm-golden")}
|
|
"""
|
|
return (".gitea/workflows/llm-golden.yml", content)
|
|
|
|
|
|
def _gen_bcda_vendor(runner: str, uv_version: str, **_kw: object) -> tuple[str, str]:
|
|
"""Weekly drift check of the sparsely vendored BCDA upstream (P52).
|
|
|
|
``dev/vendor/MANIFEST.toml`` pins CMSgov/bcda-app (the v3 handler
|
|
and its tests, request validation, the OperationOutcome writer,
|
|
the v3 Postman collection, gzip fixtures, sandbox config) and the
|
|
bcda.cms.gov v3 data dictionary + sample files that
|
|
``bcda.express.v3`` is tested against. ``--check`` compares the
|
|
pinned blob/sha256 digests with upstream and exits 1 on drift; the
|
|
failure step files the issue. Nothing is auto-committed — a
|
|
payload change upstream is something a human reads before the
|
|
flattener is re-pointed (``vendor_bcda_app.py --sync`` then
|
|
``gen_bcda_v3_crosswalk.py``). Standard library only, so it runs
|
|
before ``uv sync``.
|
|
"""
|
|
content = f"""\
|
|
{_HEADER}
|
|
name: BCDA Vendor
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
schedule:
|
|
- cron: "30 5 * * 1"
|
|
|
|
jobs:
|
|
bcda-vendor:
|
|
runs-on: {runner}
|
|
steps:
|
|
{_checkout_step()}
|
|
|
|
{_setup_uv_step(uv_version)}
|
|
|
|
- name: Verify the vendored tree matches the manifest
|
|
run: uv run --no-project python dev/scripts/vendor_bcda_app.py --verify
|
|
|
|
- name: Check CMSgov/bcda-app and bcda.cms.gov for drift
|
|
run: uv run --no-project python dev/scripts/vendor_bcda_app.py --check
|
|
|
|
{_failure_step("BCDA Vendor", "bcda-vendor")}
|
|
"""
|
|
return (".gitea/workflows/bcda-vendor.yml", content)
|
|
|
|
|
|
def _gen_release(runner: str, uv_version: str, **_kw: object) -> tuple[str, str]:
|
|
content = f"""\
|
|
{_HEADER}
|
|
name: Release
|
|
|
|
on:
|
|
push:
|
|
tags: ["v*"]
|
|
|
|
jobs:
|
|
release:
|
|
runs-on: {runner}
|
|
steps:
|
|
{_checkout_step()}
|
|
|
|
{_setup_uv_step(uv_version)}
|
|
|
|
- name: Build package
|
|
run: uv build --out-dir dist/
|
|
|
|
- name: Create release
|
|
uses: https://github.com/softprops/action-gh-release@v2
|
|
with:
|
|
files: |
|
|
dist/*.whl
|
|
dist/*.tar.gz
|
|
|
|
{_failure_step("Release", "release")}
|
|
"""
|
|
return (".gitea/workflows/release.yml", content)
|
|
|
|
|
|
# ── Public API ────────────────────────────────────────────────────
|
|
|
|
|
|
def emit(
|
|
images: list[dict],
|
|
scans: list[dict],
|
|
platform: dict,
|
|
ci_cfg: dict,
|
|
) -> dict[str, str]:
|
|
"""Return {relative_path: content} for all Gitea Actions workflows."""
|
|
# ci_registry is the internal HTTP endpoint used for push (avoids TLS)
|
|
registry = ci_cfg.get("ci_registry", platform["registry"])
|
|
owner_repo = platform["repo"]
|
|
runner = ci_cfg.get("runner_labels", ["ubuntu-latest"])[0]
|
|
uv_version = ci_cfg.get("uv_version", "latest")
|
|
|
|
common = dict(
|
|
images=images,
|
|
scans=scans,
|
|
registry=registry,
|
|
owner_repo=owner_repo,
|
|
runner=runner,
|
|
uv_version=uv_version,
|
|
coverage_threshold=ci_cfg.get("coverage_threshold", 99),
|
|
)
|
|
|
|
files: dict[str, str] = {}
|
|
for gen_fn in (
|
|
_gen_ci,
|
|
_gen_deploy,
|
|
_gen_harden,
|
|
_gen_rebuild_all,
|
|
_gen_infra_ci,
|
|
_gen_notebooks_integration,
|
|
_gen_zotero_sync,
|
|
_gen_llm_golden,
|
|
_gen_bcda_vendor,
|
|
_gen_release,
|
|
):
|
|
path, content = gen_fn(**common) # type: ignore[arg-type]
|
|
files[path] = content
|
|
|
|
return files
|