832bfd511f
The API backfill enriches ~970 comments/hr, so 15k new comments meant a next-day index. The Mirrulations mirror is a superset of the live docket (18,651 vs 18,647 comments on 2026-09-08) and runs ~600/min, creating, tagging and enriching items identically. Run mirror → extract → index first, then the API list walk + backfill + incremental extract/index to catch anything the mirror lags on.
72 lines
3.2 KiB
Bash
Executable File
72 lines
3.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Post-comment-close re-farm for docket CMS-2026-2377 (CY2027 PFS NPRM,
|
|
# CMS-1848-P). Scheduled via host crontab for 2026-09-15 (comment period
|
|
# closes 2026-09-14 per 91 FR 43842 DATES block). Tracker: issue #615.
|
|
#
|
|
# Every step is resumable/incremental — safe to re-run on partial failure.
|
|
set -uo pipefail
|
|
cd "$(dirname "$0")/../.."
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
set -a; source .env; set +a
|
|
|
|
mkdir -p .state/comments
|
|
LOG=.state/comments/refarm-CMS-2026-2377.log
|
|
exec >>"$LOG" 2>&1
|
|
echo "=== re-farm run $(date -Is) ==="
|
|
|
|
FAILURES=0
|
|
|
|
step() { # step <label> <cmd...>: run, count failures, keep going
|
|
local label=$1; shift
|
|
echo "--- $label: $(date -Is)"
|
|
"$@" || { echo "WARN: $label rc=$?"; FAILURES=$((FAILURES+1)); }
|
|
}
|
|
|
|
# Fast path first: the Mirrulations S3 mirror (#662) is a superset of the
|
|
# live docket and has no rate cap (~1,500 comments/min vs the API's ~970/hr).
|
|
# It creates comments the farm never listed, enriches, attaches and tags
|
|
# them identically (reg-docket:/rule:/enriched:ok), so the index is usable
|
|
# minutes in rather than the next day.
|
|
step "mirror backfill" uv run stack bib backfill-comments --docket CMS-2026-2377 --mirror
|
|
step "extract" uv run stack comments extract --docket CMS-2026-2377
|
|
# extract-ocr is a phase-2 stub (always exits 2, no --docket) — see #253.
|
|
# ocr_needed attachments stay flagged in combined.md frontmatter for now.
|
|
step "index" uv run stack llm index --collection comments --docket CMS-2026-2377
|
|
|
|
# Gap-fill from the reg.gov API for anything the mirror lags on. All
|
|
# resumable: the list walk upserts, the backfill skips enriched:ok, and
|
|
# extract/index only touch new or changed items.
|
|
step "api fetch" uv run stack bib fetch-pfs-comments --docket CMS-2026-2377
|
|
step "api backfill" uv run stack bib backfill-comments --docket CMS-2026-2377
|
|
step "extract (gap)" uv run stack comments extract --docket CMS-2026-2377
|
|
step "index (gap)" uv run stack llm index --collection comments --docket CMS-2026-2377
|
|
|
|
COUNTS=$(uv run python - <<'EOF'
|
|
from conf import connect
|
|
s = connect.bib()
|
|
con = s._con()
|
|
total, enriched = con.execute(
|
|
"SELECT count(*), sum(CASE WHEN coalesce(abstract,'')<>'' THEN 1 ELSE 0 END) "
|
|
"FROM items WHERE url LIKE 'https://www.regulations.gov/comment/CMS-2026-2377-%'"
|
|
).fetchone()
|
|
print(f"comments={total} enriched={enriched}")
|
|
s.close()
|
|
EOF
|
|
) || { echo "WARN: counts rc=$?"; FAILURES=$((FAILURES+1)); }
|
|
echo "$COUNTS"
|
|
|
|
if [ "$FAILURES" -eq 0 ]; then
|
|
RESULT_MSG="Automated post-close re-farm ran $(date -I): $COUNTS."
|
|
else
|
|
RESULT_MSG="Automated post-close re-farm ran $(date -I) — completed with $FAILURES failed steps — check log. $COUNTS."
|
|
fi
|
|
|
|
curl -sf -X POST \
|
|
"https://git.fhirworx.io/api/v1/repos/homelab/stack/issues/615/comments" \
|
|
-H "Authorization: token $GITEA_TOKEN" \
|
|
-H "Content-Type: application/json" \
|
|
-H "User-Agent: curl/8.5.0" \
|
|
-d "{\"body\": \"$RESULT_MSG Log: .state/comments/refarm-CMS-2026-2377.log. Review counts, spot-check indexing, then close this issue and milestone P37.\n\nIf complete: close #615, close milestone P37, and REMOVE the crontab line (re-fires annually).\"}" \
|
|
|| echo "WARN: gitea comment failed"
|
|
echo "=== done $(date -Is) ==="
|