d726dca50e
All checks were successful
CI / lint (push) Successful in 40s
CI / notebooks-smoke (push) Successful in 1m37s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Has been skipped
Deploy / llm (push) Has been skipped
Deploy / mc (push) Has been skipped
Infra CI / notebooks (push) Successful in 1m5s
Infra CI / zotero (push) Successful in 13s
Infra CI / docs (push) Successful in 1m22s
Infra CI / api (push) Successful in 1m15s
Infra CI / llm (push) Successful in 44s
Infra CI / mc (push) Successful in 15s
Deploy / report (push) Successful in 13s
CI / test (push) Successful in 17m19s
Notebooks Integration / notebooks-integration (push) Successful in 7m27s
Zotero Sync / zotero-sync (push) Successful in 1m10s
Package Supply Chain / pkg-supply-chain (push) Successful in 59s
78 lines
2.6 KiB
Python
78 lines
2.6 KiB
Python
"""One-off migration: normalize already-downloaded FR ``.txt`` files.
|
|
|
|
Follow-up to issue #617. Before this, ``dev/scripts/fetch_fr_attachments.py``
|
|
stored the Federal Register API's raw HTML-wrapped TXT response
|
|
directly as the ``.txt`` attachment, so cleanup (strip the
|
|
``<html><pre>`` wrapper, inline markup, entities, control chars) ran at
|
|
every index run instead of once at capture time.
|
|
|
|
This walks ``data/fr_downloads/*.txt`` and, for every file that still
|
|
looks HTML-wrapped, saves the raw bytes to a sibling ``.html`` (skipped
|
|
if it already exists) and rewrites the ``.txt`` in place with
|
|
``rex.frtext.clean_fr_text`` — the same normalization
|
|
``fetch_fr_attachments.py`` now applies immediately after each fresh
|
|
download.
|
|
|
|
Usage::
|
|
|
|
uv run python dev/scripts/migrate_fr_txt.py --dry-run # preview
|
|
uv run python dev/scripts/migrate_fr_txt.py # migrate
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from rex.frtext import looks_html_wrapped, normalize_txt_file
|
|
|
|
DEFAULT_DIR = Path("data/fr_downloads")
|
|
|
|
|
|
def find_candidates(root: Path) -> list[Path]:
|
|
"""``*.txt`` files directly under *root* that still look HTML-wrapped."""
|
|
out = []
|
|
for path in sorted(root.glob("*.txt")):
|
|
raw = path.read_text(encoding="utf-8", errors="replace")
|
|
if looks_html_wrapped(raw):
|
|
out.append(path)
|
|
return out
|
|
|
|
|
|
def migrate_one(path: Path, *, dry_run: bool = False) -> dict:
|
|
"""Normalize one file. Returns a report row with before/after sizes."""
|
|
before = path.stat().st_size
|
|
if dry_run:
|
|
return {"path": path, "before": before, "after": before, "changed": False}
|
|
normalize_txt_file(path)
|
|
after = path.stat().st_size
|
|
return {"path": path, "before": before, "after": after, "changed": True}
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument(
|
|
"--dir", default=str(DEFAULT_DIR), help="Directory of *.txt to scan."
|
|
)
|
|
parser.add_argument("--dry-run", action="store_true")
|
|
args = parser.parse_args(argv)
|
|
|
|
root = Path(args.dir)
|
|
candidates = find_candidates(root)
|
|
if not candidates:
|
|
print(f"No HTML-wrapped .txt files found under {root}")
|
|
return 0
|
|
|
|
verb = "would rewrite" if args.dry_run else "rewrote"
|
|
print(f"{len(candidates)} HTML-wrapped file(s) under {root}:")
|
|
for path in candidates:
|
|
row = migrate_one(path, dry_run=args.dry_run)
|
|
print(f" [{verb}] {path.name}: {row['before']:,} -> {row['after']:,} bytes")
|
|
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|