f61eae8656
- _init_schema skips the duplicate-attachments GROUP BY scan once the unique index already exists (checks sqlite_master first). - upsert_status merges extra_json per key instead of replacing the whole column: a stored key now survives unless the incoming payload sets that specific key to a non-empty value. - docket_counts is one grouped query (a single items scan, left-joined against the enriched-tag set) instead of two separate url LIKE scans. - fingerprint_files widens `except FileNotFoundError` to `except OSError` so NotADirectoryError/PermissionError are handled the same way as a plain missing file. - backfill_details and backfill_from_mirror shared an identical sealed-guard block; factored into one _sealed_backfill_skip helper. Adds coverage: extra_json per-key merge (both directions), docket_counts, the widened OSError catch, attach_file's explicit-title dedupe path (two different source files, same explicit title), a dedupe-migration conflict test for a missing kept file, and a test confirming _init_schema's new short-circuit actually skips the scan.
90 lines
2.9 KiB
Python
90 lines
2.9 KiB
Python
"""Docket-level state for the regulations.gov comment pipeline.
|
|
|
|
A *docket* row (``dockets`` table in bib.sqlite) remembers what every
|
|
stage would otherwise re-derive from the network: the reg.gov document
|
|
that carries the comments, the comment close date, the pull watermark,
|
|
and — once the docket is known complete — a *seal*. Sealed dockets are
|
|
skipped by fetch, backfill, extract and index unless ``--force``.
|
|
|
|
Pure helpers only; the store owns persistence (``Store.docket_*``).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
from dataclasses import dataclass
|
|
from datetime import date, timedelta
|
|
from pathlib import Path
|
|
from typing import Iterable
|
|
|
|
DEFAULT_QUIET_DAYS = 30
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Docket:
|
|
id: str
|
|
rule_cms_id: str = ""
|
|
fr_document_id: str = ""
|
|
fr_object_id: str = ""
|
|
comment_end_date: str = "" # YYYY-MM-DD
|
|
pull_watermark: str = "" # max lastModifiedDate seen on a clean walk
|
|
last_pull_at: str = "" # ISO-8601 UTC
|
|
last_pull_new: int | None = None
|
|
sealed_at: str = ""
|
|
seal_reason: str = ""
|
|
counts_json: str = "{}"
|
|
|
|
@property
|
|
def sealed(self) -> bool:
|
|
return bool(self.sealed_at)
|
|
|
|
|
|
def should_seal(docket: Docket, today: date, quiet_days: int) -> bool:
|
|
"""True when the comment period closed ≥ *quiet_days* ago and the
|
|
most recent completed pull, itself after that quiet boundary, found
|
|
nothing new."""
|
|
if docket.sealed or not docket.comment_end_date or not docket.last_pull_at:
|
|
return False
|
|
if docket.last_pull_new is None or docket.last_pull_new > 0:
|
|
return False
|
|
try:
|
|
end = date.fromisoformat(docket.comment_end_date[:10])
|
|
pulled = date.fromisoformat(docket.last_pull_at[:10])
|
|
except ValueError:
|
|
return False
|
|
boundary = end + timedelta(days=quiet_days)
|
|
return today >= boundary and pulled >= boundary
|
|
|
|
|
|
def fingerprint_files(paths: Iterable[Path]) -> str:
|
|
"""sha256 over sorted ``(name, size, mtime_ns)`` — cheap change
|
|
detection without reading contents. Missing files contribute their
|
|
name with size -1 so a deletion changes the fingerprint too."""
|
|
rows: list[str] = []
|
|
for p in paths:
|
|
p = Path(p)
|
|
try:
|
|
st = p.stat()
|
|
rows.append(f"{p.name}\x00{st.st_size}\x00{st.st_mtime_ns}")
|
|
except OSError:
|
|
# Missing (FileNotFoundError), unreadable (PermissionError),
|
|
# or a broken symlink/removed parent dir — any of these
|
|
# means "not there" for fingerprinting purposes.
|
|
rows.append(f"{p.name}\x00-1\x000")
|
|
rows.sort()
|
|
return hashlib.sha256("\n".join(rows).encode()).hexdigest()
|
|
|
|
|
|
def _cfg():
|
|
from conf import cfg
|
|
|
|
return cfg
|
|
|
|
|
|
def quiet_days() -> int:
|
|
"""``[comments] seal_quiet_days`` from stack.toml, default 30."""
|
|
c = _cfg()
|
|
if "comments" in c and "seal_quiet_days" in c.comments:
|
|
return int(c.comments.seal_quiet_days)
|
|
return DEFAULT_QUIET_DAYS
|