homelab/stack
1
0
Fork 0
Code Issues 85 Pull Requests 2 Actions Packages Projects 1 Releases Wiki Activity

fix(bib): tag attachment:gone when CDN serves 200 + empty body

Browse Source 
regulations.gov sometimes returns 200 OK with Content-Length: 0 for
attachment URLs whose files have been removed from object storage but
whose comment metadata still references them. download_attachment was
silently writing the 0-byte response to disk, registering it as a real
attachment, and refusing to refetch on retry (cache hit on the empty
file). Five comments in bib.sqlite ended up with phantom attachments.

- download_attachment raises new AttachmentMissing on 0-byte body and
  rejects cached 0-byte files instead of returning the stale path
- download_attachments_batch returns (paths, missing_urls); transient
  HTTP/transport failures are still untracked and get retried next run
- backfill_details tags affected items 'attachment:gone' (mirrors the
  existing 'enriched:gone' convention for 404'd comments) and counts
  them in a new att_gone stats field

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
kert
2026-04-27 15:04:58 -04:00
parent fde89c2328
commit dddd325153
2 changed files with 133 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
src/bib/regulations_gov.py
View File

@@ -44,6 +44,16 @@ log = logging.getLogger(__name__)
_BASE = "https://api.regulations.gov/v4"
class AttachmentMissing(Exception):
"""CDN returned 200 OK with an empty body for an attachment URL.
Means the file has been removed from object storage but the comment
metadata still references it — permanently unrecoverable. Distinct
from transient HTTP/transport errors, which get no exception and
are retried on the next backfill run.
"""
# ── Value objects ──────────────────────────────────────────────
@@ -297,7 +307,7 @@ class Client:
dest_dir.mkdir(parents=True, exist_ok=True)
filename = _filename_from(url) or _hash(url) + ".bin"
dest = dest_dir / filename
if dest.is_file() and not overwrite:
if dest.is_file() and dest.stat().st_size > 0 and not overwrite:
return dest
# No sleep — CDN is separate from the API rate limit.
try:
@@ -321,28 +331,43 @@ class Client:
except httpx.HTTPError as e:
log.warning("attachment %s failed: %s", url, e)
return None
if dest.stat().st_size == 0:
log.warning("attachment %s → empty body, skipping", url.rsplit("/", 1)[-1])
dest.unlink()
raise AttachmentMissing(url)
return dest
def download_attachments_batch(
self,
urls: list[str],
dest_dir: Path,
) -> list[Path]:
"""Download multiple attachments concurrently. Returns paths of
successfully downloaded files."""
) -> tuple[list[Path], list[str]]:
"""Download multiple attachments concurrently.
Returns ``(paths, missing_urls)`` where ``missing_urls`` are URLs
the CDN served as 200 OK with an empty body — verified gone from
object storage. Transient failures (HTTP errors, timeouts) are
not in either list; they get retried on the next backfill run.
"""
if not urls:
return []
return [], []
results: list[Path] = []
missing: list[str] = []
with ThreadPoolExecutor(max_workers=self._dl_workers) as pool:
futures = {
pool.submit(self.download_attachment, url, dest_dir): url
for url in urls
}
for fut in as_completed(futures):
path = fut.result()
url = futures[fut]
try:
path = fut.result()
except AttachmentMissing:
missing.append(url)
continue
if path:
results.append(path)
return results
return results, missing
# ── Bib integration ────────────────────────────────────────────
@@ -395,7 +420,14 @@ def backfill_details(
+ (f" LIMIT {int(limit)}" if limit else "")
).fetchall()
stats = {"enriched": 0, "attached": 0, "gone": 0, "errors": 0, "seen": len(rows)}
stats = {
"enriched": 0,
"attached": 0,
"att_gone": 0,
"gone": 0,
"errors": 0,
"seen": len(rows),
}
def _write_log(msg: str) -> None:
log.info(msg)
@@ -456,13 +488,23 @@ def backfill_details(
att_url = f.get("fileUrl") or ""
if att_url:
att_urls.append(att_url)
for path in client.download_attachments_batch(att_urls, dest_dir):
paths: list[Path] = []
missing_urls: list[str] = []
if att_urls:
paths, missing_urls = client.download_attachments_batch(att_urls, dest_dir)
for path in paths:
try:
store.attach_file(key, path, title=path.name)
stats["attached"] += 1
except Exception as e: # noqa: BLE001
# Usually a dup filename; fine to skip.
log.debug("attach_file skipped for %s/%s: %s", key, path.name, e)
if missing_urls:
try:
store.add_tag(key, "attachment:gone")
stats["att_gone"] += 1
except Exception as e: # noqa: BLE001
log.warning("add_tag attachment:gone failed for %s: %s", key, e)
try:
store.add_tag(key, "enriched:ok")
@@ -475,6 +517,7 @@ def backfill_details(
f" {i}/{len(rows)} "
f"enriched={stats['enriched']} "
f"attached={stats['attached']} "
f"att_gone={stats['att_gone']} "
f"gone={stats['gone']} errors={stats['errors']}"
)

82
tests/bib/test_regulations_gov_exercise.py
View File

@@ -246,13 +246,51 @@ class TestBackfillDetails:
}
],
}
api.download_attachments_batch.return_value = [tmp_path / "file.pdf"]
api.download_attachments_batch.return_value = ([tmp_path / "file.pdf"], [])
stats = backfill_details(
store, api, limit=1, scratch_root=tmp_path, commit_every=1
)
assert stats["attached"] >= 1
def test_attachment_gone_tags_item(self, tmp_path):
"""CDN serves 200 OK with empty body → item gets attachment:gone tag."""
store = MagicMock()
con = MagicMock()
store._con.return_value = con
con.execute.return_value.fetchall.return_value = [
{
"id": 1,
"key": "K1",
"url": "https://www.regulations.gov/comment/CMS-2023-0001-0001",
},
]
api = MagicMock()
api.get_comment_detail.return_value = {
"data": {"attributes": {"comment": "text", "organization": ""}},
"included": [
{
"type": "attachments",
"attributes": {
"fileFormats": [{"fileUrl": "https://example.com/gone.pdf"}]
},
}
],
}
api.download_attachments_batch.return_value = (
[],
["https://example.com/gone.pdf"],
)
stats = backfill_details(
store, api, limit=1, scratch_root=tmp_path, commit_every=1
)
assert stats["att_gone"] == 1
assert stats["attached"] == 0
store.add_tag.assert_any_call("K1", "attachment:gone")
store.add_tag.assert_any_call("K1", "enriched:ok")
def test_500_error_skips(self):
store = MagicMock()
con = MagicMock()
@@ -470,6 +508,7 @@ class TestBackfillDetailsOrgTag:
],
}
api.download_attachment.return_value = None # download fails
api.download_attachments_batch.return_value = ([], []) # all transient fails
stats = backfill_details(
store, api, limit=1, scratch_root=tmp_path, commit_every=1
@@ -544,6 +583,46 @@ class TestClientDownloadEdge:
result = c.download_attachment("https://x.com/new.pdf", tmp_path)
assert result is None
@patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": "test"})
@patch("bib.regulations_gov.time.sleep")
def test_empty_body_raises_attachment_missing(self, mc_sleep, tmp_path):
"""200 OK with 0-byte body → raises AttachmentMissing, deletes file."""
import pytest
from bib.regulations_gov import AttachmentMissing
mock_http = MagicMock()
stream_ctx = MagicMock()
stream_ctx.__enter__ = MagicMock(return_value=stream_ctx)
stream_ctx.__exit__ = MagicMock(return_value=False)
stream_ctx.status_code = 200
stream_ctx.iter_bytes.return_value = iter([]) # empty body
mock_http.stream.return_value = stream_ctx
c = Client(sleep=0, client=mock_http)
url = "https://x.com/empty.pdf"
with pytest.raises(AttachmentMissing):
c.download_attachment(url, tmp_path)
assert not (tmp_path / "empty.pdf").exists()
@patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": "test"})
@patch("bib.regulations_gov.time.sleep")
def test_existing_zero_byte_file_re_downloads(self, mc_sleep, tmp_path):
"""Cached 0-byte file is rejected — refetch happens instead of returning the stale empty path."""
f = tmp_path / "stale.pdf"
f.write_bytes(b"") # 0 bytes
mock_http = MagicMock()
stream_ctx = MagicMock()
stream_ctx.__enter__ = MagicMock(return_value=stream_ctx)
stream_ctx.__exit__ = MagicMock(return_value=False)
stream_ctx.status_code = 200
stream_ctx.iter_bytes.return_value = iter([b"real bytes"])
mock_http.stream.return_value = stream_ctx
c = Client(sleep=0, client=mock_http)
result = c.download_attachment("https://x.com/stale.pdf", tmp_path)
assert result == f
assert f.read_bytes() == b"real bytes"
mock_http.stream.assert_called_once()
class TestIterCommentsPageAdvance:
@patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": "test"})
@@ -689,6 +768,7 @@ class TestBackfillAddTagGone:
],
}
api.download_attachment.return_value = tmp_path / "doc.pdf"
api.download_attachments_batch.return_value = ([tmp_path / "doc.pdf"], [])
stats = backfill_details(
store, api, limit=1, scratch_root=tmp_path, commit_every=1