b1e06b1b9e
OEWS: the BLS API serves only the latest OEWS year, so history comes
from the annual oesm{yy}nat.zip files (2013–2024), which bls.gov only
serves to a full browser header set; each zip's national_M{year}_dl
sheet (xls through 2013, xlsx after; OCC_GROUP through 2018, O_GROUP
plus area/industry columns from 2019) is normalised to one row per SOC
occupation with employment and the hourly/annual wage distribution,
suppressed cells (* # **) as NULL, and the BLS series-id prefix kept
per row. One bib Source per release (module:bls, table:bls.oews,
source:bls-website, year:N) and a cms.ingest_log row with the zip's
sha256.
CES: a closed registry of employment (01) and average hourly earnings
(03) series for health care, ambulatory care, offices of physicians,
home health, hospitals and nursing/residential care (CES65<industry>
<type>), pulled through the public API in 10-year × 25-series chunks
(20 × 50 with BLS_API_KEY), monthly rows with the series id on every
row, one bib Source per series.
stack bls oews [--year]... [--write [--offline]] [--occ]...;
stack bls ces [--write] [--series]... [--annual]. Notebook 7f overlays
clinical-staff wages (31-9092, 29-2061, 29-1141, 29-1171) and
physician-office / ambulatory employment on the family's exposure
dates. CLI docs regenerated.
235 lines
6.9 KiB
Python
235 lines
6.9 KiB
Python
"""stack bls — oews / ces (#695)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import io
|
|
import zipfile
|
|
from unittest.mock import MagicMock
|
|
|
|
import duckdb
|
|
import openpyxl
|
|
import pytest
|
|
from typer.testing import CliRunner
|
|
|
|
import cli.bls as bls_cli
|
|
from bib.store import Store
|
|
from cli import app
|
|
|
|
runner = CliRunner()
|
|
|
|
COLS = [
|
|
"OCC_CODE",
|
|
"OCC_TITLE",
|
|
"OCC_GROUP",
|
|
"TOT_EMP",
|
|
"EMP_PRSE",
|
|
"H_MEAN",
|
|
"A_MEAN",
|
|
"MEAN_PRSE",
|
|
"H_PCT10",
|
|
"H_PCT25",
|
|
"H_MEDIAN",
|
|
"H_PCT75",
|
|
"H_PCT90",
|
|
"A_PCT10",
|
|
"A_PCT25",
|
|
"A_MEDIAN",
|
|
"A_PCT75",
|
|
"A_PCT90",
|
|
"ANNUAL",
|
|
"HOURLY",
|
|
]
|
|
ROWS = [
|
|
[
|
|
"31-9092",
|
|
"Medical Assistants",
|
|
"detailed",
|
|
571690,
|
|
0.8,
|
|
"14.8",
|
|
30780,
|
|
0.3,
|
|
"10.23",
|
|
"12.09",
|
|
"14.24",
|
|
"17.12",
|
|
"20.5",
|
|
21270,
|
|
25150,
|
|
29610,
|
|
35610,
|
|
42650,
|
|
None,
|
|
None,
|
|
]
|
|
]
|
|
|
|
|
|
def _zip_bytes(year: int) -> bytes:
|
|
wb = openpyxl.Workbook()
|
|
ws = wb.active
|
|
ws.append(COLS)
|
|
for r in ROWS:
|
|
ws.append(r)
|
|
xb = io.BytesIO()
|
|
wb.save(xb)
|
|
zb = io.BytesIO()
|
|
with zipfile.ZipFile(zb, "w") as z:
|
|
z.writestr(f"oesm{year % 100:02d}nat/national_M{year}_dl.xlsx", xb.getvalue())
|
|
return zb.getvalue()
|
|
|
|
|
|
@pytest.fixture
|
|
def con(monkeypatch, tmp_path):
|
|
c = duckdb.connect(":memory:")
|
|
|
|
class _Batch:
|
|
def __enter__(self):
|
|
return c
|
|
|
|
def __exit__(self, *a):
|
|
return False
|
|
|
|
monkeypatch.setattr(bls_cli, "_batch", lambda: _Batch())
|
|
monkeypatch.setattr(bls_cli, "_read", lambda: c.cursor())
|
|
store = Store(":memory:", storage_dir=tmp_path / "st")
|
|
monkeypatch.setattr(bls_cli, "_store", lambda: store)
|
|
published: list[bool] = []
|
|
monkeypatch.setattr(bls_cli, "_publish", lambda: published.append(True))
|
|
monkeypatch.setattr(bls_cli, "_http", lambda: MagicMock())
|
|
monkeypatch.setattr(
|
|
bls_cli,
|
|
"oews_cache_path",
|
|
lambda y, root=None: tmp_path / f"oesm{y % 100:02d}nat.zip",
|
|
)
|
|
|
|
def fake_download(client, year, dest):
|
|
dest.write_bytes(_zip_bytes(year))
|
|
return dest
|
|
|
|
monkeypatch.setattr(bls_cli, "download_oews", fake_download)
|
|
try:
|
|
yield c, store, published, tmp_path
|
|
finally:
|
|
store.close()
|
|
c.close()
|
|
|
|
|
|
class TestOews:
|
|
def test_write_loads_cites_logs_publishes(self, con):
|
|
c, store, published, tmp_path = con
|
|
res = runner.invoke(
|
|
app, ["bls", "oews", "--write", "--year", "2013", "--year", "2018"]
|
|
)
|
|
assert res.exit_code == 0, res.output
|
|
assert c.execute(
|
|
"SELECT count(*), count(distinct year) FROM bls.oews"
|
|
).fetchone() == (2, 2)
|
|
keys = [
|
|
r[0]
|
|
for r in c.execute(
|
|
"SELECT DISTINCT item_key FROM bls.oews ORDER BY 1"
|
|
).fetchall()
|
|
]
|
|
assert len(keys) == 2 and all(
|
|
"year:" in " ".join(store.get(k).tags) for k in keys
|
|
)
|
|
logs = c.execute(
|
|
"SELECT module, rows, source_file FROM cms.ingest_log ORDER BY source_file"
|
|
).fetchall()
|
|
assert [l[0] for l in logs] == ["bls.oews", "bls.oews"] and logs[0][2].endswith(
|
|
"oesm13nat.zip"
|
|
)
|
|
assert published == [True]
|
|
assert "2013: 1 occupations" in res.output
|
|
|
|
def test_offline_rerun_uses_cache_and_is_idempotent(self, con, monkeypatch):
|
|
c, store, published, tmp_path = con
|
|
runner.invoke(app, ["bls", "oews", "--write", "--year", "2013"])
|
|
monkeypatch.setattr(
|
|
bls_cli,
|
|
"download_oews",
|
|
lambda *a: (_ for _ in ()).throw(AssertionError("no download")),
|
|
)
|
|
res = runner.invoke(
|
|
app, ["bls", "oews", "--write", "--year", "2013", "--offline"]
|
|
)
|
|
assert res.exit_code == 0, res.output
|
|
assert c.execute("SELECT count(*) FROM bls.oews").fetchone()[0] == 1
|
|
|
|
def test_occ_reads_without_write_lock(self, con, monkeypatch):
|
|
c, store, published, tmp_path = con
|
|
runner.invoke(app, ["bls", "oews", "--write", "--year", "2013"])
|
|
published.clear()
|
|
monkeypatch.setattr(
|
|
bls_cli, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
|
|
)
|
|
res = runner.invoke(app, ["bls", "oews", "--occ", "31-9092"])
|
|
assert res.exit_code == 0, res.output
|
|
assert "31-9092 2013" in res.output and "$30,780.00/yr" in res.output
|
|
assert published == []
|
|
|
|
def test_unknown_year_rejected(self, con):
|
|
res = runner.invoke(app, ["bls", "oews", "--write", "--year", "1999"])
|
|
assert res.exit_code != 0
|
|
|
|
def test_no_table_yet(self, monkeypatch):
|
|
bare = duckdb.connect(":memory:")
|
|
monkeypatch.setattr(bls_cli, "_read", lambda: bare)
|
|
try:
|
|
res = runner.invoke(app, ["bls", "oews", "--occ", "31-9092"])
|
|
assert res.exit_code == 0 and "no bls.oews yet" in res.output
|
|
finally:
|
|
bare.close()
|
|
|
|
|
|
class TestCes:
|
|
def test_write_and_read(self, con, monkeypatch):
|
|
c, store, published, tmp_path = con
|
|
seen = {}
|
|
|
|
def fake_fetch(client, ids, start, end, *, api_key=""):
|
|
seen["ids"] = list(ids)
|
|
seen["key"] = api_key
|
|
return {
|
|
sid: [
|
|
{"year": "2024", "period": f"M{m:02d}", "value": str(m)}
|
|
for m in (1, 2)
|
|
]
|
|
for sid in ids
|
|
}
|
|
|
|
monkeypatch.setattr(bls_cli, "fetch_ces", fake_fetch)
|
|
monkeypatch.setenv("BLS_API_KEY", "abc")
|
|
res = runner.invoke(app, ["bls", "ces", "--write"])
|
|
assert res.exit_code == 0, res.output
|
|
assert seen["key"] == "abc" and "CES6562110001" in seen["ids"]
|
|
n_series = len(bls_cli.CES_SERIES)
|
|
assert c.execute("SELECT count(*) FROM bls.ces").fetchone()[0] == 2 * n_series
|
|
assert c.execute("SELECT module, rows FROM cms.ingest_log").fetchone() == (
|
|
"bls.ces",
|
|
2 * n_series,
|
|
)
|
|
assert published == [True]
|
|
assert f"ces: {2 * n_series} observations, {n_series} series" in res.output
|
|
|
|
published.clear()
|
|
monkeypatch.setattr(
|
|
bls_cli, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
|
|
)
|
|
res = runner.invoke(
|
|
app, ["bls", "ces", "--series", "CES6562110001", "--annual"]
|
|
)
|
|
assert res.exit_code == 0, res.output
|
|
assert "CES6562110001 2024 2 Offices of physicians employment" in res.output
|
|
assert published == []
|
|
|
|
def test_no_table_yet(self, monkeypatch):
|
|
bare = duckdb.connect(":memory:")
|
|
monkeypatch.setattr(bls_cli, "_read", lambda: bare)
|
|
try:
|
|
res = runner.invoke(app, ["bls", "ces"])
|
|
assert res.exit_code == 0 and "no bls.ces yet" in res.output
|
|
finally:
|
|
bare.close()
|