71 lines
2.1 KiB
Python
71 lines
2.1 KiB
Python
"""Two screening passes and the 10% audit of agreed excludes."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
from typer.testing import CliRunner
|
|
|
|
from cli.prisma import app
|
|
from prisma.audit import advances, agreed_excludes, audit_sample, disagreements
|
|
from prisma.prompts import PROMPT_A, PROMPT_B, prompt_hash
|
|
|
|
runner = CliRunner()
|
|
|
|
|
|
def test_prompts_differ_and_hash_to_different_digests():
|
|
assert PROMPT_A != PROMPT_B
|
|
assert prompt_hash(PROMPT_A) != prompt_hash(PROMPT_B)
|
|
|
|
|
|
def test_disagreements_are_the_sorted_ids_whose_labels_differ():
|
|
a = {"i1": "include", "i2": "exclude"}
|
|
b = {"i1": "exclude", "i2": "exclude"}
|
|
assert disagreements(a, b) == ("i1",)
|
|
assert disagreements(
|
|
{"b": "include", "a": "exclude"},
|
|
{"b": "exclude", "a": "include"},
|
|
) == ("a", "b")
|
|
|
|
|
|
def test_an_id_in_only_one_pass_raises():
|
|
with pytest.raises(KeyError):
|
|
disagreements({"i1": "include"}, {"i1": "include", "i2": "exclude"})
|
|
|
|
|
|
def test_agreed_excludes_are_ids_both_passes_exclude():
|
|
a = {"i1": "include", "i2": "exclude"}
|
|
b = {"i1": "exclude", "i2": "exclude"}
|
|
assert agreed_excludes(a, b) == ("i2",)
|
|
|
|
|
|
def test_agreed_excludes_require_the_same_ids():
|
|
with pytest.raises(KeyError):
|
|
agreed_excludes({"i1": "exclude"}, {"i1": "exclude", "i2": "exclude"})
|
|
|
|
|
|
def test_audit_sample_draws_one_when_ten_percent_floors_to_zero():
|
|
ids = [f"i{n}" for n in range(9)]
|
|
drawn = audit_sample(ids, rate=0.10, seed=1)
|
|
assert len(drawn) == 1
|
|
assert drawn[0] in ids
|
|
|
|
|
|
def test_uncertain_and_include_advance_exclude_does_not():
|
|
assert advances("uncertain") is True
|
|
assert advances("include") is True
|
|
assert advances("exclude") is False
|
|
|
|
|
|
def test_audit_sample_is_ten_percent_and_stable_for_a_seed():
|
|
ids = [f"i{n}" for n in range(10)]
|
|
first = audit_sample(ids, rate=0.10, seed=1)
|
|
again = audit_sample(list(reversed(ids)), rate=0.10, seed=1)
|
|
assert len(first) == 1
|
|
assert first == again
|
|
assert audit_sample(ids, rate=0, seed=1) == ()
|
|
|
|
|
|
def test_audit_command_is_registered():
|
|
result = runner.invoke(app, ["audit", "--help"])
|
|
assert result.exit_code == 0
|