A source link loads in the reviewer's browser instead of a new tab. Include stores the page when the site returns it and appends human focus highlights. An existing full text stays. Exclude stores nothing.
134 lines
4.3 KiB
Python
134 lines
4.3 KiB
Python
"""The review pane fetches a public page and will not follow a link onto this network."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import ipaddress
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from lead.explore.reader import ReaderError, _read_html, fetch, pdf_preview
|
|
|
|
|
|
def _client(handler) -> httpx.Client:
|
|
return httpx.Client(transport=httpx.MockTransport(handler))
|
|
|
|
|
|
def test_fetch_reads_html_and_keeps_a_real_link():
|
|
seen = {}
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
seen["ua"] = request.headers.get("user-agent")
|
|
if request.url.path == "/paper":
|
|
return httpx.Response(302, headers={"location": "https://1.1.1.1/final"})
|
|
body = (
|
|
b"<html><body><script>alert(1)</script>"
|
|
b"<p>The exposure precedes the outcome.</p>"
|
|
b"<a href='/a.pdf'>The PDF</a>"
|
|
b"<a href='javascript:alert(1)'>no</a></body></html>"
|
|
)
|
|
return httpx.Response(200, headers={"content-type": "text/html"}, content=body)
|
|
|
|
got = fetch("https://1.1.1.1/paper", client=_client(handler))
|
|
assert seen["ua"].startswith("stack-lead-review/")
|
|
assert got.url == "https://1.1.1.1/final"
|
|
assert got.kind == "html"
|
|
assert "The exposure precedes" in got.html
|
|
assert "alert" not in got.html
|
|
assert got.links == (("The PDF", "https://1.1.1.1/a.pdf"),)
|
|
|
|
|
|
def test_fetch_stops_when_a_redirect_lands_on_this_network():
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(302, headers={"location": "http://127.0.0.1/secret"})
|
|
|
|
with pytest.raises(ReaderError, match="this network"):
|
|
fetch("https://1.1.1.1/start", client=_client(handler))
|
|
|
|
|
|
def test_a_private_address_is_refused_before_a_request():
|
|
called = {"n": 0}
|
|
|
|
def handler(_request: httpx.Request) -> httpx.Response:
|
|
called["n"] += 1
|
|
return httpx.Response(200, content=b"no")
|
|
|
|
for url in (
|
|
"http://127.0.0.1/",
|
|
"http://10.1.1.1/",
|
|
"http://192.168.1.192/labels",
|
|
"http://169.254.169.254/latest",
|
|
"http://1.1.1.1:8501/",
|
|
"file:///etc/passwd",
|
|
"http://user:pass@1.1.1.1/",
|
|
):
|
|
with pytest.raises(ReaderError):
|
|
fetch(url, client=_client(handler))
|
|
assert called["n"] == 0
|
|
|
|
|
|
def test_a_hostname_that_resolves_inside_the_network_is_refused(monkeypatch):
|
|
def inside(*_args, **_kwargs):
|
|
return [(2, 1, 6, "", ("127.0.0.1", 0))]
|
|
|
|
monkeypatch.setattr("lead.explore.reader.socket.getaddrinfo", inside)
|
|
with pytest.raises(ReaderError, match="this network"):
|
|
fetch(
|
|
"https://example.test/paper", client=_client(lambda _r: httpx.Response(200))
|
|
)
|
|
|
|
|
|
def test_a_browser_check_is_not_stored_as_the_article():
|
|
def handler(_request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(
|
|
200,
|
|
headers={"content-type": "text/html"},
|
|
content=b"<html><title>Just a moment...</title><body>cloudflare</body></html>",
|
|
)
|
|
|
|
with pytest.raises(ReaderError, match="browser check"):
|
|
fetch("https://1.1.1.1/paper", client=_client(handler))
|
|
|
|
|
|
def test_a_pdf_is_kept_as_bytes():
|
|
def handler(_request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(
|
|
200,
|
|
headers={"content-type": "application/pdf"},
|
|
content=b"%PDF-1.4\npaper\n",
|
|
)
|
|
|
|
got = fetch("https://1.1.1.1/a.pdf", client=_client(handler))
|
|
assert got.kind == "pdf"
|
|
assert got.body.startswith(b"%PDF")
|
|
assert got.links == ()
|
|
|
|
|
|
def test_pdf_preview_renders_pages_and_rejects_garbage():
|
|
import io
|
|
|
|
import pypdfium2 as pdfium
|
|
|
|
document = pdfium.PdfDocument.new()
|
|
for _ in range(3):
|
|
document.new_page(80, 60)
|
|
buffer = io.BytesIO()
|
|
document.save(buffer)
|
|
document.close()
|
|
images, total = pdf_preview(buffer.getvalue(), limit=2, scale=1)
|
|
assert total == 3
|
|
assert len(images) == 2
|
|
assert images[0].startswith(b"\x89PNG")
|
|
with pytest.raises(ReaderError, match="could not be opened"):
|
|
pdf_preview(b"%PDF-1.4\nbody\n")
|
|
|
|
|
|
def test_read_html_resolves_a_relative_link_against_the_page():
|
|
html, links = _read_html(
|
|
"<p>Note</p><a href='files/a.pdf'>PDF</a>",
|
|
"https://1.1.1.1/dir/paper",
|
|
)
|
|
assert "Note" in html
|
|
assert links == (("PDF", "https://1.1.1.1/dir/files/a.pdf"),)
|
|
assert ipaddress.ip_address("1.1.1.1").is_global
|