diff --git a/moli-benchmark/moli_benchmark/wpt_cross/case_set.py b/moli-benchmark/moli_benchmark/wpt_cross/case_set.py index b68cc41459..c90de1d023 100644 --- a/moli-benchmark/moli_benchmark/wpt_cross/case_set.py +++ b/moli-benchmark/moli_benchmark/wpt_cross/case_set.py @@ -751,6 +751,32 @@ def _script_content_type_handler_reference_patterns(directory: str) -> tuple[re. ) + +@lru_cache(maxsize=None) +def _fetch_preflight_handler_reference_patterns(directory: str) -> tuple[re.Pattern[str], ...]: + names = ("preflight.py", "clean-stash.py", "inspect-headers.py") + references = [] + for name in names: + resource = "fetch/api/resources/" + name + relative = posixpath.relpath(resource, directory) + references.extend(("/" + resource, relative, "./" + relative)) + patterns = [ + re.compile( + rf"(? tuple[re.Pattern[str], ...]: resource = ( @@ -866,6 +892,8 @@ def _supported_wptserve_handler_references( supported += SUPPORTED_FETCH_ABORT_WPTSERVE_HANDLER_PATTERNS if rel is not None and rel.rsplit("/", 1)[0] == "fetch/range": supported += SUPPORTED_FETCH_RANGE_WPTSERVE_HANDLER_PATTERNS + if rel is not None and rel.startswith("fetch/api/"): + supported += _fetch_preflight_handler_reference_patterns(posixpath.dirname(rel)) if rel is not None and rel.startswith("wasm/webapi/"): supported += SUPPORTED_WASM_WEBAPI_WPTSERVE_HANDLER_PATTERNS if rel is not None: diff --git a/moli-benchmark/moli_benchmark/wpt_cross/server.py b/moli-benchmark/moli_benchmark/wpt_cross/server.py index 79d90ce589..a9061c14e3 100644 --- a/moli-benchmark/moli_benchmark/wpt_cross/server.py +++ b/moli-benchmark/moli_benchmark/wpt_cross/server.py @@ -147,6 +147,10 @@ NAVIGATION_SECOND_VISIT_PATH = ( "/navigation-api/navigation-methods/return-value/resources/" "204-205-download-on-second-visit.py" ) +FETCH_PREFLIGHT_RESOURCE_PATHS = { + "/fetch/api/resources/preflight.py", + "/fetch/api/resources/clean-stash.py", +} BENCH_TIMEOUT_MULTIPLIER_QUERY = "__moli_bench_timeout_multiplier" FORM_ECHO_PATH = "/html/semantics/forms/form-submission-0/form-echo.py" FORM_SUBMISSION_PATH = ( @@ -1832,12 +1836,16 @@ class FetchStash: def __init__(self) -> None: self._lock = threading.Lock() - self._values: dict[tuple[str, uuid.UUID], str] = {} + self._values: dict[tuple[str, uuid.UUID], object] = {} self._counts: dict[tuple[str, uuid.UUID], int] = {} - def put(self, key: str, value: str, *, path: str = "/fetch/api/resources/", - overwrite: bool = False) -> None: + def put( + self, key: str, value: object, *, overwrite: bool = False, + path: str = "/fetch/api/resources/", + ) -> None: parsed_key = (path, uuid.UUID(key)) + if value is None: + raise ValueError("Shared stash values cannot be None") with self._lock: if not overwrite and parsed_key in self._values: raise ValueError("Tried to overwrite existing shared stash value") @@ -1851,7 +1859,7 @@ class FetchStash: return value - def take(self, key: str, *, path: str = "/fetch/api/resources/") -> str | None: + def take(self, key: str, *, path: str = "/fetch/api/resources/") -> object: parsed_key = (path, uuid.UUID(key)) with self._lock: return self._values.pop(parsed_key, None) @@ -1946,6 +1954,8 @@ def _make_handler( return self._serve_fetch_resource_method if path in XHR_RESOURCE_PATHS: return self._serve_xhr_method + if path in FETCH_PREFLIGHT_RESOURCE_PATHS: + return self._serve_fetch_resource_method raise AttributeError(name) def _serve_xhr_method(self) -> None: @@ -1975,7 +1985,7 @@ def _make_handler( if path == NAVIGATION_SECOND_VISIT_PATH: self._serve_navigation_second_visit() return - if path in FETCH_ABORT_RESOURCE_PATHS | FETCH_RANGE_RESOURCE_PATHS | { + if path in FETCH_ABORT_RESOURCE_PATHS | FETCH_RANGE_RESOURCE_PATHS | FETCH_PREFLIGHT_RESOURCE_PATHS | { "/fetch/api/resources/status.py", "/fetch/api/resources/trickle.py" }: self._serve_fetch_resource_method() @@ -2012,7 +2022,7 @@ def _make_handler( if path == NAVIGATION_SECOND_VISIT_PATH: self._serve_navigation_second_visit() return - if path in FETCH_ABORT_RESOURCE_PATHS | FETCH_RANGE_RESOURCE_PATHS | { + if path in FETCH_ABORT_RESOURCE_PATHS | FETCH_RANGE_RESOURCE_PATHS | FETCH_PREFLIGHT_RESOURCE_PATHS | { "/fetch/api/resources/status.py", "/fetch/api/resources/trickle.py" }: self._serve_fetch_resource_method() @@ -2093,6 +2103,11 @@ def _make_handler( if path in FETCH_RANGE_RESOURCE_PATHS: self._serve_fetch_range_resource(path, parsed.query, emit_body=self.command != "HEAD") return + if unquote(parsed.path) in FETCH_PREFLIGHT_RESOURCE_PATHS: + self._serve_fetch_preflight_resource( + unquote(parsed.path), parsed.query, emit_body=self.command != "HEAD" + ) + return if unquote(parsed.path) in FETCH_ABORT_RESOURCE_PATHS: self._serve_fetch_abort_resource( unquote(parsed.path), parsed.query, emit_body=True @@ -2127,7 +2142,7 @@ def _make_handler( if unquote(parsed.path) == NAVIGATION_SECOND_VISIT_PATH: self._serve_navigation_second_visit() return - if unquote(parsed.path) in FETCH_ABORT_RESOURCE_PATHS | FETCH_RANGE_RESOURCE_PATHS | { + if unquote(parsed.path) in FETCH_ABORT_RESOURCE_PATHS | FETCH_RANGE_RESOURCE_PATHS | FETCH_PREFLIGHT_RESOURCE_PATHS | { "/fetch/api/resources/status.py", "/fetch/api/resources/trickle.py" }: self._serve_fetch_resource_method() @@ -2359,6 +2374,9 @@ def _make_handler( if path == LINK_STYLESHEET_COUNTER_PATH: self._serve_link_stylesheet_counter(parsed.query, emit_body=emit_body) return + if path in FETCH_PREFLIGHT_RESOURCE_PATHS: + self._serve_fetch_preflight_resource(path, parsed.query, emit_body=emit_body) + return if path == ( "/html/semantics/scripting-1/the-script-element/" "serve-with-content-type.py" @@ -3395,6 +3413,86 @@ def _make_handler( except OSError: pass # Upstream ends its stream when a write reports disconnect. + def _serve_fetch_preflight_resource( + self, path: str, query: str, *, emit_body: bool + ) -> None: + connection_headers = [] + # Neither upstream handler reads the upload. Respond immediately + # and close unread-body connections instead of waiting for EOF. + if (self.headers.get("Transfer-Encoding") is not None + or self.headers.get("Content-Length", "0").strip() not in {"", "0"}): + self.close_connection = True + connection_headers.append(("Connection", "close")) + params = parse_qs(query, keep_blank_values=True, encoding="latin-1") + stash_path = urlsplit(self.path).path + try: + if path.endswith("/clean-stash.py"): + # These handlers use the complete request path, whereas + # the abort helpers explicitly share a directory namespace. + removed = fetch_stash.take(params["token"][0], path=stash_path) + self._send_bytes(None, b"1" if removed is not None else b"0", + emit_body=emit_body, extra_headers=connection_headers) + return + + headers = [*connection_headers, ("Content-Type", "text/plain")] + for origin in params.get("origin", ["*"])[0].split(", "): + headers.append(("Access-Control-Allow-Origin", origin)) + token = params.get("token", [None])[0] + if "clear-stash" in params: + removed = fetch_stash.take(token, path=stash_path) + self._send_bytes(None, b"1" if removed is not None else b"0", + emit_body=emit_body, extra_headers=headers) + return + if "credentials" in params: + headers.append(("Access-Control-Allow-Credentials", "true")) + data = {"control_request_headers": "", "preflight": "0", "preflight_referrer": ""} + if self.command == "OPTIONS": + if "Access-Control-Request-Method" not in self.headers: + self._send_bytes("application/json", b"ERROR: No access-control-request-method in preflight!", + emit_body=emit_body, extra_headers=connection_headers, status_code=400) + return + if self.headers.get("Accept", "") != "*/*": + self._send_bytes("application/json", b"ERROR: Invalid access in preflight!", + emit_body=emit_body, extra_headers=connection_headers, status_code=400) + return + if "control_request_headers" in params: + data["control_request_headers"] = self.headers.get("Access-Control-Request-Headers") + for param, field in (("max_age", "Access-Control-Max-Age"), + ("allow_headers", "Access-Control-Allow-Headers"), + ("allow_methods", "Access-Control-Allow-Methods")): + if param in params: + headers.append((field, params[param][0])) + status = int(params.get("preflight_status", ["200"])[0]) + data.update(preflight="1", preflight_referrer=self.headers.get("Referer", ""), + preflight_user_agent=self.headers.get("User-Agent", "")) + if token: + fetch_stash.put(token, data, path=stash_path) + self._send_bytes(None, b"", emit_body=emit_body, extra_headers=headers, status_code=status) + return + + if token: + data = fetch_stash.take(token, path=stash_path) or data + if ("checkUserAgentHeaderInPreflight" in params + and self.headers.get("User-Agent") != data["preflight_user_agent"]): + self._send_bytes(None, b"ERROR: No user-agent header in preflight", + emit_body=emit_body, extra_headers=headers, status_code=400) + return + headers.extend([ + ("Access-Control-Expose-Headers", "x-did-preflight, x-control-request-headers, x-referrer, x-preflight-referrer, x-origin"), + ("x-did-preflight", data["preflight"]), + ]) + if data["control_request_headers"] is not None: + headers.append(("x-control-request-headers", data["control_request_headers"])) + headers.extend([ + ("x-preflight-referrer", data["preflight_referrer"]), + ("x-referrer", self.headers.get("Referer", "")), + ("x-origin", self.headers.get("Origin", "")), + ]) + if token: + fetch_stash.put(token, data, path=stash_path) + self._send_bytes(None, b"", emit_body=emit_body, extra_headers=headers) + except (KeyError, ValueError, TypeError): + self.send_error(500) def _serve_navigation_second_visit(self) -> None: if not self._consume_request_body(): diff --git a/moli-benchmark/tests/test_wpt_fetch_preflight_server.py b/moli-benchmark/tests/test_wpt_fetch_preflight_server.py new file mode 100644 index 0000000000..ec874e14f8 --- /dev/null +++ b/moli-benchmark/tests/test_wpt_fetch_preflight_server.py @@ -0,0 +1,244 @@ +from __future__ import annotations + +import tempfile +import unittest +import uuid +from contextlib import ExitStack +from http.client import HTTPConnection +from pathlib import Path +from unittest.mock import patch +from urllib.parse import urlencode + +from moli_benchmark.wpt_cross.case_set import enumerate_cases +from moli_benchmark.wpt_cross.server import WptFixtureServer + + +PREFLIGHT = "/fetch/api/resources/preflight.py" +CLEAN = "/fetch/api/resources/clean-stash.py" + + +class FetchPreflightFixtureTests(unittest.TestCase): + def setUp(self) -> None: + self.stack = ExitStack() + self.addCleanup(self.stack.close) + self.root = Path(self.stack.enter_context(tempfile.TemporaryDirectory())) + (self.root / "resources").mkdir() + (self.root / "resources/testharness.js").write_text("// testharness") + self.stack.enter_context(patch( + "moli_benchmark.wpt_cross.server._global_ipv6_address", return_value=None + )) + + def server(self) -> WptFixtureServer: + return self.stack.enter_context(WptFixtureServer(self.root)) + + def request(self, port, query="", *, method="GET", headers=None, path=PREFLIGHT, body=None): + connection = HTTPConnection("127.0.0.1", port, timeout=3) + try: + connection.request(method, path + "?" + query, body, headers or {}) + response = connection.getresponse() + return response.status, response.headers, response.read() + finally: + connection.close() + + def preflight(self, port, query="", *, headers=None): + return self.request(port, query, method="OPTIONS", headers={ + "Accept": "*/*", "Access-Control-Request-Method": "PUT", **(headers or {}), + }) + + def test_actual_requests_reflect_headers_for_standard_and_extension_methods(self): + server = self.server() + for method in ("GET", "HEAD", "POST", "PUT", "PATCH", "patcH", "DELETE", "YO", "chicken", "*"): + with self.subTest(method=method): + status, headers, body = self.request(server.port, method=method, body=b"payload", headers={ + "Origin": "https://caller.test", "Referer": "https://caller.test/page", + }) + self.assertEqual((status, body), (200, b"")) + self.assertEqual(headers["Content-Type"], "text/plain") + self.assertEqual(headers["Access-Control-Allow-Origin"], "*") + self.assertEqual(headers["x-origin"], "https://caller.test") + self.assertEqual(headers["x-referrer"], "https://caller.test/page") + self.assertEqual(headers["x-did-preflight"], "0") + self.assertEqual(headers["x-control-request-headers"], "") + self.assertEqual(headers["x-preflight-referrer"], "") + self.assertIn("x-origin", headers["Access-Control-Expose-Headers"]) + + def test_preflight_state_is_shared_across_origins_and_survives_actual_requests(self): + server = self.server() + key = str(uuid.uuid4()) + query = urlencode({"token": key, "control_request_headers": "", "max_age": "12000", + "allow_methods": "PUT", "allow_headers": "X-One, X-Two", "credentials": ""}) + status, headers, body = self.preflight(server.port, query, headers={ + "Access-Control-Request-Headers": "x-one,x-two", "User-Agent": "preflight-agent", + "Referer": "https://page.test/preflight", + }) + self.assertEqual((status, body), (200, b"")) + for name, value in [("Methods", "PUT"), ("Headers", "X-One, X-Two"), ("Credentials", "true")]: + self.assertEqual(headers["Access-Control-Allow-" + name], value) + self.assertEqual(headers["Access-Control-Max-Age"], "12000") + self.assertIsNone(headers.get("Access-Control-Expose-Headers")) + for method, port in [("GET", server.alternate_port), ("HEAD", server.port), ("PUT", server.port)]: + status, headers, body = self.request(port, urlencode({ + "token": key.upper().replace("-", ""), "checkUserAgentHeaderInPreflight": "", + "allow_headers": "not-an-actual-response-permission", + }), method=method, headers={"User-Agent": "preflight-agent", "Referer": "https://page.test/actual"}) + self.assertEqual((status, body), (200, b"")) + self.assertEqual(headers["x-did-preflight"], "1") + self.assertEqual(headers["x-control-request-headers"], "x-one,x-two") + self.assertEqual(headers["x-preflight-referrer"], "https://page.test/preflight") + self.assertEqual(headers["x-referrer"], "https://page.test/actual") + self.assertIsNone(headers.get("Access-Control-Allow-Headers")) + + def test_control_header_distinguishes_missing_empty_and_unrequested(self): + server = self.server() + for control, request_headers, expected in [ + (True, {}, None), (True, {"Access-Control-Request-Headers": ""}, ""), + (False, {"Access-Control-Request-Headers": "x-hidden"}, ""), + ]: + key = str(uuid.uuid4()) + query = "token=" + key + ("&control_request_headers" if control else "") + self.assertEqual(self.preflight(server.port, query, headers=request_headers)[0], 200) + _, headers, _ = self.request(server.port, "token=" + key) + self.assertEqual(headers.get("x-control-request-headers"), expected) + + def test_duplicate_origins_and_first_query_values_are_preserved(self): + server = self.server() + for origin, expected in [("https://a.test, https://b.test", ["https://a.test", "https://b.test"]), + ("https://a.test,https://b.test", ["https://a.test,https://b.test"]), + ("", [""]), ("\xff", ["\xff"])]: + query = urlencode([("origin", origin), ("origin", "ignored"), ("credentials", "false"), + ("allow_methods", "PUT"), ("allow_methods", "DELETE"), + ("allow_headers", ""), ("allow_headers", "ignored")], encoding="latin-1") + for preflight in (False, True): + status, headers, _ = self.preflight(server.port, query) if preflight else self.request(server.port, query) + self.assertEqual(status, 200) + self.assertEqual(headers.get_all("Access-Control-Allow-Origin"), expected) + self.assertEqual(headers["Access-Control-Allow-Credentials"], "true") + if preflight: + self.assertEqual(headers["Access-Control-Allow-Methods"], "PUT") + self.assertEqual(headers["Access-Control-Allow-Headers"], "") + + def test_clear_stash_uses_resource_namespaces_and_consumes_falsey_values(self): + server, other = self.server(), self.server() + key = str(uuid.uuid4()) + query = "token=" + key + self.assertEqual(self.preflight(server.port, query)[0], 200) + self.assertEqual(self.request(server.alternate_port, query, path=CLEAN)[2], b"0") + self.assertEqual(self.request(server.port, query)[1]["x-did-preflight"], "1") + self.assertEqual(self.request(other.port, query + "&clear-stash")[2], b"0") + status, headers, body = self.request(server.alternate_port, query + "&clear-stash&credentials") + self.assertEqual((status, body), (200, b"1")) + self.assertIsNone(headers.get("Access-Control-Allow-Credentials")) + self.assertIsNone(headers.get("x-did-preflight")) + self.assertEqual(self.request(server.port, query + "&clear-stash")[2], b"0") + server.fetch_stash.put(key, "", path=CLEAN) + status, headers, body = self.request(server.alternate_port, query, path=CLEAN) + self.assertEqual((status, body), (200, b"1")) + self.assertIsNone(headers.get("Content-Type")) + self.assertIsNone(headers.get("Access-Control-Allow-Origin")) + self.assertEqual(self.request(server.port, query, path=CLEAN)[2], b"0") + # The pre-existing abort fixtures keep their directory-scoped namespace. + server.fetch_stash.put(key, "abort-state") + self.assertEqual(self.request(server.port, query, path=CLEAN)[2], b"0") + self.assertEqual(server.fetch_stash.take(key), "abort-state") + + def test_preflight_rejects_overwrites_without_losing_previous_state(self): + server = self.server() + query = "token=" + str(uuid.uuid4()) + self.assertEqual(self.preflight(server.port, query)[0], 200) + self.assertEqual(self.preflight(server.port, query)[0], 500) + self.assertEqual(self.request(server.port, query)[1]["x-did-preflight"], "1") + + def test_stash_namespace_preserves_encoded_request_paths(self): + server = self.server() + query = "token=" + str(uuid.uuid4()) + encoded = PREFLIGHT.replace("preflight.py", "%70reflight.py") + status, _, _ = self.request(server.port, query, path=encoded, method="OPTIONS", headers={ + "Access-Control-Request-Method": "PUT", "Accept": "*/*", + }) + self.assertEqual(status, 200) + self.assertEqual(self.request(server.port, query)[1]["x-did-preflight"], "0") + self.assertEqual(self.request(server.port, query, path=encoded)[1]["x-did-preflight"], "1") + + def test_preflight_requires_method_header_and_exact_accept_value(self): + server = self.server() + for headers in ({}, {"Accept": "*/*"}, {"Access-Control-Request-Method": "PUT"}, + {"Access-Control-Request-Method": "PUT", "Accept": "text/plain"}): + status, response_headers, body = self.request(server.port, method="OPTIONS", headers=headers) + self.assertEqual(status, 400) + self.assertIsNone(response_headers.get("Access-Control-Allow-Origin")) + self.assertTrue(body.startswith(b"ERROR:")) + # The upstream handler checks presence, not the method header's grammar. + self.assertEqual(self.preflight(server.port, headers={"Access-Control-Request-Method": ""})[0], 200) + + def test_preflight_status_and_invalid_tokens_follow_upstream_error_order(self): + server = self.server() + for status in (200, 201, 204, 205, 299, 301, 304, 307, 400, 403, 500, 505): + self.assertEqual(self.preflight(server.port, f"preflight_status={status}&preflight_status=999")[0], status) + for query in ("preflight_status=", "preflight_status=bad", "token=not-a-uuid", "token=%FF"): + self.assertEqual(self.preflight(server.port, query)[0], 500) + self.assertEqual(self.request(server.port, "token=bad", method="OPTIONS")[0], 400) + self.assertEqual(self.preflight(server.port, "token=")[0], 200) + self.assertEqual(self.request(server.port, "token=")[0], 200) + for path, query in [(CLEAN, ""), (CLEAN, "token="), (CLEAN, "token=bad"), (PREFLIGHT, "clear-stash")]: + self.assertEqual(self.request(server.port, query, path=path)[0], 500) + + def test_user_agent_mismatch_consumes_preflight_record(self): + server = self.server() + query = "token=" + str(uuid.uuid4()) + self.assertEqual(self.preflight(server.port, query, headers={"User-Agent": "before"})[0], 200) + status, headers, body = self.request(server.port, query + "&checkUserAgentHeaderInPreflight", headers={"User-Agent": "after"}) + self.assertEqual((status, body), (400, b"ERROR: No user-agent header in preflight")) + self.assertEqual(headers["Access-Control-Allow-Origin"], "*") + self.assertEqual(self.request(server.port, query)[1]["x-did-preflight"], "0") + self.assertEqual(self.request(server.port, query + "&checkUserAgentHeaderInPreflight")[0], 500) + + def test_resource_routes_do_not_accept_prefixes_or_suffixes(self): + server = self.server() + for path in (PREFLIGHT + "2", CLEAN + ".js", "/wrong" + PREFLIGHT, "/fetch/api/cors/preflight.py"): + for method in ("GET", "OPTIONS", "chicken"): + self.assertGreaterEqual(self.request(server.port, path=path, method=method)[0], 400) + + def test_handlers_respond_before_unused_uploads_finish(self): + server = self.server() + for path, method, expected in ((PREFLIGHT, "POST", b""), + (PREFLIGHT, "OPTIONS", b""), + (CLEAN + "?token=" + str(uuid.uuid4()), "POST", b"0")): + for framing in (("Content-Length", "1000000"), ("Transfer-Encoding", "chunked")): + with self.subTest(path=path, method=method, framing=framing): + connection = HTTPConnection("127.0.0.1", server.port, timeout=2) + try: + connection.putrequest(method, path) + connection.putheader(*framing) + connection.putheader("Accept", "*/*") + connection.putheader("Access-Control-Request-Method", "PUT") + connection.endheaders() + response = connection.getresponse() + self.assertEqual((response.status, response.read()), (200, expected)) + self.assertEqual(response.headers["Connection"], "close") + finally: + connection.close() + + def test_case_selection_recognizes_supported_literal_and_resource_dir_references(self): + cases = { + "absolute": "fetch('/fetch/api/resources/preflight.py');", + "relative": "fetch('../resources/clean-stash.py?token=1');", + "concat": 'fetch(RESOURCES_DIR + "preflight.py");', + "unknown": "fetch('../resources/preflight.py'); fetch('../resources/unknown.py');", + "suffix": "fetch('../resources/preflight.py2');", + "prefix": "fetch('/wrong/fetch/api/resources/preflight.py');", + "wrong-relative": "fetch('resources/preflight.py');", + "bare": "fetch('preflight.py');", + } + utils = self.root / "fetch/api/resources/utils.js" + utils.parent.mkdir(parents=True) + utils.write_text('var RESOURCES_DIR = "../resources/"; fetch(RESOURCES_DIR + "inspect-headers.py");') + for name, source in cases.items(): + path = self.root / f"fetch/api/cors/{name}.any.js" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("// META: global=window,worker\n// META: script=../resources/utils.js\n" + source) + selected = enumerate_cases(self.root, dir_prefixes=("fetch",), any_js_global="both") + self.assertEqual([case.case_path for case in selected], [ + f"fetch/api/cors/{name}.any.js?moli-wpt-any={realm}" + for name in ("absolute", "concat", "relative") + for realm in ("dedicatedworker", "window") + ])