From ebc016ace340ce4bebdc05b938558bd41bfa8b98 Mon Sep 17 00:00:00 2001 From: ldm0 Date: Wed, 23 Sep 2026 14:19:42 +0800 Subject: [PATCH] fix(wpt): serve the common HTML echo fixture Serve common/echo.py as HTML with byte-preserving first query values, all-method handling, and ordered response pipes. Recognize exact relative and absolute references during case discovery. Validation: 582 Python tests passed with 32 workers; 144 HTTP responses matched official wptserve for status, body, and 15 response fields. New regression tests fail on the unmodified main baseline. Source-commit: 4042e1a11a5f29b89adb8e25b2c752a10d1d9ae1 --- .../moli_benchmark/wpt_cross/case_set.py | 14 ++ .../moli_benchmark/wpt_cross/server.py | 120 ++++++++++--- .../tests/test_wpt_common_echo_server.py | 170 ++++++++++++++++++ 3 files changed, 277 insertions(+), 27 deletions(-) create mode 100644 moli-benchmark/tests/test_wpt_common_echo_server.py diff --git a/moli-benchmark/moli_benchmark/wpt_cross/case_set.py b/moli-benchmark/moli_benchmark/wpt_cross/case_set.py index 4a5522a5a..6692f63a4 100644 --- a/moli-benchmark/moli_benchmark/wpt_cross/case_set.py +++ b/moli-benchmark/moli_benchmark/wpt_cross/case_set.py @@ -725,6 +725,19 @@ def _html_path_is_supported( return not any(token in rel for token in excluded) +@lru_cache(maxsize=None) +def _common_echo_handler_reference_patterns(directory: str) -> tuple[re.Pattern[str], ...]: + resource = "common/echo.py" + relative = posixpath.relpath(resource, directory) + return tuple( + re.compile( + rf"(? tuple[re.Pattern[str], ...]: resource = "html/semantics/scripting-1/the-script-element/serve-with-content-type.py" @@ -809,6 +822,7 @@ def _supported_wptserve_handler_references( if rel is not None and rel.startswith("fetch/api/"): supported += _empty_location_handler_reference_patterns(posixpath.dirname(rel)) if rel is not None: + supported += _common_echo_handler_reference_patterns(posixpath.dirname(rel) or ".") supported += _navigation_handler_reference_patterns(posixpath.dirname(rel) or ".") supported += _json_module_handler_reference_patterns(posixpath.dirname(rel) or ".") if rel is not None: diff --git a/moli-benchmark/moli_benchmark/wpt_cross/server.py b/moli-benchmark/moli_benchmark/wpt_cross/server.py index 5183ab57f..58681ba23 100644 --- a/moli-benchmark/moli_benchmark/wpt_cross/server.py +++ b/moli-benchmark/moli_benchmark/wpt_cross/server.py @@ -83,6 +83,7 @@ XHR_RESPONSE_RESOURCE_PATHS = { "/xhr/resources/status.py", "/xhr/resources/last-modified.py", } +COMMON_ECHO_PATH = "/common/echo.py" FETCH_EMPTY_LOCATION_PATH = "/fetch/api/resources/redirect-empty-location.py" FETCH_ABORT_RESOURCE_PATHS = { "/fetch/api/resources/stash-put.py", @@ -807,6 +808,7 @@ _REQUEST_TEMPLATE_RE = re.compile( _UUID_TEMPLATE_RE = re.compile(rb"\{\{\$([A-Za-z_][A-Za-z0-9_]*):uuid\(\)\}\}") _ID_TEMPLATE_RE = re.compile(rb"\{\{\$([A-Za-z_][A-Za-z0-9_]*)\}\}") _HTTP_TOKEN_RE = re.compile(r"^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$") +_TRICKLE_DELAY_RE = re.compile(r"d([0-9]+(?:\.[0-9]+)?)") _MAX_TRICKLE_DELAY_SECONDS = 10.0 _LEGACY_WPT_RESOURCE_ALIASES = { "/resources/WebIDLParser.js": "resources/webidl2/lib/webidl2.js", @@ -1699,6 +1701,8 @@ def _make_handler( self._serve(emit_body=False) def do_OPTIONS(self) -> None: # noqa: N802 + if self._serve_common_echo_resource(): + return if self._serve_empty_location_resource(emit_body=self.command != "HEAD"): return if self._serve_xhr_response_resource(): @@ -1731,6 +1735,8 @@ def _make_handler( self.send_error(404) def do_POST(self) -> None: # noqa: N802 + if self._serve_common_echo_resource(): + return if self._serve_empty_location_resource(emit_body=self.command != "HEAD"): return if self._serve_xhr_response_resource(): @@ -1784,6 +1790,8 @@ def _make_handler( self.end_headers() def _serve_fetch_resource_method(self) -> None: + if self._serve_common_echo_resource(): + return if self._serve_empty_location_resource(emit_body=self.command != "HEAD"): return if self._serve_xhr_response_resource(): @@ -1815,6 +1823,8 @@ def _make_handler( do_DELETE = _serve_fetch_resource_method def do_YO(self) -> None: # noqa: N802 (WPT custom method) + if self._serve_common_echo_resource(): + return if self._serve_empty_location_resource(emit_body=self.command != "HEAD"): return if unquote(urlparse(self.path).path) in { @@ -1950,6 +1960,81 @@ def _make_handler( except (TimeoutError, BrokenPipeError, ConnectionResetError, OSError): return + def _substitute_response_template( + self, body: bytes, path: str, query: str, *, escape_type: str, + ) -> bytes: + port = int(getattr(self.server, "wpt_primary_port", self.server.server_address[1])) + alternate_port = int(getattr(self.server, "wpt_alternate_port", port)) + remote_port = int(getattr(self.server, "wpt_remote_port", alternate_port)) + hostname = _host_header_hostname(self.headers.get("Host")) + return _substitute_wpt_template_variables( + body, + port=port, + alternate_port=alternate_port, + remote_port=remote_port, + query=query, + request_path=path, + request_hostname=hostname, + primary_hostname=str(getattr(self.server, "wpt_primary_hostname", hostname)), + request_headers=self.headers, + escape_type=escape_type, + ) + + def _serve_common_echo_resource(self) -> bool: + parsed = urlsplit(self.path) + if unquote(parsed.path) != COMMON_ECHO_PATH: + return False + # The handler only reads GET parameters, even for POST. Close the + # connection instead of waiting for an unused request body. + self.close_connection = True + try: + params = parse_qs(parsed.query, keep_blank_values=True, encoding="latin-1") + # wptserve's Request.GET preserves percent-decoded bytes and + # MultiDict.first selects the first value, including an empty one. + body = params["content"][0].encode("latin-1") + headers = [("Content-Type", "text/html"), ("X-XSS-Protection", "0")] + status, delay, auto_content_length = 200, 0.0, True + for name, args in parse_pipe_commands(parsed.query): + if name == "header": + header_name, value = args[:2] + value = value.replace("\r", " ").replace("\n", " ") + if _valid_static_response_header(header_name, value): + headers = _apply_header_operations(headers, [( + header_name, value, + len(args) == 3 and args[2].lower() in {"true", "1"}, + )]) + elif name == "status": + status = int(args[0]) + elif name == "sub": + body = self._substitute_response_template( + body, COMMON_ECHO_PATH, parsed.query, + escape_type=args[0] if args else "html", + ) + elif name == "trickle": + auto_content_length = False + if not any(_headers_include(headers, name) + for name in ("Cache-Control", "Pragma", "Expires")): + headers.extend([ + ("Cache-Control", "no-cache, no-store, must-revalidate"), + ("Pragma", "no-cache"), + ("Expires", "0"), + ]) + match = _TRICKLE_DELAY_RE.fullmatch(args[0]) + if match is not None: + delay = max(delay, float(match.group(1))) + except (KeyError, WptPipeError): + self.send_error(500) + return True + if delay: + time.sleep(min(delay, _MAX_TRICKLE_DELAY_SECONDS)) + self._send_bytes( + None, body, emit_body=self.command != "HEAD", + extra_headers=[*headers, ("Connection", "close")], + status_code=status, cache_control=None, + auto_content_length=auto_content_length, + ) + return True + def _serve(self, *, emit_body: bool) -> None: try: self._serve_response(emit_body=emit_body) @@ -1957,6 +2042,8 @@ def _make_handler( self.send_error(500, "Invalid WPT template or pipe") def _serve_response(self, *, emit_body: bool) -> None: + if self._serve_common_echo_resource(): + return if self._serve_empty_location_resource(emit_body=emit_body): return if self._serve_xhr_response_resource(emit_body=emit_body): @@ -2166,32 +2253,8 @@ def _make_handler( ) return if _needs_wpt_template_substitution(file_path.name, body, parsed.query): - port = int( - getattr( - self.server, - "wpt_primary_port", - self.server.server_address[1], - ) - ) - alternate_port = int(getattr(self.server, "wpt_alternate_port", port)) - remote_port = int(getattr(self.server, "wpt_remote_port", alternate_port)) - primary_hostname = str( - getattr( - self.server, - "wpt_primary_hostname", - _host_header_hostname(self.headers.get("Host")), - ) - ) - body = _substitute_wpt_template_variables( - body, - port=port, - alternate_port=alternate_port, - remote_port=remote_port, - query=parsed.query, - request_path=path, - request_hostname=_host_header_hostname(self.headers.get("Host")), - primary_hostname=primary_hostname, - request_headers=self.headers, + body = self._substitute_response_template( + body, path, parsed.query, escape_type=_template_escape_type(file_path.name, parsed.query), ) static_header_context = { @@ -2516,6 +2579,8 @@ def _make_handler( ) def __getattr__(self, name: str): + if name.startswith("do_") and unquote(urlsplit(self.path).path) == COMMON_ECHO_PATH: + return self._serve_common_echo_resource if name.startswith("do_") and unquote(urlparse(self.path).path) == NAVIGATION_SECOND_VISIT_PATH: return self._serve_navigation_second_visit if name.startswith("do_") and unquote(urlparse(self.path).path) == FETCH_EMPTY_LOCATION_PATH: @@ -2935,6 +3000,7 @@ def _make_handler( status_code: int = 200, status_text: str | None = None, cache_control: str | None = "no-store", + auto_content_length: bool = True, ) -> None: content_type, extra_headers = _response_content_type_and_extra_headers( content_type, @@ -2944,7 +3010,7 @@ def _make_handler( header_block = _static_response_header_block(content_type, extra_headers) for name, value in header_block: self.send_header(name, value) - if not _headers_include(header_block, "Content-Length"): + if auto_content_length and not _headers_include(header_block, "Content-Length"): self.send_header("Content-Length", str(len(body))) if cache_control is not None: self.send_header("Cache-Control", cache_control) diff --git a/moli-benchmark/tests/test_wpt_common_echo_server.py b/moli-benchmark/tests/test_wpt_common_echo_server.py new file mode 100644 index 000000000..7675a167c --- /dev/null +++ b/moli-benchmark/tests/test_wpt_common_echo_server.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +import tempfile +import unittest +from contextlib import ExitStack +from http.client import HTTPConnection +from pathlib import Path +from unittest.mock import patch +from urllib.parse import urlencode + +from moli_benchmark.wpt_cross.case_set import enumerate_cases +from moli_benchmark.wpt_cross.server import WptFixtureServer + + +ECHO_PATH = "/common/echo.py" + + +class CommonEchoFixtureTests(unittest.TestCase): + def setUp(self) -> None: + self.stack = ExitStack() + self.addCleanup(self.stack.close) + self.root = Path(self.stack.enter_context(tempfile.TemporaryDirectory())) + (self.root / "resources").mkdir() + (self.root / "resources/testharness.js").write_text("// testharness") + (self.root / "common").mkdir() + (self.root / "common/echo.py").write_text("# Python source must not be served") + self.stack.enter_context(patch( + "moli_benchmark.wpt_cross.server._global_ipv6_address", return_value=None, + )) + self.server = self.stack.enter_context(WptFixtureServer(self.root)) + + def request(self, query: str, *, method: str = "GET", path: str = ECHO_PATH, body=None): + connection = HTTPConnection("127.0.0.1", self.server.port, timeout=2) + try: + connection.request(method, path + "?" + query, body, {"Origin": "https://caller.test"}) + response = connection.getresponse() + return response.status, response.headers, response.read() + finally: + connection.close() + + def test_content_is_html_and_preserves_query_bytes_and_first_value(self) -> None: + cases = ( + ("content=%3Cscript%3EglobalThis.answer%3D42%3C%2Fscript%3E", b""), + ("content=first&content=second", b"first"), + ("content=&content=second", b""), + ("content", b""), + ("content=A+B%2BC%26D%3DE", b"A B+C&D=E"), + ("content=%FF%00%C3%A9%FE", b"\xff\x00\xc3\xa9\xfe"), + ("co%6Etent=decoded-key", b"decoded-key"), + ("content=%2520;%invalid", b"%20;%invalid"), + ("content={{host}}", b"{{host}}"), + ) + for query, expected in cases: + for method in ("GET", "HEAD"): + with self.subTest(query=query, method=method): + status, headers, body = self.request(query, method=method) + self.assertEqual(status, 200) + self.assertEqual(headers.get_all("Content-Type"), ["text/html"]) + self.assertEqual(headers["X-XSS-Protection"], "0") + self.assertEqual(headers["Content-Length"], str(len(expected))) + self.assertEqual(body, b"" if method == "HEAD" else expected) + self.assertIsNone(headers["Access-Control-Allow-Origin"]) + self.assertIsNone(headers["Cache-Control"]) + + def test_all_methods_read_query_content_instead_of_request_body(self) -> None: + for method in ("POST", "OPTIONS", "PUT", "PATCH", "DELETE", "YO", "chicken"): + with self.subTest(method=method): + status, headers, body = self.request( + "content=query", method=method, body=b"content=upload", + ) + self.assertEqual((status, body), (200, b"query")) + self.assertEqual(headers["Content-Type"], "text/html") + self.assertIsNone(headers["Access-Control-Allow-Origin"]) + + def test_unused_uploads_do_not_delay_the_response(self) -> None: + for method in ("POST", "PUT", "OPTIONS"): + for framing in (("Content-Length", "1000000"), ("Transfer-Encoding", "chunked")): + with self.subTest(method=method, framing=framing): + connection = HTTPConnection("127.0.0.1", self.server.port, timeout=2) + try: + connection.putrequest(method, ECHO_PATH + "?content=ready") + connection.putheader(*framing) + connection.endheaders() + response = connection.getresponse() + self.assertEqual((response.status, response.read()), (200, b"ready")) + self.assertEqual(response.headers["Connection"], "close") + finally: + connection.close() + + def test_missing_content_is_an_error_and_other_paths_are_not_echo_handlers(self) -> None: + for method in ("GET", "HEAD", "POST"): + for query in ("", "Content=wrong-case"): + with self.subTest(method=method, query=query): + self.assertEqual(self.request(query, method=method)[0], 500) + for path in (ECHO_PATH + "2", "/wrong" + ECHO_PATH, "/COMMON/echo.py"): + with self.subTest(path=path): + self.assertEqual(self.request("content=not-an-echo", path=path)[0], 404) + + def test_response_pipes_apply_after_the_handler(self) -> None: + query = urlencode({ + "content": "{{host}}", + "pipe": "sub(none)|status(201)|header(Content-Type,text/plain)|header(X-XSS-Protection,1)", + }) + for method in ("GET", "HEAD", "POST", "chicken"): + with self.subTest(method=method): + status, headers, body = self.request(query, method=method) + self.assertEqual(status, 201) + self.assertEqual(headers.get_all("Content-Type"), ["text/plain"]) + self.assertEqual(headers.get_all("X-XSS-Protection"), ["1"]) + self.assertEqual(headers["Content-Length"], "9") + self.assertEqual(body, b"" if method == "HEAD" else b"localhost") + with patch("moli_benchmark.wpt_cross.server.time.sleep") as sleep: + status, headers, body = self.request(urlencode({"content": "delayed", "pipe": "trickle(d0.01)"})) + self.assertEqual((status, body), (200, b"delayed")) + self.assertIsNone(headers["Content-Length"]) + self.assertEqual(headers["Cache-Control"], "no-cache, no-store, must-revalidate") + self.assertEqual(headers["Pragma"], "no-cache") + self.assertEqual(headers["Expires"], "0") + sleep.assert_called_once_with(0.01) + for command, expected, pragma, expires in ( + ("trickle(d0)|header(Cache-Control,private,true)", + ["no-cache, no-store, must-revalidate", "private"], "no-cache", "0"), + ("header(Cache-Control,private)|trickle(d0)", ["private"], None, None), + ("header(Pragma,no-cache)|trickle(d0)", None, "no-cache", None), + ("header(Expires,0)|trickle(d0)", None, None, "0"), + ): + with self.subTest(command=command): + status, headers, body = self.request(urlencode({"content": "ok", "pipe": command})) + self.assertEqual((status, body), (200, b"ok")) + self.assertEqual(headers.get_all("Cache-Control"), expected) + self.assertEqual(headers["Pragma"], pragma) + self.assertEqual(headers["Expires"], expires) + for method in ("GET", "POST", "chicken"): + self.assertEqual(self.request("content=not-served&pipe=unknown", method=method)[0], 500) + + def test_discovery_recognizes_only_references_to_the_common_echo_handler(self) -> None: + directory = "html/browsers/echo-tests" + cases = { + "absolute.html": (ECHO_PATH, True), + "relative.html": ("../../../common/echo.py", True), + "dot-relative.html": ("./../../../common/echo.py", True), + "bare.html": ("echo.py", False), + "wrong-relative.html": ("../../common/echo.py", False), + "wrong-root.html": ("/other/common/echo.py", False), + "suffix.html": (ECHO_PATH + ".extra", False), + } + for name, (reference, _) in cases.items(): + path = self.root / directory / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + '' + f'' + ) + (self.root / directory / "unknown.html").write_text( + '' + f'' + ) + (self.root / directory / "script.window.js").write_text( + f'const echoURL = content => `{ECHO_PATH}?content=${{encodeURIComponent(content)}}`;' + ) + discovered = enumerate_cases(self.root, dir_prefixes=(directory,)) + self.assertEqual( + sorted(case.case_path.split("?")[0] for case in discovered), + sorted([directory + "/" + name for name, (_, allowed) in cases.items() if allowed] + + [directory + "/script.window.js"]), + ) + + +if __name__ == "__main__": + unittest.main()