fix(wpt): serve service worker registration error fixtures

Serve Service Worker registration MIME, malformed-script and invalid-chunk resources. Preserve missing and empty Content-Type values, raw query selection, raw chunk bytes, method routing and the delayed HEAD writer behavior.

moli-benchmark/moli_benchmark/wpt_cross/case_set.py

@@ -815,10 +815,34 @@ def _empty_location_handler_reference_patterns(directory: str) -> tuple[re.Patte
     return tuple(patterns)
 
 
+@lru_cache(maxsize=None)
+def _service_worker_registration_handler_reference_patterns(directory: str) -> tuple[re.Pattern[str], ...]:
+    references = []
+    resources = (
+        "service-workers/service-worker/resources/mime-type-worker.py",
+        "service-workers/service-worker/resources/import-mime-type-worker.py",
+        "service-workers/service-worker/resources/malformed-worker.py",
+        "service-workers/service-worker/resources/invalid-chunked-encoding.py",
+        "service-workers/service-worker/resources/invalid-chunked-encoding-with-flush.py",
+    )
+    for resource in resources:
+        relative = posixpath.relpath(resource, directory)
+        references.extend(("/" + resource, relative, "./" + relative))
+    return tuple(
+        re.compile(
+            rf"(?<![A-Za-z0-9_./-]){re.escape(reference)}"
+            rf"{WPTSERVE_HANDLER_TRAILING_BOUNDARY}"
+        )
+        for reference in references
+    )
+
+
 def _supported_wptserve_handler_references(
     rel: str | None,
 ) -> tuple[re.Pattern[str], ...]:
     supported: tuple[re.Pattern[str], ...] = ()
+    if rel is not None:
+        supported += _service_worker_registration_handler_reference_patterns(posixpath.dirname(rel) or ".")
     if rel is not None and rel.startswith("fetch/api/"):
         supported += _empty_location_handler_reference_patterns(posixpath.dirname(rel))
     if rel is not None:

moli-benchmark/moli_benchmark/wpt_cross/server.py

@@ -121,6 +121,27 @@ NAVIGATION_SECOND_VISIT_PATH = (
     "/navigation-api/navigation-methods/return-value/resources/"
     "204-205-download-on-second-visit.py"
 )
+SERVICE_WORKER_REGISTRATION_RESOURCE_PATHS = {
+    "/service-workers/service-worker/resources/mime-type-worker.py",
+    "/service-workers/service-worker/resources/import-mime-type-worker.py",
+    "/service-workers/service-worker/resources/malformed-worker.py",
+    "/service-workers/service-worker/resources/invalid-chunked-encoding.py",
+    "/service-workers/service-worker/resources/invalid-chunked-encoding-with-flush.py",
+}
+SERVICE_WORKER_MALFORMED_SCRIPTS = {
+    "parse-error": 'var foo = function() {;',
+    "undefined-error": 'foo.bar = 42;',
+    "uncaught-exception": 'throw new DOMException("AbortError");',
+    "caught-exception": 'try { throw new Error; } catch(e) {}',
+    "import-malformed-script": 'importScripts("malformed-worker.py?parse-error");',
+    "import-no-such-script": 'importScripts("no-such-script.js");',
+    "top-level-await": 'await Promise.resolve(1);',
+    "instantiation-error": 'import nonexistent from "./imported-module-script.js";',
+    "instantiation-error-and-top-level-await":
+        'import nonexistent from "./imported-module-script.js"; await Promise.resolve(1);',
+}
+
+
 BENCH_TIMEOUT_MULTIPLIER_QUERY = "__moli_bench_timeout_multiplier"
 BENCH_REPORT_BRIDGE_SRC_RE = re.compile(
     rb"(?P<prefix>\bsrc\s*=\s*)(?P<quote>['\"])"
@@ -1720,6 +1741,9 @@ def _make_handler(
                 return
             if self._serve_empty_location_resource(emit_body=self.command != "HEAD"):
                 return
+            if unquote(urlsplit(self.path).path) in SERVICE_WORKER_REGISTRATION_RESOURCE_PATHS:
+                self._serve_service_worker_registration_resource()
+                return
             if self._serve_xhr_response_resource():
                 return
             parsed = urlparse(self.path)
@@ -1754,6 +1778,9 @@ def _make_handler(
                 return
             if self._serve_empty_location_resource(emit_body=self.command != "HEAD"):
                 return
+            if unquote(urlsplit(self.path).path) in SERVICE_WORKER_REGISTRATION_RESOURCE_PATHS:
+                self._serve_service_worker_registration_resource()
+                return
             if self._serve_xhr_response_resource():
                 return
             parsed = urlparse(self.path)
@@ -1809,6 +1836,9 @@ def _make_handler(
                 return
             if self._serve_empty_location_resource(emit_body=self.command != "HEAD"):
                 return
+            if unquote(urlsplit(self.path).path) in SERVICE_WORKER_REGISTRATION_RESOURCE_PATHS:
+                self._serve_service_worker_registration_resource()
+                return
             if self._serve_xhr_response_resource():
                 return
             parsed = urlparse(self.path)
@@ -1847,6 +1877,9 @@ def _make_handler(
             } | XHR_BODY_RESOURCE_PATHS:
                 self._serve_xhr_response_resource()
                 return
+            if unquote(urlsplit(self.path).path) in SERVICE_WORKER_REGISTRATION_RESOURCE_PATHS:
+                self._serve_service_worker_registration_resource()
+                return
             parsed = urlparse(self.path)
             if unquote(parsed.path) == NAVIGATION_SECOND_VISIT_PATH:
                 self._serve_navigation_second_visit()
@@ -2061,6 +2094,9 @@ def _make_handler(
                 return
             if self._serve_empty_location_resource(emit_body=emit_body):
                 return
+            if unquote(urlsplit(self.path).path) in SERVICE_WORKER_REGISTRATION_RESOURCE_PATHS:
+                self._serve_service_worker_registration_resource()
+                return
             if self._serve_xhr_response_resource(emit_body=emit_body):
                 return
             parsed = urlparse(self.path)
@@ -2549,6 +2585,66 @@ def _make_handler(
                 emit_body=emit_body, cache_control=None,
             )
 
+        def _serve_service_worker_invalid_chunked(self, *, delayed: bool) -> None:
+            self.close_connection = True
+            self.protocol_version = self.request_version
+            try:
+                self.send_response(200)
+                self.send_header("Content-Type", "application/javascript")
+                self.send_header("Transfer-Encoding", "chunked")
+                # wptserve adds a length for the returned body, but not for
+                # the explicit writer used by the delayed variant.
+                if not delayed:
+                    self.send_header("Content-Length", "6")
+                self.end_headers()
+                self.wfile.flush()
+                if delayed and stopping.wait(1):
+                    return
+                # An explicit upstream writer also emits its bytes for HEAD.
+                if delayed or self.command != "HEAD":
+                    self.wfile.write(b"XX\r\n\r\n")
+                    self.wfile.flush()
+            except OSError:
+                return
+
+
+        def _serve_service_worker_registration_resource(self) -> None:
+            parsed = urlsplit(self.path)
+            path = unquote(parsed.path)
+            if path.endswith(("/invalid-chunked-encoding.py", "/invalid-chunked-encoding-with-flush.py")):
+                self._serve_service_worker_invalid_chunked(delayed=path.endswith("-with-flush.py"))
+                return
+            if not self._consume_request_body():
+                return
+            params = parse_qs(parsed.query, keep_blank_values=True, encoding="latin-1")
+            headers: list[tuple[str, str]] = []
+            body = b""
+            try:
+                if path.endswith("/mime-type-worker.py"):
+                    if "mime" in params:
+                        headers.append(("Content-Type", params["mime"][0]))
+                elif path.endswith("/import-mime-type-worker.py"):
+                    headers.append(("Content-Type", "application/javascript"))
+                    suffix = "?mime=" + params["mime"][0] if "mime" in params else ""
+                    body = f"importScripts('./mime-type-worker.py{suffix}');".encode("latin-1")
+                else:
+                    script = SERVICE_WORKER_MALFORMED_SCRIPTS.get(parsed.query)
+                    if script is None:
+                        self.send_error(500)
+                        return
+                    headers.append(("Content-Type", "application/javascript"))
+                    body = script.encode("utf-8")
+                for _, value in headers:
+                    if "\r" in value or "\n" in value:
+                        raise ValueError("invalid response header")
+                    value.encode("latin-1")
+            except (ValueError, UnicodeError):
+                self.send_error(400)
+                return
+            self._send_bytes(None, body, emit_body=self.command != "HEAD",
+                             extra_headers=headers, cache_control=None)
+
+
         def _serve_xhr_delay(self, query: str, *, emit_body: bool) -> None:
             delay_seconds = _wpt_delay_seconds(query)
             if delay_seconds is None:
@@ -2600,6 +2696,8 @@ def _make_handler(
                 return self._serve_navigation_second_visit
             if name.startswith("do_") and unquote(urlparse(self.path).path) == FETCH_EMPTY_LOCATION_PATH:
                 return self._serve_empty_location_resource
+            if name.startswith("do_") and unquote(urlsplit(self.path).path) in SERVICE_WORKER_REGISTRATION_RESOURCE_PATHS:
+                return self._serve_service_worker_registration_resource
             if name.startswith("do_") and unquote(urlparse(self.path).path) in XHR_RESPONSE_RESOURCE_PATHS:
                 return self._serve_xhr_response_resource
             if name.startswith("do_") and unquote(urlparse(self.path).path) in FETCH_RANGE_RESOURCE_PATHS:

moli-benchmark/tests/test_wpt_service_worker_registration_server.py

@@ -0,0 +1,188 @@
+from __future__ import annotations
+
+
+import socket
+import tempfile
+
+import unittest
+
+
+from contextlib import ExitStack
+
+from http.client import HTTPConnection, IncompleteRead
+from pathlib import Path
+from threading import Event
+
+from unittest.mock import patch
+
+from moli_benchmark.wpt_cross.case_set import enumerate_cases
+from moli_benchmark.wpt_cross.server import WptFixtureServer
+
+
+DIRECTORY = "service-workers/service-worker"
+RESOURCES = "/" + DIRECTORY + "/resources/"
+
+
+class ServiceWorkerRegistrationFixtureTests(unittest.TestCase):
+    def setUp(self) -> None:
+        self.stack = ExitStack()
+        self.addCleanup(self.stack.close)
+        self.root = Path(self.stack.enter_context(tempfile.TemporaryDirectory()))
+        (self.root / "resources").mkdir()
+        for name in ('mime-type-worker.py', 'import-mime-type-worker.py', 'malformed-worker.py', 'invalid-chunked-encoding.py', 'invalid-chunked-encoding-with-flush.py'):
+            (resources / name).write_text("# Python source must not be sent as a script")
+        self.stack.enter_context(patch(
+            "moli_benchmark.wpt_cross.server._global_ipv6_address", return_value=None
+        ))
+
+
+    def server(self) -> WptFixtureServer:
+        return self.stack.enter_context(WptFixtureServer(self.root))
+
+
+    def request(
+        self, port: int, resource: str, query: str = "", *, method: str = "GET",
+    ) -> tuple[int, list[tuple[str, str]], bytes]:
+        connection = HTTPConnection("127.0.0.1", port, timeout=5)
+        try:
+            path = resource if resource.startswith("/") else RESOURCES + resource
+            connection.request(method, path + "?" + query)
+            response = connection.getresponse()
+            return (
+                response.status,
+                [(name.lower(), value) for name, value in response.getheaders()],
+                response.read(),
+            )
+        finally:
+            connection.close()
+
+
+    def test_registration_mime_handlers_preserve_missing_empty_and_raw_values(self) -> None:
+        server = self.server()
+        for query, mime in (
+            ("", None), ("mime=", ""), ("mime=text%2Fjavascript", "text/javascript"),
+            ("mime=text%2Fplain&mime=ignored", "text/plain"),
+            ("mime=application%2Fjavascript%3B+charset%3Dutf-8", "application/javascript; charset=utf-8"),
+            ("mime=%FF%27%26", "\xff'&"),
+        ):
+            for method in ("GET", "HEAD", "POST", "CUSTOM"):
+                with self.subTest(query=query, method=method):
+                    status, headers, body = self.request(server.port, "mime-type-worker.py", query, method=method)
+                    self.assertEqual((status, body), (200, b""))
+                    self.assertEqual(dict(headers).get("content-type"), mime)
+                    self.assertNotIn("cache-control", dict(headers))
+                    status, headers, body = self.request(server.alternate_port, "import-mime-type-worker.py", query, method=method)
+                    suffix = b"?mime=" + mime.encode("latin-1") if mime is not None else b""
+                    expected = b"importScripts('./mime-type-worker.py" + suffix + b"');"
+                    self.assertEqual((status, body), (200, b"" if method == "HEAD" else expected))
+                    self.assertEqual(dict(headers)["content-type"], "application/javascript")
+                    self.assertEqual(dict(headers)["content-length"], str(len(expected)))
+                    self.assertNotIn("cache-control", dict(headers))
+
+
+    def test_malformed_worker_selects_on_the_complete_undecoded_query(self) -> None:
+        server = self.server()
+        for query, expected in (
+            ("parse-error", b"var foo = function() {;"),
+            ("caught-exception", b"try { throw new Error; } catch(e) {}"),
+            ("import-malformed-script", b'importScripts("malformed-worker.py?parse-error");'),
+            ("instantiation-error-and-top-level-await", b'import nonexistent from "./imported-module-script.js"; await Promise.resolve(1);'),
+        ):
+            for method in ("GET", "HEAD", "OPTIONS"):
+                with self.subTest(query=query, method=method):
+                    status, headers, body = self.request(server.port, "%6dalformed-worker.py", query, method=method)
+                    self.assertEqual((status, body), (200, b"" if method == "HEAD" else expected))
+                    self.assertEqual(dict(headers)["content-type"], "application/javascript")
+                    self.assertEqual(dict(headers)["content-length"], str(len(expected)))
+                    self.assertNotIn("cache-control", dict(headers))
+        for query in ("", "unknown", "parse%2Derror", "parse-error=", "parse-error&ignored"):
+            with self.subTest(query=query):
+                self.assertEqual(self.request(server.port, "malformed-worker.py", query)[0], 500)
+
+
+    def test_invalid_chunked_responses_keep_raw_bytes_and_distinct_head_behavior(self) -> None:
+        server = self.server()
+        with patch.object(server._stopping, "wait", return_value=False):
+            for delayed in (False, True):
+                resource = "invalid-chunked-encoding" + ("-with-flush" if delayed else "") + ".py"
+                for method in ("GET", "HEAD", "CUSTOM"):
+                    with self.subTest(delayed=delayed, method=method), socket.create_connection(
+                        ("127.0.0.1", server.port), timeout=3,
+                    ) as connection:
+                        connection.sendall((
+                            f"{method} {RESOURCES}{resource} HTTP/1.1\r\n"
+                            f"Host: localhost:{server.port}\r\n\r\n"
+                        ).encode())
+                        with connection.makefile("rb") as response:
+                            head, body = response.read().split(b"\r\n\r\n", 1)
+                        fields = head.lower().split(b"\r\n")[1:]
+                        self.assertTrue(head.startswith(b"HTTP/1.1 200 "))
+                        self.assertIn(b"content-type: application/javascript", fields)
+                        self.assertIn(b"transfer-encoding: chunked", fields)
+                        self.assertEqual(b"content-length: 6" in fields, not delayed)
+                        self.assertEqual(body, b"" if method == "HEAD" and not delayed else b"XX\r\n\r\n")
+                connection = HTTPConnection("127.0.0.1", server.port, timeout=3)
+                try:
+                    connection.request("GET", RESOURCES + resource)
+                    response = connection.getresponse()
+                    self.assertTrue(response.chunked)
+                    with self.assertRaises(IncompleteRead):
+                        response.read()
+                finally:
+                    connection.close()
+
+
+    def test_invalid_chunk_flushes_headers_before_waiting_and_does_not_read_upload(self) -> None:
+        server = self.server()
+        waiting, release = Event(), Event()
+        def wait(timeout: float) -> bool:
+            self.assertEqual(timeout, 1)
+            waiting.set()
+            release.wait(3)
+            return False
+        with patch.object(server._stopping, "wait", side_effect=wait), socket.create_connection(
+            ("127.0.0.1", server.port), timeout=3,
+        ) as connection:
+            try:
+                connection.sendall((
+                    f"POST {RESOURCES}invalid-chunked-encoding-with-flush.py HTTP/1.1\r\n"
+                    f"Host: localhost:{server.port}\r\nContent-Length: 1000000\r\n\r\n"
+                ).encode())
+                self.assertTrue(waiting.wait(2))
+                with connection.makefile("rb") as response:
+                    headers = []
+                    while (line := response.readline()) != b"\r\n":
+                        self.assertTrue(line)
+                        headers.append(line)
+                    self.assertIn(b"Transfer-Encoding: chunked\r\n", headers)
+                    release.set()
+                    self.assertEqual(response.read(), b"XX\r\n\r\n")
+            finally:
+                release.set()
+
+
+    def test_discovery_only_accepts_supported_handler_locations(self) -> None:
+        cases = {
+            "mime.html": ("resources/mime-type-worker.py", True),
+            "import-mime.html": ("resources/import-mime-type-worker.py", True),
+            "malformed.html": ("resources/malformed-worker.py", True),
+            "chunked.html": ("resources/invalid-chunked-encoding.py", True),
+            "chunked-flush.html": ("resources/invalid-chunked-encoding-with-flush.py", True),
+            "sub/malformed.html": ("../resources/malformed-worker.py", True),
+            "sub/malformed-wrong.html": ("resources/malformed-worker.py", False),
+            "malformed-suffix.html": ("resources/malformed-worker.py.extra", False),
+            "unknown.html": ("resources/unknown.py", False),
+        }
+        for name, (reference, _) in cases.items():
+            path = self.root / DIRECTORY / name
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_text(
+                '<!doctype html><script src="/resources/testharness.js"></script>'
+                '<script src="/resources/testharnessreport.js"></script>'
+                f'<script src="{reference}?Key=ignored"></script>'
+            )
+        discovered = enumerate_cases(self.root, dir_prefixes=(DIRECTORY,))
+        self.assertEqual(
+            sorted(case.case_path for case in discovered),
+            sorted(DIRECTORY + "/" + name for name, (_, allowed) in cases.items() if allowed),
+        )