diff --git a/moli-benchmark/moli_benchmark/wpt_cross/case_set.py b/moli-benchmark/moli_benchmark/wpt_cross/case_set.py
index 4a5522a5a..6692f63a4 100644
--- a/moli-benchmark/moli_benchmark/wpt_cross/case_set.py
+++ b/moli-benchmark/moli_benchmark/wpt_cross/case_set.py
@@ -725,6 +725,19 @@ def _html_path_is_supported(
return not any(token in rel for token in excluded)
+@lru_cache(maxsize=None)
+def _common_echo_handler_reference_patterns(directory: str) -> tuple[re.Pattern[str], ...]:
+ resource = "common/echo.py"
+ relative = posixpath.relpath(resource, directory)
+ return tuple(
+ re.compile(
+ rf"(? tuple[re.Pattern[str], ...]:
resource = "html/semantics/scripting-1/the-script-element/serve-with-content-type.py"
@@ -809,6 +822,7 @@ def _supported_wptserve_handler_references(
if rel is not None and rel.startswith("fetch/api/"):
supported += _empty_location_handler_reference_patterns(posixpath.dirname(rel))
if rel is not None:
+ supported += _common_echo_handler_reference_patterns(posixpath.dirname(rel) or ".")
supported += _navigation_handler_reference_patterns(posixpath.dirname(rel) or ".")
supported += _json_module_handler_reference_patterns(posixpath.dirname(rel) or ".")
if rel is not None:
diff --git a/moli-benchmark/moli_benchmark/wpt_cross/server.py b/moli-benchmark/moli_benchmark/wpt_cross/server.py
index 5183ab57f..58681ba23 100644
--- a/moli-benchmark/moli_benchmark/wpt_cross/server.py
+++ b/moli-benchmark/moli_benchmark/wpt_cross/server.py
@@ -83,6 +83,7 @@ XHR_RESPONSE_RESOURCE_PATHS = {
"/xhr/resources/status.py",
"/xhr/resources/last-modified.py",
}
+COMMON_ECHO_PATH = "/common/echo.py"
FETCH_EMPTY_LOCATION_PATH = "/fetch/api/resources/redirect-empty-location.py"
FETCH_ABORT_RESOURCE_PATHS = {
"/fetch/api/resources/stash-put.py",
@@ -807,6 +808,7 @@ _REQUEST_TEMPLATE_RE = re.compile(
_UUID_TEMPLATE_RE = re.compile(rb"\{\{\$([A-Za-z_][A-Za-z0-9_]*):uuid\(\)\}\}")
_ID_TEMPLATE_RE = re.compile(rb"\{\{\$([A-Za-z_][A-Za-z0-9_]*)\}\}")
_HTTP_TOKEN_RE = re.compile(r"^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$")
+_TRICKLE_DELAY_RE = re.compile(r"d([0-9]+(?:\.[0-9]+)?)")
_MAX_TRICKLE_DELAY_SECONDS = 10.0
_LEGACY_WPT_RESOURCE_ALIASES = {
"/resources/WebIDLParser.js": "resources/webidl2/lib/webidl2.js",
@@ -1699,6 +1701,8 @@ def _make_handler(
self._serve(emit_body=False)
def do_OPTIONS(self) -> None: # noqa: N802
+ if self._serve_common_echo_resource():
+ return
if self._serve_empty_location_resource(emit_body=self.command != "HEAD"):
return
if self._serve_xhr_response_resource():
@@ -1731,6 +1735,8 @@ def _make_handler(
self.send_error(404)
def do_POST(self) -> None: # noqa: N802
+ if self._serve_common_echo_resource():
+ return
if self._serve_empty_location_resource(emit_body=self.command != "HEAD"):
return
if self._serve_xhr_response_resource():
@@ -1784,6 +1790,8 @@ def _make_handler(
self.end_headers()
def _serve_fetch_resource_method(self) -> None:
+ if self._serve_common_echo_resource():
+ return
if self._serve_empty_location_resource(emit_body=self.command != "HEAD"):
return
if self._serve_xhr_response_resource():
@@ -1815,6 +1823,8 @@ def _make_handler(
do_DELETE = _serve_fetch_resource_method
def do_YO(self) -> None: # noqa: N802 (WPT custom method)
+ if self._serve_common_echo_resource():
+ return
if self._serve_empty_location_resource(emit_body=self.command != "HEAD"):
return
if unquote(urlparse(self.path).path) in {
@@ -1950,6 +1960,81 @@ def _make_handler(
except (TimeoutError, BrokenPipeError, ConnectionResetError, OSError):
return
+ def _substitute_response_template(
+ self, body: bytes, path: str, query: str, *, escape_type: str,
+ ) -> bytes:
+ port = int(getattr(self.server, "wpt_primary_port", self.server.server_address[1]))
+ alternate_port = int(getattr(self.server, "wpt_alternate_port", port))
+ remote_port = int(getattr(self.server, "wpt_remote_port", alternate_port))
+ hostname = _host_header_hostname(self.headers.get("Host"))
+ return _substitute_wpt_template_variables(
+ body,
+ port=port,
+ alternate_port=alternate_port,
+ remote_port=remote_port,
+ query=query,
+ request_path=path,
+ request_hostname=hostname,
+ primary_hostname=str(getattr(self.server, "wpt_primary_hostname", hostname)),
+ request_headers=self.headers,
+ escape_type=escape_type,
+ )
+
+ def _serve_common_echo_resource(self) -> bool:
+ parsed = urlsplit(self.path)
+ if unquote(parsed.path) != COMMON_ECHO_PATH:
+ return False
+ # The handler only reads GET parameters, even for POST. Close the
+ # connection instead of waiting for an unused request body.
+ self.close_connection = True
+ try:
+ params = parse_qs(parsed.query, keep_blank_values=True, encoding="latin-1")
+ # wptserve's Request.GET preserves percent-decoded bytes and
+ # MultiDict.first selects the first value, including an empty one.
+ body = params["content"][0].encode("latin-1")
+ headers = [("Content-Type", "text/html"), ("X-XSS-Protection", "0")]
+ status, delay, auto_content_length = 200, 0.0, True
+ for name, args in parse_pipe_commands(parsed.query):
+ if name == "header":
+ header_name, value = args[:2]
+ value = value.replace("\r", " ").replace("\n", " ")
+ if _valid_static_response_header(header_name, value):
+ headers = _apply_header_operations(headers, [(
+ header_name, value,
+ len(args) == 3 and args[2].lower() in {"true", "1"},
+ )])
+ elif name == "status":
+ status = int(args[0])
+ elif name == "sub":
+ body = self._substitute_response_template(
+ body, COMMON_ECHO_PATH, parsed.query,
+ escape_type=args[0] if args else "html",
+ )
+ elif name == "trickle":
+ auto_content_length = False
+ if not any(_headers_include(headers, name)
+ for name in ("Cache-Control", "Pragma", "Expires")):
+ headers.extend([
+ ("Cache-Control", "no-cache, no-store, must-revalidate"),
+ ("Pragma", "no-cache"),
+ ("Expires", "0"),
+ ])
+ match = _TRICKLE_DELAY_RE.fullmatch(args[0])
+ if match is not None:
+ delay = max(delay, float(match.group(1)))
+ except (KeyError, WptPipeError):
+ self.send_error(500)
+ return True
+ if delay:
+ time.sleep(min(delay, _MAX_TRICKLE_DELAY_SECONDS))
+ self._send_bytes(
+ None, body, emit_body=self.command != "HEAD",
+ extra_headers=[*headers, ("Connection", "close")],
+ status_code=status, cache_control=None,
+ auto_content_length=auto_content_length,
+ )
+ return True
+
def _serve(self, *, emit_body: bool) -> None:
try:
self._serve_response(emit_body=emit_body)
@@ -1957,6 +2042,8 @@ def _make_handler(
self.send_error(500, "Invalid WPT template or pipe")
def _serve_response(self, *, emit_body: bool) -> None:
+ if self._serve_common_echo_resource():
+ return
if self._serve_empty_location_resource(emit_body=emit_body):
return
if self._serve_xhr_response_resource(emit_body=emit_body):
@@ -2166,32 +2253,8 @@ def _make_handler(
)
return
if _needs_wpt_template_substitution(file_path.name, body, parsed.query):
- port = int(
- getattr(
- self.server,
- "wpt_primary_port",
- self.server.server_address[1],
- )
- )
- alternate_port = int(getattr(self.server, "wpt_alternate_port", port))
- remote_port = int(getattr(self.server, "wpt_remote_port", alternate_port))
- primary_hostname = str(
- getattr(
- self.server,
- "wpt_primary_hostname",
- _host_header_hostname(self.headers.get("Host")),
- )
- )
- body = _substitute_wpt_template_variables(
- body,
- port=port,
- alternate_port=alternate_port,
- remote_port=remote_port,
- query=parsed.query,
- request_path=path,
- request_hostname=_host_header_hostname(self.headers.get("Host")),
- primary_hostname=primary_hostname,
- request_headers=self.headers,
+ body = self._substitute_response_template(
+ body, path, parsed.query,
escape_type=_template_escape_type(file_path.name, parsed.query),
)
static_header_context = {
@@ -2516,6 +2579,8 @@ def _make_handler(
)
def __getattr__(self, name: str):
+ if name.startswith("do_") and unquote(urlsplit(self.path).path) == COMMON_ECHO_PATH:
+ return self._serve_common_echo_resource
if name.startswith("do_") and unquote(urlparse(self.path).path) == NAVIGATION_SECOND_VISIT_PATH:
return self._serve_navigation_second_visit
if name.startswith("do_") and unquote(urlparse(self.path).path) == FETCH_EMPTY_LOCATION_PATH:
@@ -2935,6 +3000,7 @@ def _make_handler(
status_code: int = 200,
status_text: str | None = None,
cache_control: str | None = "no-store",
+ auto_content_length: bool = True,
) -> None:
content_type, extra_headers = _response_content_type_and_extra_headers(
content_type,
@@ -2944,7 +3010,7 @@ def _make_handler(
header_block = _static_response_header_block(content_type, extra_headers)
for name, value in header_block:
self.send_header(name, value)
- if not _headers_include(header_block, "Content-Length"):
+ if auto_content_length and not _headers_include(header_block, "Content-Length"):
self.send_header("Content-Length", str(len(body)))
if cache_control is not None:
self.send_header("Cache-Control", cache_control)
diff --git a/moli-benchmark/tests/test_wpt_common_echo_server.py b/moli-benchmark/tests/test_wpt_common_echo_server.py
new file mode 100644
index 000000000..7675a167c
--- /dev/null
+++ b/moli-benchmark/tests/test_wpt_common_echo_server.py
@@ -0,0 +1,170 @@
+from __future__ import annotations
+
+import tempfile
+import unittest
+from contextlib import ExitStack
+from http.client import HTTPConnection
+from pathlib import Path
+from unittest.mock import patch
+from urllib.parse import urlencode
+
+from moli_benchmark.wpt_cross.case_set import enumerate_cases
+from moli_benchmark.wpt_cross.server import WptFixtureServer
+
+
+ECHO_PATH = "/common/echo.py"
+
+
+class CommonEchoFixtureTests(unittest.TestCase):
+ def setUp(self) -> None:
+ self.stack = ExitStack()
+ self.addCleanup(self.stack.close)
+ self.root = Path(self.stack.enter_context(tempfile.TemporaryDirectory()))
+ (self.root / "resources").mkdir()
+ (self.root / "resources/testharness.js").write_text("// testharness")
+ (self.root / "common").mkdir()
+ (self.root / "common/echo.py").write_text("# Python source must not be served")
+ self.stack.enter_context(patch(
+ "moli_benchmark.wpt_cross.server._global_ipv6_address", return_value=None,
+ ))
+ self.server = self.stack.enter_context(WptFixtureServer(self.root))
+
+ def request(self, query: str, *, method: str = "GET", path: str = ECHO_PATH, body=None):
+ connection = HTTPConnection("127.0.0.1", self.server.port, timeout=2)
+ try:
+ connection.request(method, path + "?" + query, body, {"Origin": "https://caller.test"})
+ response = connection.getresponse()
+ return response.status, response.headers, response.read()
+ finally:
+ connection.close()
+
+ def test_content_is_html_and_preserves_query_bytes_and_first_value(self) -> None:
+ cases = (
+ ("content=%3Cscript%3EglobalThis.answer%3D42%3C%2Fscript%3E", b""),
+ ("content=first&content=second", b"first"),
+ ("content=&content=second", b""),
+ ("content", b""),
+ ("content=A+B%2BC%26D%3DE", b"A B+C&D=E"),
+ ("content=%FF%00%C3%A9%FE", b"\xff\x00\xc3\xa9\xfe"),
+ ("co%6Etent=decoded-key", b"decoded-key"),
+ ("content=%2520;%invalid", b"%20;%invalid"),
+ ("content={{host}}", b"{{host}}"),
+ )
+ for query, expected in cases:
+ for method in ("GET", "HEAD"):
+ with self.subTest(query=query, method=method):
+ status, headers, body = self.request(query, method=method)
+ self.assertEqual(status, 200)
+ self.assertEqual(headers.get_all("Content-Type"), ["text/html"])
+ self.assertEqual(headers["X-XSS-Protection"], "0")
+ self.assertEqual(headers["Content-Length"], str(len(expected)))
+ self.assertEqual(body, b"" if method == "HEAD" else expected)
+ self.assertIsNone(headers["Access-Control-Allow-Origin"])
+ self.assertIsNone(headers["Cache-Control"])
+
+ def test_all_methods_read_query_content_instead_of_request_body(self) -> None:
+ for method in ("POST", "OPTIONS", "PUT", "PATCH", "DELETE", "YO", "chicken"):
+ with self.subTest(method=method):
+ status, headers, body = self.request(
+ "content=query", method=method, body=b"content=upload",
+ )
+ self.assertEqual((status, body), (200, b"query"))
+ self.assertEqual(headers["Content-Type"], "text/html")
+ self.assertIsNone(headers["Access-Control-Allow-Origin"])
+
+ def test_unused_uploads_do_not_delay_the_response(self) -> None:
+ for method in ("POST", "PUT", "OPTIONS"):
+ for framing in (("Content-Length", "1000000"), ("Transfer-Encoding", "chunked")):
+ with self.subTest(method=method, framing=framing):
+ connection = HTTPConnection("127.0.0.1", self.server.port, timeout=2)
+ try:
+ connection.putrequest(method, ECHO_PATH + "?content=ready")
+ connection.putheader(*framing)
+ connection.endheaders()
+ response = connection.getresponse()
+ self.assertEqual((response.status, response.read()), (200, b"ready"))
+ self.assertEqual(response.headers["Connection"], "close")
+ finally:
+ connection.close()
+
+ def test_missing_content_is_an_error_and_other_paths_are_not_echo_handlers(self) -> None:
+ for method in ("GET", "HEAD", "POST"):
+ for query in ("", "Content=wrong-case"):
+ with self.subTest(method=method, query=query):
+ self.assertEqual(self.request(query, method=method)[0], 500)
+ for path in (ECHO_PATH + "2", "/wrong" + ECHO_PATH, "/COMMON/echo.py"):
+ with self.subTest(path=path):
+ self.assertEqual(self.request("content=not-an-echo", path=path)[0], 404)
+
+ def test_response_pipes_apply_after_the_handler(self) -> None:
+ query = urlencode({
+ "content": "{{host}}",
+ "pipe": "sub(none)|status(201)|header(Content-Type,text/plain)|header(X-XSS-Protection,1)",
+ })
+ for method in ("GET", "HEAD", "POST", "chicken"):
+ with self.subTest(method=method):
+ status, headers, body = self.request(query, method=method)
+ self.assertEqual(status, 201)
+ self.assertEqual(headers.get_all("Content-Type"), ["text/plain"])
+ self.assertEqual(headers.get_all("X-XSS-Protection"), ["1"])
+ self.assertEqual(headers["Content-Length"], "9")
+ self.assertEqual(body, b"" if method == "HEAD" else b"localhost")
+ with patch("moli_benchmark.wpt_cross.server.time.sleep") as sleep:
+ status, headers, body = self.request(urlencode({"content": "delayed", "pipe": "trickle(d0.01)"}))
+ self.assertEqual((status, body), (200, b"delayed"))
+ self.assertIsNone(headers["Content-Length"])
+ self.assertEqual(headers["Cache-Control"], "no-cache, no-store, must-revalidate")
+ self.assertEqual(headers["Pragma"], "no-cache")
+ self.assertEqual(headers["Expires"], "0")
+ sleep.assert_called_once_with(0.01)
+ for command, expected, pragma, expires in (
+ ("trickle(d0)|header(Cache-Control,private,true)",
+ ["no-cache, no-store, must-revalidate", "private"], "no-cache", "0"),
+ ("header(Cache-Control,private)|trickle(d0)", ["private"], None, None),
+ ("header(Pragma,no-cache)|trickle(d0)", None, "no-cache", None),
+ ("header(Expires,0)|trickle(d0)", None, None, "0"),
+ ):
+ with self.subTest(command=command):
+ status, headers, body = self.request(urlencode({"content": "ok", "pipe": command}))
+ self.assertEqual((status, body), (200, b"ok"))
+ self.assertEqual(headers.get_all("Cache-Control"), expected)
+ self.assertEqual(headers["Pragma"], pragma)
+ self.assertEqual(headers["Expires"], expires)
+ for method in ("GET", "POST", "chicken"):
+ self.assertEqual(self.request("content=not-served&pipe=unknown", method=method)[0], 500)
+
+ def test_discovery_recognizes_only_references_to_the_common_echo_handler(self) -> None:
+ directory = "html/browsers/echo-tests"
+ cases = {
+ "absolute.html": (ECHO_PATH, True),
+ "relative.html": ("../../../common/echo.py", True),
+ "dot-relative.html": ("./../../../common/echo.py", True),
+ "bare.html": ("echo.py", False),
+ "wrong-relative.html": ("../../common/echo.py", False),
+ "wrong-root.html": ("/other/common/echo.py", False),
+ "suffix.html": (ECHO_PATH + ".extra", False),
+ }
+ for name, (reference, _) in cases.items():
+ path = self.root / directory / name
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(
+ ''
+ f''
+ )
+ (self.root / directory / "unknown.html").write_text(
+ ''
+ f''
+ )
+ (self.root / directory / "script.window.js").write_text(
+ f'const echoURL = content => `{ECHO_PATH}?content=${{encodeURIComponent(content)}}`;'
+ )
+ discovered = enumerate_cases(self.root, dir_prefixes=(directory,))
+ self.assertEqual(
+ sorted(case.case_path.split("?")[0] for case in discovered),
+ sorted([directory + "/" + name for name, (_, allowed) in cases.items() if allowed]
+ + [directory + "/script.window.js"]),
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()