diff --git a/python/python/tests/test_catalog.py b/python/python/tests/test_catalog.py index b259778dd..95b03c6e7 100644 --- a/python/python/tests/test_catalog.py +++ b/python/python/tests/test_catalog.py @@ -3,13 +3,13 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import json -from threading import Thread import pytest import lancedb from lancedb.db import AsyncConnection, DBConnection from lancedb.remote.errors import HttpError +from utils import start_server_thread @pytest.fixture @@ -38,8 +38,7 @@ def catalog_server(): do_POST = handle_request with ThreadingHTTPServer(("127.0.0.1", 0), Handler) as server: - thread = Thread(target=server.serve_forever, daemon=True) - thread.start() + thread = start_server_thread(server, daemon=True) try: yield f"http://127.0.0.1:{server.server_port}", requests, responses finally: diff --git a/python/python/tests/test_first_class_function_slice2.py b/python/python/tests/test_first_class_function_slice2.py index 47c021b15..0e8aca4e9 100644 --- a/python/python/tests/test_first_class_function_slice2.py +++ b/python/python/tests/test_first_class_function_slice2.py @@ -15,7 +15,6 @@ import os from pathlib import Path import subprocess import sys -import threading import urllib.parse from typing import Optional @@ -34,6 +33,7 @@ from lancedb.functions import ( udf, ) from lancedb.secrets import EnvVarSecret +from utils import start_server_thread THRESHOLD = 20 _CACHE = None @@ -295,13 +295,15 @@ class Handler(http.server.BaseHTTPRequestHandler): self.wfile.write(payload) -server = http.server.ThreadingHTTPServer(("localhost", 0), Handler) -threading.Thread(target=server.serve_forever, daemon=True).start() +server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) +threading.Thread( + target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True +).start() try: db = lancedb.connect( "db://dev", api_key="API_KEY_SENTINEL", - host_override="http://localhost:%d" % server.server_address[1], + host_override="http://127.0.0.1:%d" % server.server_address[1], client_config={"retry_config": {"retries": 0}}, ) db.create_secret("openai-prod", "SECRET_VALUE_SENTINEL") @@ -1731,11 +1733,10 @@ def _mock_remote_function_catalog(): response = {"functions": []} self._write_response(200, response) - with http.server.HTTPServer(("localhost", 0), Handler) as server: - thread = threading.Thread(target=server.serve_forever) - thread.start() + with http.server.HTTPServer(("127.0.0.1", 0), Handler) as server: + thread = start_server_thread(server) try: - yield f"http://localhost:{server.server_address[1]}", state + yield f"http://127.0.0.1:{server.server_address[1]}", state finally: server.shutdown() thread.join() diff --git a/python/python/tests/test_listing.py b/python/python/tests/test_listing.py index 2d1e6497a..4cf37b631 100644 --- a/python/python/tests/test_listing.py +++ b/python/python/tests/test_listing.py @@ -5,12 +5,12 @@ import asyncio from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import json from pathlib import Path -from threading import Thread from urllib.parse import parse_qs, urlsplit import lancedb from lancedb.remote.errors import HttpError import pytest +from utils import start_server_thread @pytest.fixture @@ -31,8 +31,7 @@ def listing_server(): do_POST = handle_request with ThreadingHTTPServer(("127.0.0.1", 0), Handler) as server: - thread = Thread(target=server.serve_forever, daemon=True) - thread.start() + thread = start_server_thread(server, daemon=True) try: yield f"http://127.0.0.1:{server.server_port}", requests, responses finally: diff --git a/python/python/tests/test_materialized_views.py b/python/python/tests/test_materialized_views.py index 85c02ccc3..891b49164 100644 --- a/python/python/tests/test_materialized_views.py +++ b/python/python/tests/test_materialized_views.py @@ -5,7 +5,6 @@ import contextlib import http.server import json -import threading import lancedb import pytest @@ -13,6 +12,7 @@ import pyarrow as pa from lancedb.db import AsyncConnection from lancedb.materialized_view import MaterializedViewDefinition from lancedb.remote.db import RemoteDBConnection +from utils import start_server_thread STABLE_ROW_IDS = {"new_table_enable_stable_row_ids": "true"} @@ -50,11 +50,10 @@ def mock_remote_materialized_view_refresh(): self.end_headers() self.wfile.write(encoded) - with http.server.HTTPServer(("localhost", 0), Handler) as server: - thread = threading.Thread(target=server.serve_forever) - thread.start() + with http.server.HTTPServer(("127.0.0.1", 0), Handler) as server: + thread = start_server_thread(server) try: - yield f"http://localhost:{server.server_address[1]}", requests + yield f"http://127.0.0.1:{server.server_address[1]}", requests finally: server.shutdown() thread.join() diff --git a/python/python/tests/test_namespace.py b/python/python/tests/test_namespace.py index c70ba3d8e..663172e77 100644 --- a/python/python/tests/test_namespace.py +++ b/python/python/tests/test_namespace.py @@ -9,12 +9,12 @@ import importlib import warnings import http.server import json -import threading from urllib.parse import parse_qs, unquote, urlparse import pytest import pyarrow as pa import lancedb from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError +from utils import start_server_thread from lancedb.listing import AsyncListing, Listing from lancedb.materialized_view import DEFINITION_FORMAT, DEFINITION_META_KEY from lancedb.namespace import _MAX_QUERY_K, _query_to_namespace_request @@ -267,8 +267,7 @@ def paginated_rest_namespace(): pass with http.server.HTTPServer(("127.0.0.1", 0), Handler) as server: - thread = threading.Thread(target=server.serve_forever) - thread.start() + thread = start_server_thread(server) try: yield {"uri": f"http://127.0.0.1:{server.server_port}"}, requests finally: @@ -1205,7 +1204,7 @@ class TestPushdownOperations: Rust so reads carry the x-lancedb-min-timestamp read-freshness header.""" db = lancedb.connect_namespace( "rest", - {"uri": "http://localhost:12345"}, + {"uri": "http://127.0.0.1:12345"}, namespace_client_pushdown_operations=["QueryTable"], ) assert db._route_pushdown_to_rust is True @@ -1222,7 +1221,7 @@ class TestPushdownOperations: for the async path omitting the freshness header).""" db = lancedb.connect_namespace_async( "rest", - {"uri": "http://localhost:12345"}, + {"uri": "http://127.0.0.1:12345"}, namespace_client_pushdown_operations=["QueryTable"], ) assert db._route_pushdown_to_rust is True diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 0b32376e8..415f8be3c 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -34,6 +34,7 @@ from lancedb.remote import ClientConfig from lancedb.remote.errors import HttpError, RetryError import pytest import pyarrow as pa +from utils import start_server_thread def make_mock_http_handler(handler): @@ -56,16 +57,15 @@ def test_connect_rejects_invalid_cloud_dns_hostname(db_name): @contextlib.contextmanager def mock_lancedb_connection(handler, **client_config): with http.server.HTTPServer( - ("localhost", 0), make_mock_http_handler(handler) + ("127.0.0.1", 0), make_mock_http_handler(handler) ) as server: port = server.server_address[1] - handle = threading.Thread(target=server.serve_forever) - handle.start() + handle = start_server_thread(server) db = lancedb.connect( "db://dev", api_key="fake", - host_override=f"http://localhost:{port}", + host_override=f"http://127.0.0.1:{port}", client_config={ "retry_config": {"retries": 2}, "timeout_config": { @@ -85,16 +85,15 @@ def mock_lancedb_connection(handler, **client_config): @contextlib.asynccontextmanager async def mock_lancedb_connection_async(handler, **client_config): with http.server.HTTPServer( - ("localhost", 0), make_mock_http_handler(handler) + ("127.0.0.1", 0), make_mock_http_handler(handler) ) as server: port = server.server_address[1] - handle = threading.Thread(target=server.serve_forever) - handle.start() + handle = start_server_thread(server) db = await lancedb.connect_async( "db://dev", api_key="fake", - host_override=f"http://localhost:{port}", + host_override=f"http://127.0.0.1:{port}", client_config={ "retry_config": {"retries": 2}, "timeout_config": { @@ -660,16 +659,15 @@ def test_remote_table_open_does_not_require_picklable_client_config(): request.end_headers() with http.server.HTTPServer( - ("localhost", 0), make_mock_http_handler(handler) + ("127.0.0.1", 0), make_mock_http_handler(handler) ) as server: port = server.server_address[1] - handle = threading.Thread(target=server.serve_forever) - handle.start() + handle = start_server_thread(server) try: db = lancedb.connect( "db://dev", api_key="fake", - host_override=f"http://localhost:{port}", + host_override=f"http://127.0.0.1:{port}", client_config={ "retry_config": {"retries": 0}, "timeout_config": {"connect_timeout": 2, "read_timeout": 2}, @@ -3249,17 +3247,16 @@ def test_header_provider_with_sync_connection(): # Create connection with custom client config with http.server.HTTPServer( - ("localhost", 0), make_mock_http_handler(handler) + ("127.0.0.1", 0), make_mock_http_handler(handler) ) as server: port = server.server_address[1] - handle = threading.Thread(target=server.serve_forever) - handle.start() + handle = start_server_thread(server) try: db = lancedb.connect( "db://dev", api_key="fake", - host_override=f"http://localhost:{port}", + host_override=f"http://127.0.0.1:{port}", client_config={ "retry_config": {"retries": 2}, "timeout_config": {"connect_timeout": 1}, @@ -3463,7 +3460,7 @@ def _remote_fork_child(port: int, queue) -> None: db = lancedb.connect( "db://dev", api_key="fake", - host_override=f"http://localhost:{port}", + host_override=f"http://127.0.0.1:{port}", client_config={ "retry_config": {"retries": 0}, "timeout_config": {"connect_timeout": 2, "read_timeout": 2}, @@ -3497,17 +3494,16 @@ def test_remote_connection_after_fork(): request.end_headers() request.wfile.write(b'{"tables": []}') - server = http.server.HTTPServer(("localhost", 0), make_mock_http_handler(handler)) + server = http.server.HTTPServer(("127.0.0.1", 0), make_mock_http_handler(handler)) port = server.server_address[1] - server_thread = threading.Thread(target=server.serve_forever) - server_thread.start() + server_thread = start_server_thread(server) try: # Hit the server in the parent first so the runtime + LOOP are warm # before fork; a fresh child must still succeed. parent_db = lancedb.connect( "db://dev", api_key="fake", - host_override=f"http://localhost:{port}", + host_override=f"http://127.0.0.1:{port}", client_config={ "retry_config": {"retries": 0}, "timeout_config": {"connect_timeout": 2, "read_timeout": 2}, @@ -3563,15 +3559,14 @@ def test_inherited_remote_table_reopens_after_fork(): request.send_response(404) request.end_headers() - server = http.server.HTTPServer(("localhost", 0), make_mock_http_handler(handler)) + server = http.server.HTTPServer(("127.0.0.1", 0), make_mock_http_handler(handler)) port = server.server_address[1] - server_thread = threading.Thread(target=server.serve_forever) - server_thread.start() + server_thread = start_server_thread(server) try: db = lancedb.connect( "db://dev", api_key="fake", - host_override=f"http://localhost:{port}", + host_override=f"http://127.0.0.1:{port}", client_config={ "retry_config": {"retries": 0}, "timeout_config": {"connect_timeout": 2, "read_timeout": 2}, diff --git a/python/python/tests/test_torch.py b/python/python/tests/test_torch.py index 38a367330..500378fef 100644 --- a/python/python/tests/test_torch.py +++ b/python/python/tests/test_torch.py @@ -9,13 +9,13 @@ import multiprocessing as mp import pickle import re import sys -import threading import lancedb import pyarrow as pa import pytest from lancedb.permutation import Permutation, Permutations, permutation_builder from lancedb.util import tbl_to_tensor +from utils import start_server_thread torch = pytest.importorskip("torch") @@ -100,16 +100,15 @@ def _remote_dataset_handler(request): @contextlib.contextmanager def _remote_dataset_table(): with http.server.ThreadingHTTPServer( - ("localhost", 0), _make_mock_http_handler(_remote_dataset_handler) + ("127.0.0.1", 0), _make_mock_http_handler(_remote_dataset_handler) ) as server: port = server.server_address[1] - handle = threading.Thread(target=server.serve_forever) - handle.start() + handle = start_server_thread(server) try: db = lancedb.connect( "db://dev", api_key="fake", - host_override=f"http://localhost:{port}", + host_override=f"http://127.0.0.1:{port}", client_config={ "retry_config": {"retries": 0}, "timeout_config": {"connect_timeout": 2, "read_timeout": 2}, @@ -349,7 +348,7 @@ def _remote_multiworker_dataloader_target(port: int, result_queue): db = lancedb.connect( "db://dev", api_key="fake", - host_override=f"http://localhost:{port}", + host_override=f"http://127.0.0.1:{port}", client_config={ "retry_config": {"retries": 0}, "timeout_config": {"connect_timeout": 2, "read_timeout": 2}, @@ -419,11 +418,10 @@ def test_permutation_dataloader_fork_workers(tmp_path): ) def test_remote_permutation_dataloader_fork_workers(): with http.server.ThreadingHTTPServer( - ("localhost", 0), _make_mock_http_handler(_remote_dataset_handler) + ("127.0.0.1", 0), _make_mock_http_handler(_remote_dataset_handler) ) as server: port = server.server_address[1] - handle = threading.Thread(target=server.serve_forever) - handle.start() + handle = start_server_thread(server) try: ctx = mp.get_context("spawn") queue = ctx.Queue() diff --git a/python/python/tests/utils.py b/python/python/tests/utils.py index ca72a4dfd..7c4c30b71 100644 --- a/python/python/tests/utils.py +++ b/python/python/tests/utils.py @@ -153,6 +153,15 @@ class MockPermutationServer: return self._arrow(request, pa.table({"id": pa.array([], pa.int64())})) +def start_server_thread(server, daemon=False) -> threading.Thread: + """Serve on a thread; a short poll interval lets ``shutdown()`` return promptly.""" + thread = threading.Thread( + target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=daemon + ) + thread.start() + return thread + + def _make_handler(serve): class MockLanceDBHandler(http.server.BaseHTTPRequestHandler): def do_GET(self): @@ -175,15 +184,14 @@ def mock_remote_table(server): serialize, hiding the prefetch overlap under test. """ with http.server.ThreadingHTTPServer( - ("localhost", 0), _make_handler(server) + ("127.0.0.1", 0), _make_handler(server) ) as srv: - thread = threading.Thread(target=srv.serve_forever) - thread.start() + thread = start_server_thread(srv) try: db = lancedb.connect( "db://dev", api_key="fake", - host_override=f"http://localhost:{srv.server_address[1]}", + host_override=f"http://127.0.0.1:{srv.server_address[1]}", client_config={"timeout_config": {"connect_timeout": 5}}, ) yield db.open_table(server.name)