diff --git a/python/python/lancedb/query.py b/python/python/lancedb/query.py
index 271028d9..53bcb434 100644
--- a/python/python/lancedb/query.py
+++ b/python/python/lancedb/query.py
@@ -357,7 +357,10 @@ class LanceQueryBuilder(ABC):
             The LanceQueryBuilder object.
         """
         if limit is None or limit <= 0:
-            self._limit = None
+            if isinstance(self, LanceVectorQueryBuilder):
+                raise ValueError("Limit is required for ANN/KNN queries")
+            else:
+                self._limit = None
         else:
             self._limit = limit
         return self
diff --git a/python/python/tests/test_query.py b/python/python/tests/test_query.py
index 30eba26e..ae50c991 100644
--- a/python/python/tests/test_query.py
+++ b/python/python/tests/test_query.py
@@ -117,6 +117,18 @@ def test_query_builder(table):
     assert all(np.array(rs[0]["vector"]) == [1, 2])
 
 
+def test_vector_query_with_no_limit(table):
+    with pytest.raises(ValueError):
+        LanceVectorQueryBuilder(table, [0, 0], "vector").limit(0).select(
+            ["id", "vector"]
+        ).to_list()
+
+    with pytest.raises(ValueError):
+        LanceVectorQueryBuilder(table, [0, 0], "vector").limit(None).select(
+            ["id", "vector"]
+        ).to_list()
+
+
 def test_query_builder_batches(table):
     rs = (
         LanceVectorQueryBuilder(table, [0, 0], "vector")