chore: upgrade Lance and rename score to _distance (#398)

BREAKING CHANGE: The `score` column has been renamed to `_distance` to
more accurately describe the semantics (smaller means closer / better).

---------

Co-authored-by: Lei Xu <lei@lancedb.com>

python/lancedb/query.py

@@ -73,8 +73,8 @@ class LanceQueryBuilder:
     ...       .select(["b"])
     ...       .limit(2)
     ...       .to_df())
-       b      vector  score
-    0  6  [0.4, 0.4]    0.0
+       b      vector  _distance
+    0  6  [0.4, 0.4]        0.0
     """
 
     def __init__(
@@ -205,7 +205,7 @@ class LanceQueryBuilder:
         """
         Execute the query and return the results as a pandas DataFrame.
         In addition to the selected columns, LanceDB also returns a vector
-        and also the "score" column which is the distance between the query
+        and also the "_distance" column which is the distance between the query
         vector and the returned vector.
         """
 
@@ -217,7 +217,7 @@ class LanceQueryBuilder:
         [Apache Arrow Table](https://arrow.apache.org/docs/python/generated/pyarrow.Table.html#pyarrow.Table).
 
         In addition to the selected columns, LanceDB also returns a vector
-        and also the "score" column which is the distance between the query
+        and also the "_distance" column which is the distance between the query
         vector and the returned vectors.
         """
         vector = self._query if isinstance(self._query, list) else self._query.tolist()

python/lancedb/table.py

@@ -47,10 +47,15 @@ def _sanitize_data(data, schema, on_bad_vectors, fill_value):
     if isinstance(data, dict):
         data = vec_to_table(data)
     if pd is not None and isinstance(data, pd.DataFrame):
-        data = pa.Table.from_pandas(data)
+        data = pa.Table.from_pandas(data, preserve_index=False)
         data = _sanitize_schema(
             data, schema=schema, on_bad_vectors=on_bad_vectors, fill_value=fill_value
         )
+        # Do not serialize Pandas metadata
+        metadata = data.schema.metadata if data.schema.metadata is not None else {}
+        metadata = {k: v for k, v in metadata.items() if k != b"pandas"}
+        schema = data.schema.with_metadata(metadata)
+        data = pa.Table.from_arrays(data.columns, schema=schema)
     if not isinstance(data, (pa.Table, Iterable)):
         raise TypeError(f"Unsupported data type: {type(data)}")
     return data
@@ -85,9 +90,9 @@ class Table(ABC):
     Can query the table with [Table.search][lancedb.table.Table.search].
 
     >>> table.search([0.4, 0.4]).select(["b"]).to_df()
-       b      vector  score
-    0  4  [0.5, 1.3]   0.82
-    1  2  [1.1, 1.2]   1.13
+       b      vector  _distance
+    0  4  [0.5, 1.3]       0.82
+    1  2  [1.1, 1.2]       1.13
 
     Search queries are much faster when an index is created. See
     [Table.create_index][lancedb.table.Table.create_index].
@@ -196,7 +201,7 @@ class Table(ABC):
         LanceQueryBuilder
             A query builder object representing the query.
             Once executed, the query returns selected columns, the vector,
-            and also the "score" column which is the distance between the query
+            and also the "_distance" column which is the distance between the query
             vector and the returned vector.
         """
         raise NotImplementedError
@@ -457,7 +462,7 @@ class LanceTable(Table):
         LanceQueryBuilder
             A query builder object representing the query.
             Once executed, the query returns selected columns, the vector,
-            and also the "score" column which is the distance between the query
+            and also the "_distance" column which is the distance between the query
             vector and the returned vector.
         """
         if isinstance(query, str):

python/pyproject.toml

@@ -2,7 +2,7 @@
 name = "lancedb"
 version = "0.1.16"
 dependencies = [
-    "pylance==0.5.10",
+    "pylance==0.6.1",
     "ratelimiter",
     "retry",
     "tqdm",

python/tests/test_fts.py

@@ -66,7 +66,7 @@ def test_search_index(tmp_path, table):
     results = ldb.fts.search_index(index, query="puppy", limit=10)
     assert len(results) == 2
     assert len(results[0]) == 10  # row_ids
-    assert len(results[1]) == 10  # scores
+    assert len(results[1]) == 10  # _distance
 
 
 def test_create_index_from_table(tmp_path, table):

python/tests/test_query.py

@@ -108,11 +108,11 @@ def test_query_builder_with_metric(table):
         .limit(1)
         .to_df()
     )
-    assert df_cosine.score[0] == pytest.approx(
+    assert df_cosine._distance[0] == pytest.approx(
         cosine_distance(query, df_cosine.vector[0]),
         abs=1e-6,
     )
-    assert 0 <= df_cosine.score[0] <= 1
+    assert 0 <= df_cosine._distance[0] <= 1
 
 
 def test_query_builder_with_different_vector_column():