feat: add the ability to create scalar indices (#679)

This is a pretty direct binding to the underlying lance capability

python/tests/test_table.py

@@ -532,6 +532,33 @@ def test_multiple_vector_columns(db):
     assert result1["text"].iloc[0] != result2["text"].iloc[0]
 
 
+def test_create_scalar_index(db):
+    vec_array = pa.array(
+        [[1, 1], [2, 2], [3, 3], [4, 4], [5, 5]], pa.list_(pa.float32(), 2)
+    )
+    test_data = pa.Table.from_pydict(
+        {"x": ["c", "b", "a", "e", "b"], "y": [1, 2, 3, 4, 5], "vector": vec_array}
+    )
+    table = LanceTable.create(
+        db,
+        "my_table",
+        data=test_data,
+    )
+    table.create_scalar_index("x")
+    indices = table.to_lance().list_indices()
+    assert len(indices) == 1
+    scalar_index = indices[0]
+    assert scalar_index["type"] == "Scalar"
+
+    # Confirm that prefiltering still works with the scalar index column
+    results = table.search().where("x = 'c'").to_arrow()
+    assert results == test_data.slice(0, 1)
+    results = table.search([5, 5]).to_arrow()
+    assert results["_distance"][0].as_py() == 0
+    results = table.search([5, 5]).where("x != 'b'").to_arrow()
+    assert results["_distance"][0].as_py() > 0
+
+
 def test_empty_query(db):
     table = LanceTable.create(
         db,