fix: error during deserialization of "INVERTED" index type

Cargo.toml

@@ -20,15 +20,15 @@ keywords = ["lancedb", "lance", "database", "vector", "search"]
 categories = ["database-implementations"]
 
 [workspace.dependencies]
-lance = { "version" = "=0.18.3", "features" = [
+lance = { "version" = "=0.19.1", "features" = [
     "dynamodb",
-], git = "https://github.com/lancedb/lance.git", tag = "v0.18.3-beta.2" }
+]}
-lance-index = { "version" = "=0.18.3", git = "https://github.com/lancedb/lance.git", tag = "v0.18.3-beta.2" }
+lance-index = { "version" = "=0.19.1"}
-lance-linalg = { "version" = "=0.18.3", git = "https://github.com/lancedb/lance.git", tag = "v0.18.3-beta.2" }
+lance-linalg = { "version" = "=0.19.1"}
-lance-table = { "version" = "=0.18.3", git = "https://github.com/lancedb/lance.git", tag = "v0.18.3-beta.2" }
+lance-table = { "version" = "=0.19.1"}
-lance-testing = { "version" = "=0.18.3", git = "https://github.com/lancedb/lance.git", tag = "v0.18.3-beta.2" }
+lance-testing = { "version" = "=0.19.1"}
-lance-datafusion = { "version" = "=0.18.3", git = "https://github.com/lancedb/lance.git", tag = "v0.18.3-beta.2" }
+lance-datafusion = { "version" = "=0.19.1" }
-lance-encoding = { "version" = "=0.18.3", git = "https://github.com/lancedb/lance.git", tag = "v0.18.3-beta.2" }
+lance-encoding = { "version" = "=0.19.1" }
 # Note that this one does not include pyarrow
 arrow = { version = "52.2", optional = false }
 arrow-array = "52.2"

python/pyproject.toml

@@ -3,7 +3,7 @@ name = "lancedb"
 # version in Cargo.toml
 dependencies = [
     "deprecation",
-    "pylance==0.18.3-beta.2",
+    "pylance==0.19.1",
     "requests>=2.31.0",
     "tqdm>=4.27.0",
     "pydantic>=1.10",

rust/lancedb/src/index.rs

@@ -119,7 +119,9 @@ pub enum IndexType {
     #[serde(alias = "LABEL_LIST")]
     LabelList,
     // FTS
+    #[serde(alias = "INVERTED")]
     FTS,
+    INVERTED,
 }
 
 impl std::fmt::Display for IndexType {
@@ -132,6 +134,7 @@ impl std::fmt::Display for IndexType {
             Self::Bitmap => write!(f, "BITMAP"),
             Self::LabelList => write!(f, "LABEL_LIST"),
             Self::FTS => write!(f, "FTS"),
+            Self::INVERTED => write!(f, "FTS"),
         }
     }
 }

rust/lancedb/src/table.rs

@@ -82,6 +82,7 @@ pub mod merge;
 pub use chrono::Duration;
 pub use lance::dataset::optimize::CompactionOptions;
 pub use lance_index::optimize::OptimizeOptions;
+use lance_index::scalar::inverted::TokenizerConfig;
 
 /// Defines the type of column
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -1568,6 +1569,7 @@ impl NativeTable {
         let mut dataset = self.dataset.get_mut().await?;
         let fts_params = lance_index::scalar::InvertedIndexParams {
             with_position: fts_opts.with_position,
+            tokenizer_config: TokenizerConfig::default()
         };
         dataset
             .create_index(
@@ -2002,7 +2004,7 @@ impl TableInternal for NativeTable {
         self.dataset
             .get_mut()
             .await?
-            .add_columns(transforms, read_columns)
+            .add_columns(transforms, read_columns, None)
             .await?;
         Ok(())
     }