Bump version: 0.39.0-beta.1 → 0.39.0-beta.2

.bumpversion.toml

@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.39.0-beta.3"
+current_version = "0.39.0-beta.2"
 parse = """(?x)
     (?P<major>0|[1-9]\\d*)\\.
     (?P<minor>0|[1-9]\\d*)\\.

Cargo.lock

@@ -5474,7 +5474,7 @@ dependencies = [
 
 [[package]]
 name = "lancedb"
-version = "0.39.0-beta.3"
+version = "0.39.0-beta.1"
 dependencies = [
  "ahash",
  "anyhow",
@@ -5565,7 +5565,7 @@ dependencies = [
 
 [[package]]
 name = "lancedb-nodejs"
-version = "0.39.0-beta.3"
+version = "0.39.0-beta.1"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -5590,7 +5590,7 @@ dependencies = [
 
 [[package]]
 name = "lancedb-python"
-version = "0.39.0-beta.3"
+version = "0.39.0-beta.1"
 dependencies = [
  "arrow",
  "async-trait",

docs/src/java/java.md

@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
 <dependency>
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-core</artifactId>
-    <version>0.39.0-beta.3</version>
+    <version>0.39.0-beta.2</version>
 </dependency>
 ```
 

java/lancedb-core/pom.xml

@@ -8,7 +8,7 @@
     <parent>
       <groupId>com.lancedb</groupId>
       <artifactId>lancedb-parent</artifactId>
-      <version>0.39.0-beta.3</version>
+      <version>0.39.0-beta.2</version>
       <relativePath>../pom.xml</relativePath>
     </parent>
 

java/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.lancedb</groupId>
     <artifactId>lancedb-parent</artifactId>
-    <version>0.39.0-beta.3</version>
+    <version>0.39.0-beta.2</version>
     <packaging>pom</packaging>
     <name>${project.artifactId}</name>
     <description>LanceDB Java SDK Parent POM</description>

nodejs/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.39.0-beta.3"
+version = "0.39.0-beta.2"
 publish = false
 license.workspace = true
 description.workspace = true

nodejs/npm/darwin-arm64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.39.0-beta.3",
+	"version": "0.39.0-beta.2",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",

nodejs/npm/linux-arm64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.39.0-beta.3",
+	"version": "0.39.0-beta.2",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",

nodejs/npm/linux-arm64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.39.0-beta.3",
+	"version": "0.39.0-beta.2",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",

nodejs/npm/linux-x64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.39.0-beta.3",
+	"version": "0.39.0-beta.2",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",

nodejs/npm/linux-x64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.39.0-beta.3",
+	"version": "0.39.0-beta.2",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",

nodejs/npm/win32-arm64-msvc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.39.0-beta.3",
+  "version": "0.39.0-beta.2",
   "os": [
     "win32"
   ],

nodejs/npm/win32-x64-msvc/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.39.0-beta.3",
+	"version": "0.39.0-beta.2",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",

nodejs/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.39.0-beta.3",
+  "version": "0.39.0-beta.2",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",

python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.39.0-beta.3"
+version = "0.39.0-beta.2"
 publish = false
 edition.workspace = true
 description = "Python bindings for LanceDB"

rust/lancedb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb"
-version = "0.39.0-beta.3"
+version = "0.39.0-beta.2"
 edition.workspace = true
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
 license.workspace = true

rust/lancedb/src/remote/db.rs

@@ -688,7 +688,6 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
     async fn list_jobs(&self) -> Result<Vec<JobInfo>> {
         let mut out = Vec::new();
         let mut page_token: Option<String> = None;
-        let mut seen_page_tokens = HashSet::new();
         for page in 0..MAX_LIST_JOBS_PAGES {
             let mut body = serde_json::json!({});
             if let Some(token) = &page_token {
@@ -697,8 +696,7 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
             let req = self.client.post("/v1/jobs/list").json(&body);
             let (request_id, rsp) = self.client.send(req).await?;
             let rsp = self.client.check_response(&request_id, rsp).await?;
-            let status = rsp.status();
-            let body: RemoteListJobsResponse = rsp.json().await.err_to_http(request_id.clone())?;
+            let body: RemoteListJobsResponse = rsp.json().await.err_to_http(request_id)?;
             out.extend(body.jobs.into_iter().map(|row| JobInfo {
                 job_id: row.job_id,
                 table: row.table,
@@ -706,17 +704,10 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
                 state: job_state_to_client(&row.state),
                 created_at_millis: row.created_at_millis,
             }));
-            let Some(next_page_token) = body.page_token.filter(|token| !token.is_empty()) else {
+            page_token = body.page_token;
+            if page_token.is_none() {
                 break;
-            };
-            if !seen_page_tokens.insert(next_page_token.clone()) {
-                return Err(Error::Http {
-                    source: "Job listing response repeated a page_token".into(),
-                    request_id,
-                    status_code: Some(status),
-                });
             }
-            page_token = Some(next_page_token);
             if page + 1 == MAX_LIST_JOBS_PAGES {
                 log::warn!(
                     "list_jobs truncated after {} pages ({} jobs)",
@@ -2631,37 +2622,6 @@ mod tests {
         assert_eq!(jobs[2].state, "failed");
     }
 
-    #[tokio::test]
-    async fn test_list_jobs_rejects_a_page_token_cycle() {
-        let requests = Arc::new(AtomicUsize::new(0));
-        let seen = requests.clone();
-        let conn = Connection::new_with_handler(move |request| {
-            let body: serde_json::Value =
-                serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
-            match seen.fetch_add(1, Ordering::SeqCst) {
-                0 => assert!(body.get("page_token").is_none()),
-                _ => assert_eq!(body["page_token"], "loop"),
-            }
-            http::Response::builder()
-                .status(200)
-                .body(r#"{"jobs": [], "page_token": "loop"}"#)
-                .unwrap()
-        });
-
-        let error = conn.list_jobs().await.unwrap_err();
-        assert!(
-            matches!(
-                &error,
-                Error::Http {
-                    status_code: Some(http::StatusCode::OK),
-                    ..
-                }
-            ),
-            "got {error:?}"
-        );
-        assert_eq!(requests.load(Ordering::SeqCst), 2);
-    }
-
     #[tokio::test]
     async fn test_open_job() {
         let conn = Connection::new_with_handler(|request| {

rust/lancedb/src/table/computed_columns.rs

@@ -759,33 +759,14 @@ fn function_output_field(name: &str, nullable: bool, raw: &str) -> Result<JsonAr
     Ok(field)
 }
 
-/// Whether two fields describe the same Function output.
-///
-/// `compare_identity` covers the field's own name and nullability. Struct
-/// children carry both as part of the declaration and compare with it on. List
-/// children do not: Lance rewrites a list item's name and nullability when it
-/// writes, so a stored `fixed_size_list<item: float not null>` comes back as
-/// `fixed_size_list<item: float>` and never matches the declaration again.
-/// Comparing those by type alone keeps this agreeing with the server, which
-/// draws the same distinction and is what accepted the column when it was
-/// declared.
-fn function_output_field_matches(
-    expected: &ArrowField,
-    actual: &ArrowField,
-    compare_identity: bool,
-) -> bool {
-    if compare_identity
-        && (expected.name() != actual.name() || expected.is_nullable() != actual.is_nullable())
-    {
-        return false;
-    }
-    match (expected.is_blob_v2(), actual.is_blob_v2()) {
-        (false, false) => function_output_type_matches(expected.data_type(), actual.data_type()),
-        (true, true) => {
+fn function_output_field_matches(expected: &ArrowField, actual: &ArrowField) -> bool {
+    expected.name() == actual.name()
+        && expected.is_nullable() == actual.is_nullable()
+        && if expected.is_blob_v2() {
             has_supported_blob_v2_layout(expected) && has_supported_blob_v2_layout(actual)
+        } else {
+            function_output_type_matches(expected.data_type(), actual.data_type())
         }
-        _ => false,
-    }
 }
 
 fn function_output_type_matches(expected: &DataType, actual: &DataType) -> bool {
@@ -798,19 +779,33 @@ fn function_output_type_matches(expected: &DataType, actual: &DataType) -> bool
                 && expected
                     .iter()
                     .zip(actual)
-                    .all(|(expected, actual)| function_output_field_matches(expected, actual, true))
+                    .all(|(expected, actual)| function_output_field_matches(expected, actual))
         }
         (DataType::List(expected), DataType::List(actual))
         | (DataType::LargeList(expected), DataType::LargeList(actual)) => {
-            function_output_field_matches(expected, actual, false)
+            function_output_field_matches(expected, actual)
         }
         (
             DataType::FixedSizeList(expected, expected_size),
             DataType::FixedSizeList(actual, actual_size),
-        ) => expected_size == actual_size && function_output_field_matches(expected, actual, false),
+        ) => expected_size == actual_size && function_output_field_matches(expected, actual),
         (DataType::Map(expected, expected_sorted), DataType::Map(actual, actual_sorted)) => {
-            expected_sorted == actual_sorted
-                && function_output_field_matches(expected, actual, true)
+            expected_sorted == actual_sorted && function_output_field_matches(expected, actual)
+        }
+        _ => false,
+    }
+}
+
+fn function_output_type_has_blob(data_type: &DataType) -> bool {
+    match data_type {
+        DataType::Struct(fields) => fields
+            .iter()
+            .any(|field| field.is_blob_v2() || function_output_type_has_blob(field.data_type())),
+        DataType::List(field)
+        | DataType::LargeList(field)
+        | DataType::FixedSizeList(field, _)
+        | DataType::Map(field, _) => {
+            field.is_blob_v2() || function_output_type_has_blob(field.data_type())
         }
         _ => false,
     }
@@ -896,13 +891,16 @@ fn ensure_binding_matches_schema(schema: &ArrowSchema, binding: &FunctionBinding
                 binding.binding_id()
             )));
         }
-        let type_matches = if output.arrow_type == FUNCTION_BLOB_V2_TYPE {
-            has_supported_blob_v2_layout(field)
+        let (type_matches, has_semantic_blob) = if output.arrow_type == FUNCTION_BLOB_V2_TYPE {
+            (has_supported_blob_v2_layout(field), true)
         } else {
             let expected_type = parse_output_arrow_type(&output.arrow_type)?;
             let expected_type = lance_namespace::schema::convert_json_arrow_type(&expected_type)
                 .map_err(|e| invalid_function(format!("invalid Function output type: {e}")))?;
-            function_output_type_matches(&expected_type, field.data_type())
+            (
+                function_output_type_matches(&expected_type, field.data_type()),
+                function_output_type_has_blob(&expected_type),
+            )
         };
         if !type_matches {
             return Err(invalid_function(format!(
@@ -933,16 +931,19 @@ fn ensure_binding_matches_schema(schema: &ArrowSchema, binding: &FunctionBinding
                 binding.binding_id()
             )));
         }
-        // Rebuild from the declaration rather than from the stored field. The
-        // stored field carries Lance's write-time normalization, which would
-        // never round-trip back to the schema the binding recorded -- the same
-        // reason list children compare by type above. Whether the column on
-        // disk still matches is settled by that comparison, not here.
-        output_fields.push(function_output_field(
-            field.name(),
-            true,
-            &output.arrow_type,
-        )?);
+        if has_semantic_blob {
+            output_fields.push(function_output_field(
+                field.name(),
+                true,
+                &output.arrow_type,
+            )?);
+        } else {
+            let json = lance_namespace::schema::arrow_schema_to_json(&ArrowSchema::new(vec![
+                ArrowField::new(field.name().clone(), field.data_type().clone(), true),
+            ]))
+            .map_err(|e| invalid_function(format!("invalid Function output schema: {e}")))?;
+            output_fields.push(json.fields.into_iter().next().unwrap());
+        }
     }
     if let Some(assignment) = binding.assignment() {
         if binding
@@ -1814,69 +1815,6 @@ mod tests {
         assert!(super::validate_declarations(schema, &declarations).is_err());
     }
 
-    #[test]
-    fn list_children_match_by_type_but_struct_children_by_identity() {
-        use arrow_schema::Field as F;
-
-        // Lance rewrites a list item's name and nullability on write, so the
-        // stored field is no longer identical to what was declared. Comparing
-        // those by type keeps a table with a vector output usable.
-        let declared =
-            DataType::FixedSizeList(Arc::new(F::new("item", DataType::Float32, false)), 4);
-        let stored = DataType::FixedSizeList(Arc::new(F::new("item", DataType::Float32, true)), 4);
-        assert!(super::function_output_type_matches(&declared, &stored));
-
-        let renamed =
-            DataType::FixedSizeList(Arc::new(F::new("element", DataType::Float32, true)), 4);
-        assert!(super::function_output_type_matches(&declared, &renamed));
-
-        // The dimension is still part of the declaration.
-        let resized = DataType::FixedSizeList(Arc::new(F::new("item", DataType::Float32, true)), 8);
-        assert!(!super::function_output_type_matches(&declared, &resized));
-
-        // Struct children keep comparing by name and nullability.
-        let struct_declared =
-            DataType::Struct(vec![F::new("changed", DataType::Boolean, false)].into());
-        let struct_nullable =
-            DataType::Struct(vec![F::new("changed", DataType::Boolean, true)].into());
-        let struct_renamed =
-            DataType::Struct(vec![F::new("altered", DataType::Boolean, false)].into());
-        assert!(super::function_output_type_matches(
-            &struct_declared,
-            &struct_declared
-        ));
-        assert!(!super::function_output_type_matches(
-            &struct_declared,
-            &struct_nullable
-        ));
-        assert!(!super::function_output_type_matches(
-            &struct_declared,
-            &struct_renamed
-        ));
-
-        // A list nested inside a struct gets the list rule.
-        let nested_declared = DataType::Struct(
-            vec![F::new(
-                "tokens",
-                DataType::List(Arc::new(F::new("item", DataType::Utf8, false))),
-                true,
-            )]
-            .into(),
-        );
-        let nested_stored = DataType::Struct(
-            vec![F::new(
-                "tokens",
-                DataType::List(Arc::new(F::new("item", DataType::Utf8, true))),
-                true,
-            )]
-            .into(),
-        );
-        assert!(super::function_output_type_matches(
-            &nested_declared,
-            &nested_stored
-        ));
-    }
-
     #[test]
     fn output_arrow_type_grammar_matches_the_shared_golden() {
         let golden: serde_json::Value = serde_json::from_str(include_str!(
@@ -3351,70 +3289,6 @@ mod tests {
         assert!(output_schema.field(0).is_blob_v2());
     }
 
-    #[test]
-    fn binding_accepts_a_lance_normalized_list_child() {
-        // The whole guard, not just the type helper: this also reaches the
-        // output-schema comparison at the end of ensure_binding_matches_schema,
-        // which used to rebuild the schema from the stored field and so failed
-        // on exactly the same normalization.
-        let input = ArrowField::new("value", DataType::Int64, false);
-        let application = FunctionApplication::from_json(
-            &serde_json::json!({
-                "function": {"name": "embed", "version": "fv_embed"},
-                "inputs": [{
-                    "parameter": "value",
-                    "kind": "column",
-                    "value": {"path": "value"}
-                }],
-                "output": {
-                    "kind": "scalar",
-                    "arrow_type": "fixed_size_list<float32, 4>",
-                    "nullable": false
-                }
-            })
-            .to_string(),
-        )
-        .unwrap();
-        let plan = plan_function_application(
-            &ArrowSchema::new(vec![input.clone()]),
-            &application,
-            Some("embedding"),
-        )
-        .unwrap();
-        let binding = binding_from_plan(&plan);
-
-        // The declaration says the item is non-nullable; Lance rewrites it to
-        // nullable on write, so this is what the column looks like on disk.
-        let stored = DataType::FixedSizeList(
-            Arc::new(ArrowField::new("item", DataType::Float32, true)),
-            4,
-        );
-        let output = ArrowField::new("embedding", stored, true).with_metadata(
-            function_computed_column_metadata(binding.binding_id(), 0, &["value".into()]),
-        );
-
-        ensure_binding_matches_schema(&ArrowSchema::new(vec![input.clone(), output]), &binding)
-            .unwrap();
-
-        // A different element type is still a mismatch.
-        let wrong = ArrowField::new(
-            "embedding",
-            DataType::FixedSizeList(
-                Arc::new(ArrowField::new("item", DataType::Float64, true)),
-                4,
-            ),
-            true,
-        )
-        .with_metadata(function_computed_column_metadata(
-            binding.binding_id(),
-            0,
-            &["value".into()],
-        ));
-        assert!(
-            ensure_binding_matches_schema(&ArrowSchema::new(vec![input, wrong]), &binding).is_err()
-        );
-    }
-
     #[test]
     fn test_blob_scalar_binding_accepts_full_logical_layout() {
         let input = crate::blob("image", false);