From db712b0f9916b344009b53e165a4fbab3eb073a7 Mon Sep 17 00:00:00 2001 From: LuQQiu Date: Fri, 24 May 2024 11:49:11 -0700 Subject: [PATCH] feat(java): add table names java api (#1279) Add lancedb-jni and table names API --------- Co-authored-by: Lei Xu --- .github/workflows/java.yml | 85 +++++++ Cargo.toml | 2 +- java/core/lancedb-jni/Cargo.toml | 27 +++ java/core/lancedb-jni/src/connection.rs | 130 +++++++++++ java/core/lancedb-jni/src/error.rs | 220 ++++++++++++++++++ java/core/lancedb-jni/src/ffi.rs | 199 ++++++++++++++++ java/core/lancedb-jni/src/lib.rs | 68 ++++++ java/core/lancedb-jni/src/traits.rs | 122 ++++++++++ java/core/pom.xml | 94 ++++++++ .../java/com/lancedb/lancedb/Connection.java | 120 ++++++++++ .../com/lancedb/lancedb/ConnectionTest.java | 135 +++++++++++ .../dataset_version.lance/_latest.manifest | Bin 0 -> 273 bytes ...0-d51afd07-e3cd-4c76-9b9b-787e13fd55b0.txn | 1 + ...1-336c3e56-33fd-45d8-bbfb-95ebb563cbe0.txn | Bin 0 -> 99 bytes ...2-3344b369-7471-4e23-8865-c949b6e19bc2.txn | Bin 0 -> 99 bytes .../_versions/1.manifest | Bin 0 -> 159 bytes .../_versions/2.manifest | Bin 0 -> 217 bytes .../_versions/3.manifest | Bin 0 -> 273 bytes ...60a9b599-f79f-48a8-bffa-b495762b622a.lance | Bin 0 -> 682 bytes ...a13f68ba-04e6-48b5-bec0-bf54444be5f0.lance | Bin 0 -> 642 bytes .../new_empty_dataset.lance/_latest.manifest | Bin 0 -> 159 bytes ...0-15648e72-076f-4ef1-8b90-10d305b95b3b.txn | 1 + .../_versions/1.manifest | Bin 0 -> 159 bytes .../example_db/test.lance/_latest.manifest | Bin 0 -> 264 bytes ...0-a3689caf-4f6b-4afc-a3c7-97af75661843.txn | 1 + ...1-3f0fa7b9-7311-4945-9b0f-57dff4c04ee2.txn | Bin 0 -> 98 bytes .../test.lance/_versions/1.manifest | Bin 0 -> 209 bytes .../test.lance/_versions/2.manifest | Bin 0 -> 264 bytes ...cd209a1b-00e0-4adf-93b2-2547c866e1ef.lance | Bin 0 -> 694 bytes .../write_stream.lance/_latest.manifest | Bin 0 -> 214 bytes ...0-ea2f0479-36d1-4302-908a-dae45b9eb443.txn | Bin 0 -> 157 bytes .../write_stream.lance/_versions/1.manifest | Bin 0 -> 214 bytes ...665ff491-6dc5-4496-b292-166ed5c2a309.lance | Bin 0 -> 728 bytes java/pom.xml | 129 ++++++++++ 34 files changed, 1333 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/java.yml create mode 100644 java/core/lancedb-jni/Cargo.toml create mode 100644 java/core/lancedb-jni/src/connection.rs create mode 100644 java/core/lancedb-jni/src/error.rs create mode 100644 java/core/lancedb-jni/src/ffi.rs create mode 100644 java/core/lancedb-jni/src/lib.rs create mode 100644 java/core/lancedb-jni/src/traits.rs create mode 100644 java/core/pom.xml create mode 100644 java/core/src/main/java/com/lancedb/lancedb/Connection.java create mode 100644 java/core/src/test/java/com/lancedb/lancedb/ConnectionTest.java create mode 100644 java/core/src/test/resources/example_db/dataset_version.lance/_latest.manifest create mode 100644 java/core/src/test/resources/example_db/dataset_version.lance/_transactions/0-d51afd07-e3cd-4c76-9b9b-787e13fd55b0.txn create mode 100644 java/core/src/test/resources/example_db/dataset_version.lance/_transactions/1-336c3e56-33fd-45d8-bbfb-95ebb563cbe0.txn create mode 100644 java/core/src/test/resources/example_db/dataset_version.lance/_transactions/2-3344b369-7471-4e23-8865-c949b6e19bc2.txn create mode 100644 java/core/src/test/resources/example_db/dataset_version.lance/_versions/1.manifest create mode 100644 java/core/src/test/resources/example_db/dataset_version.lance/_versions/2.manifest create mode 100644 java/core/src/test/resources/example_db/dataset_version.lance/_versions/3.manifest create mode 100644 java/core/src/test/resources/example_db/dataset_version.lance/data/60a9b599-f79f-48a8-bffa-b495762b622a.lance create mode 100644 java/core/src/test/resources/example_db/dataset_version.lance/data/a13f68ba-04e6-48b5-bec0-bf54444be5f0.lance create mode 100644 java/core/src/test/resources/example_db/new_empty_dataset.lance/_latest.manifest create mode 100644 java/core/src/test/resources/example_db/new_empty_dataset.lance/_transactions/0-15648e72-076f-4ef1-8b90-10d305b95b3b.txn create mode 100644 java/core/src/test/resources/example_db/new_empty_dataset.lance/_versions/1.manifest create mode 100644 java/core/src/test/resources/example_db/test.lance/_latest.manifest create mode 100644 java/core/src/test/resources/example_db/test.lance/_transactions/0-a3689caf-4f6b-4afc-a3c7-97af75661843.txn create mode 100644 java/core/src/test/resources/example_db/test.lance/_transactions/1-3f0fa7b9-7311-4945-9b0f-57dff4c04ee2.txn create mode 100644 java/core/src/test/resources/example_db/test.lance/_versions/1.manifest create mode 100644 java/core/src/test/resources/example_db/test.lance/_versions/2.manifest create mode 100644 java/core/src/test/resources/example_db/test.lance/data/cd209a1b-00e0-4adf-93b2-2547c866e1ef.lance create mode 100644 java/core/src/test/resources/example_db/write_stream.lance/_latest.manifest create mode 100644 java/core/src/test/resources/example_db/write_stream.lance/_transactions/0-ea2f0479-36d1-4302-908a-dae45b9eb443.txn create mode 100644 java/core/src/test/resources/example_db/write_stream.lance/_versions/1.manifest create mode 100644 java/core/src/test/resources/example_db/write_stream.lance/data/665ff491-6dc5-4496-b292-166ed5c2a309.lance create mode 100644 java/pom.xml diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml new file mode 100644 index 00000000..c6e45c2d --- /dev/null +++ b/.github/workflows/java.yml @@ -0,0 +1,85 @@ +name: Build and Run Java JNI Tests +on: + push: + branches: + - main + pull_request: + paths: + - java/** + - rust/** + - .github/workflows/java.yml +env: + # This env var is used by Swatinem/rust-cache@v2 for the cache + # key, so we set it to make sure it is always consistent. + CARGO_TERM_COLOR: always + # Disable full debug symbol generation to speed up CI build and keep memory down + # "1" means line tables only, which is useful for panic tracebacks. + RUSTFLAGS: "-C debuginfo=1" + RUST_BACKTRACE: "1" + # according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html + # CI builds are faster with incremental disabled. + CARGO_INCREMENTAL: "0" + CARGO_BUILD_JOBS: "1" +jobs: + linux-build: + runs-on: ubuntu-22.04 + name: ubuntu-22.04 + Java 11 & 17 + defaults: + run: + working-directory: ./java + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - uses: Swatinem/rust-cache@v2 + with: + workspaces: java/core/lancedb-jni + - name: Run cargo fmt + run: cargo fmt --check + working-directory: ./java/core/lancedb-jni + - name: Install dependencies + run: | + sudo apt update + sudo apt install -y protobuf-compiler libssl-dev + - name: Install Java 17 + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 17 + cache: "maven" + - run: echo "JAVA_17=$JAVA_HOME" >> $GITHUB_ENV + - name: Install Java 11 + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 11 + cache: "maven" + - name: Java Style Check + run: mvn checkstyle:check + # Disable because of issues in lancedb rust core code + # - name: Rust Clippy + # working-directory: java/core/lancedb-jni + # run: cargo clippy --all-targets -- -D warnings + - name: Running tests with Java 11 + run: mvn clean test + - name: Running tests with Java 17 + run: | + export JAVA_TOOL_OPTIONS="$JAVA_TOOL_OPTIONS \ + -XX:+IgnoreUnrecognizedVMOptions \ + --add-opens=java.base/java.lang=ALL-UNNAMED \ + --add-opens=java.base/java.lang.invoke=ALL-UNNAMED \ + --add-opens=java.base/java.lang.reflect=ALL-UNNAMED \ + --add-opens=java.base/java.io=ALL-UNNAMED \ + --add-opens=java.base/java.net=ALL-UNNAMED \ + --add-opens=java.base/java.nio=ALL-UNNAMED \ + --add-opens=java.base/java.util=ALL-UNNAMED \ + --add-opens=java.base/java.util.concurrent=ALL-UNNAMED \ + --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED \ + --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED \ + --add-opens=java.base/sun.nio.ch=ALL-UNNAMED \ + --add-opens=java.base/sun.nio.cs=ALL-UNNAMED \ + --add-opens=java.base/sun.security.action=ALL-UNNAMED \ + --add-opens=java.base/sun.util.calendar=ALL-UNNAMED \ + --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED \ + -Djdk.reflect.useDirectMethodHandle=false \ + -Dio.netty.tryReflectionSetAccessible=true" + JAVA_HOME=$JAVA_17 mvn clean test diff --git a/Cargo.toml b/Cargo.toml index 82c5cdd1..3316b1c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["rust/ffi/node", "rust/lancedb", "nodejs", "python"] +members = ["rust/ffi/node", "rust/lancedb", "nodejs", "python", "java/core/lancedb-jni"] # Python package needs to be built by maturin. exclude = ["python"] resolver = "2" diff --git a/java/core/lancedb-jni/Cargo.toml b/java/core/lancedb-jni/Cargo.toml new file mode 100644 index 00000000..ad9381f5 --- /dev/null +++ b/java/core/lancedb-jni/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "lancedb-jni" +description = "JNI bindings for LanceDB" +# TODO modify lancedb/Cargo.toml for version and dependencies +version = "0.4.18" +edition.workspace = true +repository.workspace = true +readme.workspace = true +license.workspace = true +keywords.workspace = true +categories.workspace = true +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies] +lancedb = { path = "../../../rust/lancedb" } +lance = { workspace = true } +arrow = { workspace = true, features = ["ffi"] } +arrow-schema.workspace = true +tokio = "1.23" +jni = "0.21.1" +snafu.workspace = true +lazy_static.workspace = true +serde = { version = "^1" } +serde_json = { version = "1" } diff --git a/java/core/lancedb-jni/src/connection.rs b/java/core/lancedb-jni/src/connection.rs new file mode 100644 index 00000000..750a879b --- /dev/null +++ b/java/core/lancedb-jni/src/connection.rs @@ -0,0 +1,130 @@ +use crate::ffi::JNIEnvExt; +use crate::traits::IntoJava; +use crate::{Error, RT}; +use jni::objects::{JObject, JString, JValue}; +use jni::JNIEnv; +pub const NATIVE_CONNECTION: &str = "nativeConnectionHandle"; +use crate::Result; +use lancedb::connection::{connect, Connection}; + +#[derive(Clone)] +pub struct BlockingConnection { + pub(crate) inner: Connection, +} + +impl BlockingConnection { + pub fn create(dataset_uri: &str) -> Result { + let inner = RT.block_on(connect(dataset_uri).execute())?; + Ok(Self { inner }) + } + + pub fn table_names( + &self, + start_after: Option, + limit: Option, + ) -> Result> { + let mut op = self.inner.table_names(); + if let Some(start_after) = start_after { + op = op.start_after(start_after); + } + if let Some(limit) = limit { + op = op.limit(limit as u32); + } + Ok(RT.block_on(op.execute())?) + } +} + +impl IntoJava for BlockingConnection { + fn into_java<'a>(self, env: &mut JNIEnv<'a>) -> JObject<'a> { + attach_native_connection(env, self) + } +} + +fn attach_native_connection<'local>( + env: &mut JNIEnv<'local>, + connection: BlockingConnection, +) -> JObject<'local> { + let j_connection = create_java_connection_object(env); + // This block sets a native Rust object (Connection) as a field in the Java object (j_Connection). + // Caution: This creates a potential for memory leaks. The Rust object (Connection) is not + // automatically garbage-collected by Java, and its memory will not be freed unless + // explicitly handled. + // + // To prevent memory leaks, ensure the following: + // 1. The Java object (`j_Connection`) should implement the `java.io.Closeable` interface. + // 2. Users of this Java object should be instructed to always use it within a try-with-resources + // statement (or manually call the `close()` method) to ensure that `self.close()` is invoked. + match unsafe { env.set_rust_field(&j_connection, NATIVE_CONNECTION, connection) } { + Ok(_) => j_connection, + Err(err) => { + env.throw_new( + "java/lang/RuntimeException", + format!("Failed to set native handle for Connection: {}", err), + ) + .expect("Error throwing exception"); + JObject::null() + } + } +} + +fn create_java_connection_object<'a>(env: &mut JNIEnv<'a>) -> JObject<'a> { + env.new_object("com/lancedb/lancedb/Connection", "()V", &[]) + .expect("Failed to create Java Lance Connection instance") +} + +#[no_mangle] +pub extern "system" fn Java_com_lancedb_lancedb_Connection_releaseNativeConnection( + mut env: JNIEnv, + j_connection: JObject, +) { + let _: BlockingConnection = unsafe { + env.take_rust_field(j_connection, NATIVE_CONNECTION) + .expect("Failed to take native Connection handle") + }; +} + +#[no_mangle] +pub extern "system" fn Java_com_lancedb_lancedb_Connection_connect<'local>( + mut env: JNIEnv<'local>, + _obj: JObject, + dataset_uri_object: JString, +) -> JObject<'local> { + let dataset_uri: String = ok_or_throw!(env, env.get_string(&dataset_uri_object)).into(); + let blocking_connection = ok_or_throw!(env, BlockingConnection::create(&dataset_uri)); + blocking_connection.into_java(&mut env) +} + +#[no_mangle] +pub extern "system" fn Java_com_lancedb_lancedb_Connection_tableNames<'local>( + mut env: JNIEnv<'local>, + j_connection: JObject, + start_after_obj: JObject, // Optional + limit_obj: JObject, // Optional +) -> JObject<'local> { + ok_or_throw!( + env, + inner_table_names(&mut env, j_connection, start_after_obj, limit_obj) + ) +} + +fn inner_table_names<'local>( + env: &mut JNIEnv<'local>, + j_connection: JObject, + start_after_obj: JObject, // Optional + limit_obj: JObject, // Optional +) -> Result> { + let start_after = env.get_string_opt(&start_after_obj)?; + let limit = env.get_int_opt(&limit_obj)?; + let conn = + unsafe { env.get_rust_field::<_, _, BlockingConnection>(j_connection, NATIVE_CONNECTION) }?; + let table_names = conn.table_names(start_after, limit)?; + drop(conn); + let j_names = env.new_object("java/util/ArrayList", "()V", &[])?; + for item in table_names { + let jstr_item = env.new_string(item)?; + let item_jobj = JObject::from(jstr_item); + let item_gen = JValue::Object(&item_jobj); + env.call_method(&j_names, "add", "(Ljava/lang/Object;)Z", &[item_gen])?; + } + Ok(j_names) +} diff --git a/java/core/lancedb-jni/src/error.rs b/java/core/lancedb-jni/src/error.rs new file mode 100644 index 00000000..e6738c3b --- /dev/null +++ b/java/core/lancedb-jni/src/error.rs @@ -0,0 +1,220 @@ +// Copyright 2024 Lance Developers. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::str::Utf8Error; + +use arrow_schema::ArrowError; +use jni::errors::Error as JniError; +use serde_json::Error as JsonError; +use snafu::{Location, Snafu}; + +/// Java Exception types +pub enum JavaException { + IllegalArgumentException, + IOException, + RuntimeException, +} + +impl JavaException { + pub fn as_str(&self) -> &str { + match self { + Self::IllegalArgumentException => "java/lang/IllegalArgumentException", + Self::IOException => "java/io/IOException", + Self::RuntimeException => "java/lang/RuntimeException", + } + } +} +/// TODO(lu) change to lancedb-jni +#[derive(Debug, Snafu)] +#[snafu(visibility(pub))] +pub enum Error { + #[snafu(display("JNI error: {message}, {location}"))] + Jni { message: String, location: Location }, + #[snafu(display("Invalid argument: {message}, {location}"))] + InvalidArgument { message: String, location: Location }, + #[snafu(display("IO error: {message}, {location}"))] + IO { message: String, location: Location }, + #[snafu(display("Arrow error: {message}, {location}"))] + Arrow { message: String, location: Location }, + #[snafu(display("Index error: {message}, {location}"))] + Index { message: String, location: Location }, + #[snafu(display("JSON error: {message}, {location}"))] + JSON { message: String, location: Location }, + #[snafu(display("Dataset at path {path} was not found, {location}"))] + DatasetNotFound { path: String, location: Location }, + #[snafu(display("Dataset already exists: {uri}, {location}"))] + DatasetAlreadyExists { uri: String, location: Location }, + #[snafu(display("Table '{name}' already exists"))] + TableAlreadyExists { name: String }, + #[snafu(display("Table '{name}' was not found"))] + TableNotFound { name: String }, + #[snafu(display("Invalid table name '{name}': {reason}"))] + InvalidTableName { name: String, reason: String }, + #[snafu(display("Embedding function '{name}' was not found: {reason}, {location}"))] + EmbeddingFunctionNotFound { + name: String, + reason: String, + location: Location, + }, + #[snafu(display("Other Lance error: {message}, {location}"))] + OtherLance { message: String, location: Location }, + #[snafu(display("Other LanceDB error: {message}, {location}"))] + OtherLanceDB { message: String, location: Location }, +} + +impl Error { + /// Throw as Java Exception + pub fn throw(&self, env: &mut jni::JNIEnv) { + match self { + Self::InvalidArgument { .. } + | Self::DatasetNotFound { .. } + | Self::DatasetAlreadyExists { .. } + | Self::TableAlreadyExists { .. } + | Self::TableNotFound { .. } + | Self::InvalidTableName { .. } + | Self::EmbeddingFunctionNotFound { .. } => { + self.throw_as(env, JavaException::IllegalArgumentException) + } + Self::IO { .. } | Self::Index { .. } => self.throw_as(env, JavaException::IOException), + Self::Arrow { .. } + | Self::JSON { .. } + | Self::OtherLance { .. } + | Self::OtherLanceDB { .. } + | Self::Jni { .. } => self.throw_as(env, JavaException::RuntimeException), + } + } + + /// Throw as an concrete Java Exception + pub fn throw_as(&self, env: &mut jni::JNIEnv, exception: JavaException) { + let message = &format!( + "Error when throwing Java exception: {}:{}", + exception.as_str(), + self + ); + env.throw_new(exception.as_str(), self.to_string()) + .expect(message); + } +} + +pub type Result = std::result::Result; + +trait ToSnafuLocation { + fn to_snafu_location(&'static self) -> snafu::Location; +} + +impl ToSnafuLocation for std::panic::Location<'static> { + fn to_snafu_location(&'static self) -> snafu::Location { + snafu::Location::new(self.file(), self.line(), self.column()) + } +} + +impl From for Error { + #[track_caller] + fn from(source: JniError) -> Self { + Self::Jni { + message: source.to_string(), + location: std::panic::Location::caller().to_snafu_location(), + } + } +} + +impl From for Error { + #[track_caller] + fn from(source: Utf8Error) -> Self { + Self::InvalidArgument { + message: source.to_string(), + location: std::panic::Location::caller().to_snafu_location(), + } + } +} + +impl From for Error { + #[track_caller] + fn from(source: ArrowError) -> Self { + Self::Arrow { + message: source.to_string(), + location: std::panic::Location::caller().to_snafu_location(), + } + } +} + +impl From for Error { + #[track_caller] + fn from(source: JsonError) -> Self { + Self::JSON { + message: source.to_string(), + location: std::panic::Location::caller().to_snafu_location(), + } + } +} + +impl From for Error { + #[track_caller] + fn from(source: lance::Error) -> Self { + match source { + lance::Error::DatasetNotFound { + path, + source: _, + location, + } => Self::DatasetNotFound { path, location }, + lance::Error::DatasetAlreadyExists { uri, location } => { + Self::DatasetAlreadyExists { uri, location } + } + lance::Error::IO { message, location } => Self::IO { message, location }, + lance::Error::Arrow { message, location } => Self::Arrow { message, location }, + lance::Error::Index { message, location } => Self::Index { message, location }, + lance::Error::InvalidInput { source, location } => Self::InvalidArgument { + message: source.to_string(), + location, + }, + _ => Self::OtherLance { + message: source.to_string(), + location: std::panic::Location::caller().to_snafu_location(), + }, + } + } +} + +impl From for Error { + #[track_caller] + fn from(source: lancedb::Error) -> Self { + match source { + lancedb::Error::InvalidTableName { name, reason } => { + Self::InvalidTableName { name, reason } + } + lancedb::Error::InvalidInput { message } => Self::InvalidArgument { + message, + location: std::panic::Location::caller().to_snafu_location(), + }, + lancedb::Error::TableNotFound { name } => Self::TableNotFound { name }, + lancedb::Error::TableAlreadyExists { name } => Self::TableAlreadyExists { name }, + lancedb::Error::EmbeddingFunctionNotFound { name, reason } => { + Self::EmbeddingFunctionNotFound { + name, + reason, + location: std::panic::Location::caller().to_snafu_location(), + } + } + lancedb::Error::Arrow { source } => Self::Arrow { + message: source.to_string(), + location: std::panic::Location::caller().to_snafu_location(), + }, + lancedb::Error::Lance { source } => Self::from(source), + _ => Self::OtherLanceDB { + message: source.to_string(), + location: std::panic::Location::caller().to_snafu_location(), + }, + } + } +} diff --git a/java/core/lancedb-jni/src/ffi.rs b/java/core/lancedb-jni/src/ffi.rs new file mode 100644 index 00000000..032b938c --- /dev/null +++ b/java/core/lancedb-jni/src/ffi.rs @@ -0,0 +1,199 @@ +// Copyright 2024 Lance Developers. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use core::slice; + +use jni::objects::{JByteBuffer, JObjectArray, JString}; +use jni::sys::jobjectArray; +use jni::{objects::JObject, JNIEnv}; + +use crate::error::{Error, Result}; + +/// TODO(lu) import from lance-jni without duplicate +/// Extend JNIEnv with helper functions. +pub trait JNIEnvExt { + /// Get integers from Java List object. + fn get_integers(&mut self, obj: &JObject) -> Result>; + + /// Get strings from Java List object. + fn get_strings(&mut self, obj: &JObject) -> Result>; + + /// Get strings from Java String[] object. + /// Note that get Option> from Java Optional just doesn't work. + fn get_strings_array(&mut self, obj: jobjectArray) -> Result>; + + /// Get Option from Java Optional. + fn get_string_opt(&mut self, obj: &JObject) -> Result>; + + /// Get Option> from Java Optional>. + fn get_strings_opt(&mut self, obj: &JObject) -> Result>>; + + /// Get Option from Java Optional. + fn get_int_opt(&mut self, obj: &JObject) -> Result>; + + /// Get Option> from Java Optional>. + fn get_ints_opt(&mut self, obj: &JObject) -> Result>>; + + /// Get Option from Java Optional. + fn get_long_opt(&mut self, obj: &JObject) -> Result>; + + /// Get Option from Java Optional. + fn get_u64_opt(&mut self, obj: &JObject) -> Result>; + + /// Get Option<&[u8]> from Java Optional. + fn get_bytes_opt(&mut self, obj: &JObject) -> Result>; + + fn get_optional(&mut self, obj: &JObject, f: F) -> Result> + where + F: FnOnce(&mut JNIEnv, &JObject) -> Result; +} + +impl JNIEnvExt for JNIEnv<'_> { + fn get_integers(&mut self, obj: &JObject) -> Result> { + let list = self.get_list(obj)?; + let mut iter = list.iter(self)?; + let mut results = Vec::with_capacity(list.size(self)? as usize); + while let Some(elem) = iter.next(self)? { + let int_obj = self.call_method(elem, "intValue", "()I", &[])?; + let int_value = int_obj.i()?; + results.push(int_value); + } + Ok(results) + } + + fn get_strings(&mut self, obj: &JObject) -> Result> { + let list = self.get_list(obj)?; + let mut iter = list.iter(self)?; + let mut results = Vec::with_capacity(list.size(self)? as usize); + while let Some(elem) = iter.next(self)? { + let jstr = JString::from(elem); + let val = self.get_string(&jstr)?; + results.push(val.to_str()?.to_string()) + } + Ok(results) + } + + fn get_strings_array(&mut self, obj: jobjectArray) -> Result> { + let jobject_array = unsafe { JObjectArray::from_raw(obj) }; + let array_len = self.get_array_length(&jobject_array)?; + let mut res: Vec = Vec::new(); + for i in 0..array_len { + let item: JString = self.get_object_array_element(&jobject_array, i)?.into(); + res.push(self.get_string(&item)?.into()); + } + Ok(res) + } + + fn get_string_opt(&mut self, obj: &JObject) -> Result> { + self.get_optional(obj, |env, inner_obj| { + let java_obj_gen = env.call_method(inner_obj, "get", "()Ljava/lang/Object;", &[])?; + let java_string_obj = java_obj_gen.l()?; + let jstr = JString::from(java_string_obj); + let val = env.get_string(&jstr)?; + Ok(val.to_str()?.to_string()) + }) + } + + fn get_strings_opt(&mut self, obj: &JObject) -> Result>> { + self.get_optional(obj, |env, inner_obj| { + let java_obj_gen = env.call_method(inner_obj, "get", "()Ljava/lang/Object;", &[])?; + let java_list_obj = java_obj_gen.l()?; + env.get_strings(&java_list_obj) + }) + } + + fn get_int_opt(&mut self, obj: &JObject) -> Result> { + self.get_optional(obj, |env, inner_obj| { + let java_obj_gen = env.call_method(inner_obj, "get", "()Ljava/lang/Object;", &[])?; + let java_int_obj = java_obj_gen.l()?; + let int_obj = env.call_method(java_int_obj, "intValue", "()I", &[])?; + let int_value = int_obj.i()?; + Ok(int_value) + }) + } + + fn get_ints_opt(&mut self, obj: &JObject) -> Result>> { + self.get_optional(obj, |env, inner_obj| { + let java_obj_gen = env.call_method(inner_obj, "get", "()Ljava/lang/Object;", &[])?; + let java_list_obj = java_obj_gen.l()?; + env.get_integers(&java_list_obj) + }) + } + + fn get_long_opt(&mut self, obj: &JObject) -> Result> { + self.get_optional(obj, |env, inner_obj| { + let java_obj_gen = env.call_method(inner_obj, "get", "()Ljava/lang/Object;", &[])?; + let java_long_obj = java_obj_gen.l()?; + let long_obj = env.call_method(java_long_obj, "longValue", "()J", &[])?; + let long_value = long_obj.j()?; + Ok(long_value) + }) + } + + fn get_u64_opt(&mut self, obj: &JObject) -> Result> { + self.get_optional(obj, |env, inner_obj| { + let java_obj_gen = env.call_method(inner_obj, "get", "()Ljava/lang/Object;", &[])?; + let java_long_obj = java_obj_gen.l()?; + let long_obj = env.call_method(java_long_obj, "longValue", "()J", &[])?; + let long_value = long_obj.j()?; + Ok(long_value as u64) + }) + } + + fn get_bytes_opt(&mut self, obj: &JObject) -> Result> { + self.get_optional(obj, |env, inner_obj| { + let java_obj_gen = env.call_method(inner_obj, "get", "()Ljava/lang/Object;", &[])?; + let java_byte_buffer_obj = java_obj_gen.l()?; + let j_byte_buffer = JByteBuffer::from(java_byte_buffer_obj); + let raw_data = env.get_direct_buffer_address(&j_byte_buffer)?; + let capacity = env.get_direct_buffer_capacity(&j_byte_buffer)?; + let data = unsafe { slice::from_raw_parts(raw_data, capacity) }; + Ok(data) + }) + } + + fn get_optional(&mut self, obj: &JObject, f: F) -> Result> + where + F: FnOnce(&mut JNIEnv, &JObject) -> Result, + { + if obj.is_null() { + return Ok(None); + } + let is_empty = self.call_method(obj, "isEmpty", "()Z", &[])?; + if is_empty.z()? { + // TODO(lu): put get java object into here cuz can only get java Object + Ok(None) + } else { + f(self, obj).map(Some) + } + } +} + +#[no_mangle] +pub extern "system" fn Java_com_lancedb_lance_test_JniTestHelper_parseInts( + mut env: JNIEnv, + _obj: JObject, + list_obj: JObject, // List +) { + ok_or_throw_without_return!(env, env.get_integers(&list_obj)); +} + +#[no_mangle] +pub extern "system" fn Java_com_lancedb_lance_test_JniTestHelper_parseIntsOpt( + mut env: JNIEnv, + _obj: JObject, + list_obj: JObject, // Optional> +) { + ok_or_throw_without_return!(env, env.get_ints_opt(&list_obj)); +} diff --git a/java/core/lancedb-jni/src/lib.rs b/java/core/lancedb-jni/src/lib.rs new file mode 100644 index 00000000..490d9c97 --- /dev/null +++ b/java/core/lancedb-jni/src/lib.rs @@ -0,0 +1,68 @@ +// Copyright 2024 Lance Developers. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use lazy_static::lazy_static; + +// TODO import from lance-jni without duplicate +#[macro_export] +macro_rules! ok_or_throw { + ($env:expr, $result:expr) => { + match $result { + Ok(value) => value, + Err(err) => { + Error::from(err).throw(&mut $env); + return JObject::null(); + } + } + }; +} + +macro_rules! ok_or_throw_without_return { + ($env:expr, $result:expr) => { + match $result { + Ok(value) => value, + Err(err) => { + Error::from(err).throw(&mut $env); + return; + } + } + }; +} + +#[macro_export] +macro_rules! ok_or_throw_with_return { + ($env:expr, $result:expr, $ret:expr) => { + match $result { + Ok(value) => value, + Err(err) => { + Error::from(err).throw(&mut $env); + return $ret; + } + } + }; +} + +mod connection; +pub mod error; +mod ffi; +mod traits; + +pub use error::{Error, Result}; + +lazy_static! { + static ref RT: tokio::runtime::Runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("Failed to create tokio runtime"); +} diff --git a/java/core/lancedb-jni/src/traits.rs b/java/core/lancedb-jni/src/traits.rs new file mode 100644 index 00000000..01a7a490 --- /dev/null +++ b/java/core/lancedb-jni/src/traits.rs @@ -0,0 +1,122 @@ +// Copyright 2024 Lance Developers. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use jni::objects::{JMap, JObject, JString, JValue}; +use jni::JNIEnv; + +use crate::Result; + +pub trait FromJObject { + fn extract(&self) -> Result; +} + +/// Convert a Rust type into a Java Object. +pub trait IntoJava { + fn into_java<'a>(self, env: &mut JNIEnv<'a>) -> JObject<'a>; +} + +impl FromJObject for JObject<'_> { + fn extract(&self) -> Result { + Ok(JValue::from(self).i()?) + } +} + +impl FromJObject for JObject<'_> { + fn extract(&self) -> Result { + Ok(JValue::from(self).j()?) + } +} + +impl FromJObject for JObject<'_> { + fn extract(&self) -> Result { + Ok(JValue::from(self).f()?) + } +} + +impl FromJObject for JObject<'_> { + fn extract(&self) -> Result { + Ok(JValue::from(self).d()?) + } +} + +pub trait FromJString { + fn extract(&self, env: &mut JNIEnv) -> Result; +} + +impl FromJString for JString<'_> { + fn extract(&self, env: &mut JNIEnv) -> Result { + Ok(env.get_string(self)?.into()) + } +} + +pub trait JMapExt { + #[allow(dead_code)] + fn get_string(&self, env: &mut JNIEnv, key: &str) -> Result>; + + #[allow(dead_code)] + fn get_i32(&self, env: &mut JNIEnv, key: &str) -> Result>; + + #[allow(dead_code)] + fn get_i64(&self, env: &mut JNIEnv, key: &str) -> Result>; + + #[allow(dead_code)] + fn get_f32(&self, env: &mut JNIEnv, key: &str) -> Result>; + + #[allow(dead_code)] + fn get_f64(&self, env: &mut JNIEnv, key: &str) -> Result>; +} + +fn get_map_value(env: &mut JNIEnv, map: &JMap, key: &str) -> Result> +where + for<'a> JObject<'a>: FromJObject, +{ + let key_obj: JObject = env.new_string(key)?.into(); + if let Some(value) = map.get(env, &key_obj)? { + if value.is_null() { + Ok(None) + } else { + Ok(Some(value.extract()?)) + } + } else { + Ok(None) + } +} + +impl JMapExt for JMap<'_, '_, '_> { + fn get_string(&self, env: &mut JNIEnv, key: &str) -> Result> { + let key_obj: JObject = env.new_string(key)?.into(); + if let Some(value) = self.get(env, &key_obj)? { + let value_str: JString = value.into(); + Ok(Some(value_str.extract(env)?)) + } else { + Ok(None) + } + } + + fn get_i32(&self, env: &mut JNIEnv, key: &str) -> Result> { + get_map_value(env, self, key) + } + + fn get_i64(&self, env: &mut JNIEnv, key: &str) -> Result> { + get_map_value(env, self, key) + } + + fn get_f32(&self, env: &mut JNIEnv, key: &str) -> Result> { + get_map_value(env, self, key) + } + + fn get_f64(&self, env: &mut JNIEnv, key: &str) -> Result> { + get_map_value(env, self, key) + } +} diff --git a/java/core/pom.xml b/java/core/pom.xml new file mode 100644 index 00000000..a469c3ae --- /dev/null +++ b/java/core/pom.xml @@ -0,0 +1,94 @@ + + + + 4.0.0 + + + com.lancedb + lancedb-parent + 0.1-SNAPSHOT + ../pom.xml + + + lancedb-core + LanceDB Core + jar + + + + org.apache.arrow + arrow-vector + + + org.apache.arrow + arrow-memory-netty + + + org.apache.arrow + arrow-c-data + + + org.apache.arrow + arrow-dataset + + + org.json + json + + + org.questdb + jar-jni + + + org.junit.jupiter + junit-jupiter + test + + + + + + build-jni + + true + + + + + org.questdb + rust-maven-plugin + 1.1.1 + + + lancedb-jni + + build + + + lancedb-jni + + + ${project.build.directory}/classes/nativelib + true + + + + lancedb-jni-test + + test + + + lancedb-jni + false + -v + + + + + + + + + diff --git a/java/core/src/main/java/com/lancedb/lancedb/Connection.java b/java/core/src/main/java/com/lancedb/lancedb/Connection.java new file mode 100644 index 00000000..a667e724 --- /dev/null +++ b/java/core/src/main/java/com/lancedb/lancedb/Connection.java @@ -0,0 +1,120 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.lancedb.lancedb; + +import io.questdb.jar.jni.JarJniLoader; +import java.io.Closeable; +import java.util.List; +import java.util.Optional; + +/** + * Represents LanceDB database. + */ +public class Connection implements Closeable { + static { + JarJniLoader.loadLib(Connection.class, "/nativelib", "lancedb_jni"); + } + + private long nativeConnectionHandle; + + /** + * Connect to a LanceDB instance. + */ + public static native Connection connect(String uri); + + /** + * Get the names of all tables in the database. The names are sorted in + * ascending order. + * + * @return the table names + */ + public List tableNames() { + return tableNames(Optional.empty(), Optional.empty()); + } + + /** + * Get the names of filtered tables in the database. The names are sorted in + * ascending order. + * + * @param limit The number of results to return. + * @return the table names + */ + public List tableNames(int limit) { + return tableNames(Optional.empty(), Optional.of(limit)); + } + + /** + * Get the names of filtered tables in the database. The names are sorted in + * ascending order. + * + * @param startAfter If present, only return names that come lexicographically after the supplied + * value. This can be combined with limit to implement pagination + * by setting this to the last table name from the previous page. + * @return the table names + */ + public List tableNames(String startAfter) { + return tableNames(Optional.of(startAfter), Optional.empty()); + } + + /** + * Get the names of filtered tables in the database. The names are sorted in + * ascending order. + * + * @param startAfter If present, only return names that come lexicographically after the supplied + * value. This can be combined with limit to implement pagination + * by setting this to the last table name from the previous page. + * @param limit The number of results to return. + * @return the table names + */ + public List tableNames(String startAfter, int limit) { + return tableNames(Optional.of(startAfter), Optional.of(limit)); + } + + /** + * Get the names of filtered tables in the database. The names are sorted in + * ascending order. + * + * @param startAfter If present, only return names that come lexicographically after the supplied + * value. This can be combined with limit to implement pagination + * by setting this to the last table name from the previous page. + * @param limit The number of results to return. + * @return the table names + */ + public native List tableNames( + Optional startAfter, Optional limit); + + /** + * Closes this connection and releases any system resources associated with it. If + * the connection is + * already closed, then invoking this method has no effect. + */ + @Override + public void close() { + if (nativeConnectionHandle != 0) { + releaseNativeConnection(nativeConnectionHandle); + nativeConnectionHandle = 0; + } + } + + /** + * Native method to release the Lance connection resources associated with the + * given handle. + * + * @param handle The native handle to the connection resource. + */ + private native void releaseNativeConnection(long handle); + + private Connection() {} +} \ No newline at end of file diff --git a/java/core/src/test/java/com/lancedb/lancedb/ConnectionTest.java b/java/core/src/test/java/com/lancedb/lancedb/ConnectionTest.java new file mode 100644 index 00000000..65b3bc5a --- /dev/null +++ b/java/core/src/test/java/com/lancedb/lancedb/ConnectionTest.java @@ -0,0 +1,135 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.lancedb.lancedb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Path; +import java.util.List; +import java.net.URL; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +public class ConnectionTest { + private static final String[] TABLE_NAMES = { + "dataset_version", + "new_empty_dataset", + "test", + "write_stream" + }; + + @TempDir + static Path tempDir; // Temporary directory for the tests + private static URL lanceDbURL; + + @BeforeAll + static void setUp() { + ClassLoader classLoader = ConnectionTest.class.getClassLoader(); + lanceDbURL = classLoader.getResource("example_db"); + } + + @Test + void emptyDB() { + String databaseUri = tempDir.resolve("emptyDB").toString(); + try (Connection conn = Connection.connect(databaseUri)) { + List tableNames = conn.tableNames(); + assertTrue(tableNames.isEmpty()); + } + } + + @Test + void tableNames() { + try (Connection conn = Connection.connect(lanceDbURL.toString())) { + List tableNames = conn.tableNames(); + assertEquals(4, tableNames.size()); + for (int i = 0; i < TABLE_NAMES.length; i++) { + assertEquals(TABLE_NAMES[i], tableNames.get(i)); + } + } + } + + @Test + void tableNamesStartAfter() { + try (Connection conn = Connection.connect(lanceDbURL.toString())) { + assertTableNamesStartAfter(conn, TABLE_NAMES[0], 3, TABLE_NAMES[1], TABLE_NAMES[2], TABLE_NAMES[3]); + assertTableNamesStartAfter(conn, TABLE_NAMES[1], 2, TABLE_NAMES[2], TABLE_NAMES[3]); + assertTableNamesStartAfter(conn, TABLE_NAMES[2], 1, TABLE_NAMES[3]); + assertTableNamesStartAfter(conn, TABLE_NAMES[3], 0); + assertTableNamesStartAfter(conn, "a_dataset", 4, TABLE_NAMES[0], TABLE_NAMES[1], TABLE_NAMES[2], TABLE_NAMES[3]); + assertTableNamesStartAfter(conn, "o_dataset", 2, TABLE_NAMES[2], TABLE_NAMES[3]); + assertTableNamesStartAfter(conn, "v_dataset", 1, TABLE_NAMES[3]); + assertTableNamesStartAfter(conn, "z_dataset", 0); + } + } + + private void assertTableNamesStartAfter(Connection conn, String startAfter, int expectedSize, String... expectedNames) { + List tableNames = conn.tableNames(startAfter); + assertEquals(expectedSize, tableNames.size()); + for (int i = 0; i < expectedNames.length; i++) { + assertEquals(expectedNames[i], tableNames.get(i)); + } + } + + @Test + void tableNamesLimit() { + try (Connection conn = Connection.connect(lanceDbURL.toString())) { + for (int i = 0; i <= TABLE_NAMES.length; i++) { + List tableNames = conn.tableNames(i); + assertEquals(i, tableNames.size()); + for (int j = 0; j < i; j++) { + assertEquals(TABLE_NAMES[j], tableNames.get(j)); + } + } + } + } + + @Test + void tableNamesStartAfterLimit() { + try (Connection conn = Connection.connect(lanceDbURL.toString())) { + List tableNames = conn.tableNames(TABLE_NAMES[0], 2); + assertEquals(2, tableNames.size()); + assertEquals(TABLE_NAMES[1], tableNames.get(0)); + assertEquals(TABLE_NAMES[2], tableNames.get(1)); + tableNames = conn.tableNames(TABLE_NAMES[1], 1); + assertEquals(1, tableNames.size()); + assertEquals(TABLE_NAMES[2], tableNames.get(0)); + tableNames = conn.tableNames(TABLE_NAMES[2], 2); + assertEquals(1, tableNames.size()); + assertEquals(TABLE_NAMES[3], tableNames.get(0)); + tableNames = conn.tableNames(TABLE_NAMES[3], 2); + assertEquals(0, tableNames.size()); + tableNames = conn.tableNames(TABLE_NAMES[0], 0); + assertEquals(0, tableNames.size()); + + // Limit larger than the number of remaining tables + tableNames = conn.tableNames(TABLE_NAMES[0], 10); + assertEquals(3, tableNames.size()); + assertEquals(TABLE_NAMES[1], tableNames.get(0)); + assertEquals(TABLE_NAMES[2], tableNames.get(1)); + assertEquals(TABLE_NAMES[3], tableNames.get(2)); + + // Start after a value not in the list + tableNames = conn.tableNames("non_existent_table", 2); + assertEquals(2, tableNames.size()); + assertEquals(TABLE_NAMES[2], tableNames.get(0)); + assertEquals(TABLE_NAMES[3], tableNames.get(1)); + + // Start after the last table with a limit + tableNames = conn.tableNames(TABLE_NAMES[3], 1); + assertEquals(0, tableNames.size()); + } + } +} diff --git a/java/core/src/test/resources/example_db/dataset_version.lance/_latest.manifest b/java/core/src/test/resources/example_db/dataset_version.lance/_latest.manifest new file mode 100644 index 0000000000000000000000000000000000000000..f09f8e8be9cfc9aba64234f812dfa3f8dea0e6c9 GIT binary patch literal 273 zcmYk1Jx;_h5Jv3;FRO@RTi2&Y%jdU`*Tw4b<>{Sn8MYFXDiO+h81&GCj#7bh*1+hsM;m*OqwMB~e7$Pe UZ;3wIH>{_6u_;&i$M|wV{_xuB^r5E7< literal 0 HcmV?d00001 diff --git a/java/core/src/test/resources/example_db/dataset_version.lance/_transactions/2-3344b369-7471-4e23-8865-c949b6e19bc2.txn b/java/core/src/test/resources/example_db/dataset_version.lance/_transactions/2-3344b369-7471-4e23-8865-c949b6e19bc2.txn new file mode 100644 index 0000000000000000000000000000000000000000..c0119b6ea5c6371535ac17c6fdda93531af9fae8 GIT binary patch literal 99 zcmWN;%ME}a3;<9+i6I`Dz=J1~8la_?aVRM#<4{hj@43nEh(wB9;G!!6Swaxu46RU6 rwpg^KcafFd?8AE0h@C|%1TqE^&7jr-XwURh%RQWTeVX}c7`OQYLsS;8 literal 0 HcmV?d00001 diff --git a/java/core/src/test/resources/example_db/dataset_version.lance/_versions/1.manifest b/java/core/src/test/resources/example_db/dataset_version.lance/_versions/1.manifest new file mode 100644 index 0000000000000000000000000000000000000000..d94ff721e987921a75ccd51ff3afb8cadd99b97b GIT binary patch literal 159 zcmeBXU|`^q5@O0sQTPu7j9RRjc_qe128aaxM0 lX_A3nNkv|k02gacVqS8p5W9h%ArM&rT>}M-3_gy2&H!ijGB*GK literal 0 HcmV?d00001 diff --git a/java/core/src/test/resources/example_db/dataset_version.lance/_versions/2.manifest b/java/core/src/test/resources/example_db/dataset_version.lance/_versions/2.manifest new file mode 100644 index 0000000000000000000000000000000000000000..f8764e44c397e4e66eb1a1a811e91426adadc709 GIT binary patch literal 217 zcmX@gz`(#ICB&4OqVOLE7`0e4^Gb}33>YmKx#Wdd@)C1XB^Z(A*@{bwGV{`b3Ydh< zI9P=YxU|d+5-pQVEiHA^%q`P&O)L^Ebd%E35_OYIEKSYLjFQZZj1u*767!N%g_sx^ z6<8&htav!)Y<{3OgButlSd+92b&ZY9l8sYM&47%w6kQY36rd?dX-T@4rm0Cu mre?;;NvQ^UB^7yD0$i*R!`Kb<41vf3=np7hWbkqHa|Qropg^tw literal 0 HcmV?d00001 diff --git a/java/core/src/test/resources/example_db/dataset_version.lance/_versions/3.manifest b/java/core/src/test/resources/example_db/dataset_version.lance/_versions/3.manifest new file mode 100644 index 0000000000000000000000000000000000000000..f09f8e8be9cfc9aba64234f812dfa3f8dea0e6c9 GIT binary patch literal 273 zcmYk1Jx;_h5Jv3;FRO@RTi2&Y%jdU`*Tw4b<>{Sn8MYFXDiO+h81&GCj#7bh*1+hsM;m*OqwMB~e7$Pe UZ;3M@{hnYpwT^0KFFUSFa<(^{S9KyK;uKX zaB2>kJj{NWc`$on{6L_3E-4|V%oK(HFu6y>hH_kC5@KRtmB`V0$jBuo#Atx7zcjgo zOI(O6uQVqoJ~_WMuS5c3ubBzZRtX`_+|0cAvc#OyR0$?z_kd&)E1)vWs4|R(5-ixP zkwmeERe=Sp4Q!D#vPEn_86>Z9fh6H}v7<@~aWF74F|)9;u}fTGTnP(KVA}L?^m7IP Dk;8(} literal 0 HcmV?d00001 diff --git a/java/core/src/test/resources/example_db/dataset_version.lance/data/a13f68ba-04e6-48b5-bec0-bf54444be5f0.lance b/java/core/src/test/resources/example_db/dataset_version.lance/data/a13f68ba-04e6-48b5-bec0-bf54444be5f0.lance new file mode 100644 index 0000000000000000000000000000000000000000..c7d88b1cfa0a15f4dd0176443f705014af6a470f GIT binary patch literal 642 zcmZva%}T^D6oqrszcILwAR2$bE?mt*+EHY-gDyq%35L=sQt4!5CKVTM1Ruerk1#Ja z58yLs(&i3r@X~O4a&q$}rGyXydQd$cisfn{*7zQ9NfJJ?2IY z2Mc-ccH+}2a=)zq#s)`c)|>O2s#mOycX6k`L^BumpI k<0QS3io7fVF4mmHyyR3Nb^|>_AhH0u1_~G%d>s9p0dv|iE&u=k literal 0 HcmV?d00001 diff --git a/java/core/src/test/resources/example_db/new_empty_dataset.lance/_transactions/0-15648e72-076f-4ef1-8b90-10d305b95b3b.txn b/java/core/src/test/resources/example_db/new_empty_dataset.lance/_transactions/0-15648e72-076f-4ef1-8b90-10d305b95b3b.txn new file mode 100644 index 00000000..4ca22d68 --- /dev/null +++ b/java/core/src/test/resources/example_db/new_empty_dataset.lance/_transactions/0-15648e72-076f-4ef1-8b90-10d305b95b3b.txn @@ -0,0 +1 @@ +$15648e72-076f-4ef1-8b90-10d305b95b3b²=id ÿÿÿÿÿÿÿÿÿ*int3208name ÿÿÿÿÿÿÿÿÿ*string08 \ No newline at end of file diff --git a/java/core/src/test/resources/example_db/new_empty_dataset.lance/_versions/1.manifest b/java/core/src/test/resources/example_db/new_empty_dataset.lance/_versions/1.manifest new file mode 100644 index 0000000000000000000000000000000000000000..4f5495c6bda1795cd1822bd823bbced7cfccc024 GIT binary patch literal 159 zcmeBXU|`^q5@O0sQTPu7j9RRjc_qe128mOycX6k`L^BumpI k<0QS3io7fVF4mmHyyR3Nb^|>_AhH0u1_~G%d>s9p0dv|iE&u=k literal 0 HcmV?d00001 diff --git a/java/core/src/test/resources/example_db/test.lance/_latest.manifest b/java/core/src/test/resources/example_db/test.lance/_latest.manifest new file mode 100644 index 0000000000000000000000000000000000000000..d1d46b3e71227417409b0899bd6670aa0a717927 GIT binary patch literal 264 zcmY+9yGq1B7=<&Vo0VX(Ok+S;whl~~nM^j9PHe?jSmyGNjKoBeQ5N6I!pg?ZSFzg4 zT3t{$*XOR@8`;geRG?f;Eb3$`~*zm36c!w4kDtm8sWj;NbF3 z^#;g6gzyTdxII4lc-=k5i*MgTr>KX@N&DUr(pen6%TSO)1gu;^2Rc+) t?mUsr^8J)p6UFyj8MV^@L_^@d9JGuKxw6b}YewdutN*5@{Smg;_6LPs7gPWM literal 0 HcmV?d00001 diff --git a/java/core/src/test/resources/example_db/test.lance/_versions/1.manifest b/java/core/src/test/resources/example_db/test.lance/_versions/1.manifest new file mode 100644 index 0000000000000000000000000000000000000000..b0c30b9da3173d8d61f32da259f9033ffea25ea0 GIT binary patch literal 209 zcmdnXz`(#IEyR*plA5dV9|jn;*osSvGV{_cn7HJGSPP0WlT#%aQRGwdOOtX^Ef~2B zgxJbblS}f8B$$vDh^J*%q^86dXI7=g=VTU_Sf%CUCzeICMTxpnxvT}>6#>_B?E<$&2=r!6VuF1&CCofOpNtPD)O=fxL9)% Z^O94A*bVdyfyfBxekfpM@Nx8W1^|aVO1c05 literal 0 HcmV?d00001 diff --git a/java/core/src/test/resources/example_db/test.lance/_versions/2.manifest b/java/core/src/test/resources/example_db/test.lance/_versions/2.manifest new file mode 100644 index 0000000000000000000000000000000000000000..d1d46b3e71227417409b0899bd6670aa0a717927 GIT binary patch literal 264 zcmY+9yGq1B7=<&Vo0VX(Ok+S;whl~~nM^j9PHe?jSmyGNjKoBeQ5N6I!pg?ZSFzg4 zT3t{$*XOR@8`;geRG?f;Eb3$`~*zm36c!w4kDtm8sWj;NbF3 z^#;g6gzyTdxII4lc-=k5i;_ij=^7LoH@YIt}LR zo(7!F6~6w_p*L}LR>y`03n-fu11{SpYo&1&j$6uCKRLWH1WFxX za30w80!b9(z&ap?cez$*UDXi9X+rLR(ij1e2?oy|IwDPR^b#DmcgdsLVrS};z1TyR fht{7T)0D6Ex#ZcIzSw#%v!2cWAD6razqR=SQGGGg literal 0 HcmV?d00001 diff --git a/java/core/src/test/resources/example_db/write_stream.lance/_versions/1.manifest b/java/core/src/test/resources/example_db/write_stream.lance/_versions/1.manifest new file mode 100644 index 0000000000000000000000000000000000000000..ac2cd0f69b1b3461577f1073bf93dfe47938c3c0 GIT binary patch literal 214 zcmX@az`(#IE5wqQn47Bb9|jn;*osSvGV{_67%iB9(#(nJsS=FH@~oM8CB{ZTX+|Lv zAp6~6w_p*L}LR>y`03n-fu11{SpYo&1&jL>A|IJ&sI zxYREojt+t*?VH*=e7W$;b5Gv)$#Iu_BDz2%lB-Tc^OfVOerg|jYGLM52Nl^P4&O1^p}4Z%Z{{bR znCw!b5-HoAVWMJ&Lo@gMtcFQIivywB6E2g8IjQ>?i%-FFQx>DR%Dn%Af>e-S_Wq*w k)}wOZEI*DIZ + + 4.0.0 + + com.lancedb + lancedb-parent + 0.1-SNAPSHOT + pom + + Lance Parent + + + UTF-8 + 11 + 11 + 15.0.0 + + + + core + + + + + + org.apache.arrow + arrow-vector + ${arrow.version} + + + org.apache.arrow + arrow-memory-netty + ${arrow.version} + + + org.apache.arrow + arrow-c-data + ${arrow.version} + + + org.apache.arrow + arrow-dataset + ${arrow.version} + + + org.questdb + jar-jni + 1.1.1 + + + org.junit.jupiter + junit-jupiter + 5.10.1 + + + org.json + json + 20210307 + + + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + 3.3.1 + + google_checks.xml + true + true + warning + false + + + + validate + validate + + check + + + + + + + + + maven-clean-plugin + 3.1.0 + + + maven-resources-plugin + 3.0.2 + + + maven-compiler-plugin + 3.8.1 + + + -h + target/headers + + + + + maven-surefire-plugin + 3.2.5 + + --add-opens=java.base/java.nio=ALL-UNNAMED + + false + + + + maven-jar-plugin + 3.0.2 + + + maven-install-plugin + 2.5.2 + + + + +