ensure table names are uri encoded for tables (#1189)

This prevents an issue where users can do something like:
```js
db.createTable('my-table#123123')
```
The server has logic to determine that '#' character is not allowed in
the table name, but currently this is being returned as 404 error
because it routes to `/v1/my-table#123123/create` and `#123123/create`
will not be parsed as part of path

.github/workflows/build_linux_wheel/action.yml

@@ -14,6 +14,10 @@ inputs:
     # Note: this does *not* mean the host is arm64, since we might be cross-compiling.
     required: false
     default: "false"
+  manylinux:
+    description: "The manylinux version to build for"
+    required: false
+    default: "2_17"
 runs:
   using: "composite"
   steps:
@@ -28,7 +32,7 @@ runs:
         command: build
         working-directory: python
         target: x86_64-unknown-linux-gnu
-        manylinux: "2_17"
+        manylinux: ${{ inputs.manylinux }}
         args: ${{ inputs.args }}
         before-script-linux: |
           set -e
@@ -43,7 +47,7 @@ runs:
         command: build
         working-directory: python
         target: aarch64-unknown-linux-gnu
-        manylinux: "2_24"
+        manylinux: ${{ inputs.manylinux }}
         args: ${{ inputs.args }}
         before-script-linux: |
           set -e

.github/workflows/pypi-publish.yml

@@ -6,13 +6,23 @@ on:
 
 jobs:
   linux:
+    name: Python ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }}
     timeout-minutes: 60
     strategy:
       matrix:
         python-minor-version: ["8"]
-        platform:
+        config:
-          - x86_64
+          - platform: x86_64
-          - aarch64
+            manylinux: "2_17"
+            extra_args: ""
+          - platform: x86_64
+            manylinux: "2_28"
+            extra_args: "--features fp16kernels"
+          - platform: aarch64
+            manylinux: "2_24"
+            extra_args: ""
+          # We don't build fp16 kernels for aarch64, because it uses 
+          # cross compilation image, which doesn't have a new enough compiler.
     runs-on: "ubuntu-22.04"
     steps:
       - uses: actions/checkout@v4
@@ -26,8 +36,9 @@ jobs:
       - uses: ./.github/workflows/build_linux_wheel
         with:
           python-minor-version: ${{ matrix.python-minor-version }}
-          args: "--release --strip"
+          args: "--release --strip ${{ matrix.config.extra_args }}"
-          arm-build: ${{ matrix.platform == 'aarch64' }}
+          arm-build: ${{ matrix.config.platform == 'aarch64' }}
+          manylinux: ${{ matrix.config.manylinux }}
       - uses: ./.github/workflows/upload_wheel
         with:
           token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
@@ -58,7 +69,7 @@ jobs:
       - uses: ./.github/workflows/build_mac_wheel
         with:
           python-minor-version: ${{ matrix.python-minor-version }}
-          args: "--release --strip --target ${{ matrix.config.target }}"
+          args: "--release --strip --target ${{ matrix.config.target }} --features fp16kernels"
       - uses: ./.github/workflows/upload_wheel
         with:
           python-minor-version: ${{ matrix.python-minor-version }}

.github/workflows/rust.yml

@@ -31,6 +31,10 @@ jobs:
       run:
         shell: bash
         working-directory: rust
+    env:
+      # Need up-to-date compilers for kernels
+      CC: gcc-12
+      CXX: g++-12
     steps:
     - uses: actions/checkout@v4
       with:
@@ -54,6 +58,10 @@ jobs:
       run:
         shell: bash
         working-directory: rust
+    env:
+      # Need up-to-date compilers for kernels
+      CC: gcc-12
+      CXX: g++-12
     steps:
     - uses: actions/checkout@v4
       with:

Cargo.toml

@@ -14,10 +14,10 @@ keywords = ["lancedb", "lance", "database", "vector", "search"]
 categories = ["database-implementations"]
 
 [workspace.dependencies]
-lance = { "version" = "=0.10.6", "features" = ["dynamodb"] }
+lance = { "version" = "=0.10.8", "features" = ["dynamodb"] }
-lance-index = { "version" = "=0.10.6" }
+lance-index = { "version" = "=0.10.8" }
-lance-linalg = { "version" = "=0.10.6" }
+lance-linalg = { "version" = "=0.10.8" }
-lance-testing = { "version" = "=0.10.6" }
+lance-testing = { "version" = "=0.10.8" }
 # Note that this one does not include pyarrow
 arrow = { version = "50.0", optional = false }
 arrow-array = "50.0"

docs/src/sql.md

@@ -66,6 +66,7 @@ Currently, Lance supports a growing list of SQL expressions.
 - `LIKE`, `NOT LIKE`
 - `CAST`
 - `regexp_match(column, pattern)`
+- [DataFusion Functions](https://arrow.apache.org/datafusion/user-guide/sql/scalar_functions.html)
 
 For example, the following filter string is acceptable:
 

node/src/query.ts

@@ -38,7 +38,7 @@ export class Query<T = number[]> {
   constructor (query?: T, tbl?: any, embeddings?: EmbeddingFunction<T>) {
     this._tbl = tbl
     this._query = query
-    this._limit = undefined
+    this._limit = 10
     this._nprobes = 20
     this._refineFactor = undefined
     this._select = undefined

node/src/remote/client.ts

@@ -103,6 +103,18 @@ function toLanceRes (res: AxiosResponse): RemoteResponse {
   }
 }
 
+async function decodeErrorData(
+  res: RemoteResponse,
+  responseType?: ResponseType
+): Promise<string> {
+  const errorData = await res.body()
+  if (responseType === 'arraybuffer') {
+      return new TextDecoder().decode(errorData)
+  } else {
+      return errorData
+  }
+}
+
 export class HttpLancedbClient {
   private readonly _url: string
   private readonly _apiKey: () => string
@@ -180,7 +192,7 @@ export class HttpLancedbClient {
     }
 
     if (response.status !== 200) {
-      const errorData = new TextDecoder().decode(await response.body())
+      const errorData = await decodeErrorData(response)
       throw new Error(
         `Server Error, status: ${response.status}, ` +
         `message: ${response.statusText}: ${errorData}`
@@ -226,7 +238,7 @@ export class HttpLancedbClient {
     }
 
     if (response.status !== 200) {
-      const errorData = new TextDecoder().decode(await response.body())
+      const errorData = await decodeErrorData(response, responseType)
       throw new Error(
         `Server Error, status: ${response.status}, ` +
         `message: ${response.statusText}: ${errorData}`

node/src/remote/index.ts

@@ -156,7 +156,7 @@ export class RemoteConnection implements Connection {
     }
 
     const res = await this._client.post(
-      `/v1/table/${tableName}/create/`,
+      `/v1/table/${encodeURIComponent(tableName)}/create/`,
       buffer,
       undefined,
       'application/vnd.apache.arrow.stream'
@@ -177,7 +177,7 @@ export class RemoteConnection implements Connection {
   }
 
   async dropTable (name: string): Promise<void> {
-    await this._client.post(`/v1/table/${name}/drop/`)
+    await this._client.post(`/v1/table/${encodeURIComponent(name)}/drop/`)
   }
 
   withMiddleware (middleware: HttpMiddleware): Connection {
@@ -268,7 +268,7 @@ export class RemoteTable<T = number[]> implements Table<T> {
 
   get schema (): Promise<any> {
     return this._client
-      .post(`/v1/table/${this._name}/describe/`)
+      .post(`/v1/table/${encodeURIComponent(this._name)}/describe/`)
       .then(async (res) => {
         if (res.status !== 200) {
           throw new Error(
@@ -282,7 +282,7 @@ export class RemoteTable<T = number[]> implements Table<T> {
   }
 
   search (query: T): Query<T> {
-    return new RemoteQuery(query, this._client, this._name) //, this._embeddings_new)
+    return new RemoteQuery(query, this._client, encodeURIComponent(this._name)) //, this._embeddings_new)
   }
 
   filter (where: string): Query<T> {
@@ -324,7 +324,7 @@ export class RemoteTable<T = number[]> implements Table<T> {
 
     const buffer = await fromTableToStreamBuffer(tbl, this._embeddings)
     const res = await this._client.post(
-      `/v1/table/${this._name}/merge_insert/`,
+      `/v1/table/${encodeURIComponent(this._name)}/merge_insert/`,
       buffer,
       queryParams,
       'application/vnd.apache.arrow.stream'
@@ -348,7 +348,7 @@ export class RemoteTable<T = number[]> implements Table<T> {
 
     const buffer = await fromTableToStreamBuffer(tbl, this._embeddings)
     const res = await this._client.post(
-      `/v1/table/${this._name}/insert/`,
+      `/v1/table/${encodeURIComponent(this._name)}/insert/`,
       buffer,
       {
         mode: 'append'
@@ -374,7 +374,7 @@ export class RemoteTable<T = number[]> implements Table<T> {
     }
     const buffer = await fromTableToStreamBuffer(tbl, this._embeddings)
     const res = await this._client.post(
-      `/v1/table/${this._name}/insert/`,
+      `/v1/table/${encodeURIComponent(this._name)}/insert/`,
       buffer,
       {
         mode: 'overwrite'
@@ -421,7 +421,7 @@ export class RemoteTable<T = number[]> implements Table<T> {
       index_cache_size: indexCacheSize
     }
     const res = await this._client.post(
-      `/v1/table/${this._name}/create_index/`,
+      `/v1/table/${encodeURIComponent(this._name)}/create_index/`,
       data
     )
     if (res.status !== 200) {
@@ -442,7 +442,7 @@ export class RemoteTable<T = number[]> implements Table<T> {
       replace: true
     }
     const res = await this._client.post(
-      `/v1/table/${this._name}/create_scalar_index/`,
+      `/v1/table/${encodeURIComponent(this._name)}/create_scalar_index/`,
       data
     )
     if (res.status !== 200) {
@@ -455,14 +455,14 @@ export class RemoteTable<T = number[]> implements Table<T> {
   }
 
   async countRows (filter?: string): Promise<number> {
-    const result = await this._client.post(`/v1/table/${this._name}/count_rows/`, {
+    const result = await this._client.post(`/v1/table/${encodeURIComponent(this._name)}/count_rows/`, {
       predicate: filter
     })
-    return (await result.body())?.stats?.num_rows
+    return (await result.body())
   }
 
   async delete (filter: string): Promise<void> {
-    await this._client.post(`/v1/table/${this._name}/delete/`, {
+    await this._client.post(`/v1/table/${encodeURIComponent(this._name)}/delete/`, {
       predicate: filter
     })
   }
@@ -481,7 +481,7 @@ export class RemoteTable<T = number[]> implements Table<T> {
         updates[key] = toSQL(value)
       }
     }
-    await this._client.post(`/v1/table/${this._name}/update/`, {
+    await this._client.post(`/v1/table/${encodeURIComponent(this._name)}/update/`, {
       predicate: filter,
       updates: Object.entries(updates).map(([key, value]) => [key, value])
     })
@@ -489,7 +489,7 @@ export class RemoteTable<T = number[]> implements Table<T> {
 
   async listIndices (): Promise<VectorIndex[]> {
     const results = await this._client.post(
-      `/v1/table/${this._name}/index/list/`
+      `/v1/table/${encodeURIComponent(this._name)}/index/list/`
     )
     return (await results.body()).indexes?.map((index: any) => ({
       columns: index.columns,
@@ -500,7 +500,7 @@ export class RemoteTable<T = number[]> implements Table<T> {
 
   async indexStats (indexUuid: string): Promise<IndexStats> {
     const results = await this._client.post(
-      `/v1/table/${this._name}/index/${indexUuid}/stats/`
+      `/v1/table/${encodeURIComponent(this._name)}/index/${indexUuid}/stats/`
     )
     const body = await results.body()
     return {

python/Cargo.toml

@@ -31,3 +31,6 @@ pyo3-build-config = { version = "0.20.3", features = [
     "extension-module",
     "abi3-py38",
 ] }
+
+[features]
+fp16kernels = ["lancedb/fp16kernels"]

python/pyproject.toml

@@ -3,7 +3,7 @@ name = "lancedb"
 version = "0.6.6"
 dependencies = [
     "deprecation",
-    "pylance==0.10.6",
+    "pylance==0.10.8",
     "ratelimiter~=1.0",
     "retry>=0.9.2",
     "tqdm>=4.27.0",
@@ -41,6 +41,7 @@ classifiers = [
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "Topic :: Scientific/Engineering",
 ]
 

python/python/lancedb/remote/table.py

@@ -500,7 +500,10 @@ class RemoteTable(Table):
 
     def count_rows(self, filter: Optional[str] = None) -> int:
         payload = {"predicate": filter}
-        self._conn._client.post(f"/v1/table/{self._name}/count_rows/", data=payload)
+        resp = self._conn._client.post(
+            f"/v1/table/{self._name}/count_rows/", data=payload
+        )
+        return resp
 
     def add_columns(self, transforms: Dict[str, str]):
         raise NotImplementedError(

rust/lancedb/Cargo.toml

@@ -50,3 +50,4 @@ walkdir = "2"
 [features]
 default = ["remote"]
 remote = ["dep:reqwest"]
+fp16kernels = ["lance-linalg/fp16kernels"]

rust/lancedb/src/table.rs

@@ -1304,14 +1304,7 @@ impl TableInternal for NativeTable {
     }
 
     async fn count_rows(&self, filter: Option<String>) -> Result<usize> {
-        let dataset = self.dataset.get().await?;
+        Ok(self.dataset.get().await?.count_rows(filter).await?)
-        if let Some(filter) = filter {
-            let mut scanner = dataset.scan();
-            scanner.filter(&filter)?;
-            Ok(scanner.count_rows().await? as usize)
-        } else {
-            Ok(dataset.count_rows().await?)
-        }
     }
 
     async fn add(