feat: support 4bit PQ (#1916)

nodejs/__test__/table.test.ts

@@ -567,6 +567,15 @@ describe("When creating an index", () => {
     // TODO: Verify parameters when we can load index config as part of list indices
   });
 
+  it("should be able to create 4bit IVF_PQ", async () => {
+    await tbl.createIndex("vec", {
+      config: Index.ivfPq({
+        numPartitions: 10,
+        numBits: 4,
+      }),
+    });
+  });
+
   it("should allow me to replace (or not) an existing index", async () => {
     await tbl.createIndex("id");
     // Default is replace=true

nodejs/lancedb/indices.ts

@@ -47,6 +47,16 @@ export interface IvfPqOptions {
    */
   numSubVectors?: number;
 
+  /**
+   * Number of bits per sub-vector.
+   *
+   * This value controls how much each subvector is compressed.  The more bits the more
+   * accurate the index will be but the slower search.  The default is 8 bits.
+   *
+   * The number of bits must be 4 or 8.
+   */
+  numBits?: number;
+
   /**
    * Distance type to use to build the index.
    *

nodejs/src/index.rs

@@ -45,6 +45,7 @@ impl Index {
         distance_type: Option<String>,
         num_partitions: Option<u32>,
         num_sub_vectors: Option<u32>,
+        num_bits: Option<u32>,
         max_iterations: Option<u32>,
         sample_rate: Option<u32>,
     ) -> napi::Result<Self> {
@@ -59,6 +60,9 @@ impl Index {
         if let Some(num_sub_vectors) = num_sub_vectors {
             ivf_pq_builder = ivf_pq_builder.num_sub_vectors(num_sub_vectors);
         }
+        if let Some(num_bits) = num_bits {
+            ivf_pq_builder = ivf_pq_builder.num_bits(num_bits);
+        }
         if let Some(max_iterations) = max_iterations {
             ivf_pq_builder = ivf_pq_builder.max_iterations(max_iterations);
         }