feat: add flag to enable faster manifest paths (#1612)

The new V2 manifest path scheme makes discovering the latest version of
a table constant time on object stores, regardless of the number of
versions in the table. See benchmarks in the PR here:
https://github.com/lancedb/lance/pull/2798

Closes #1583

nodejs/lancedb/connection.ts

@@ -52,6 +52,15 @@ export interface CreateTableOptions {
    */
   dataStorageVersion?: string;
 
+  /**
+   * Use the new V2 manifest paths. These paths provide more efficient
+   * opening of datasets with many versions on object stores.  WARNING:
+   * turning this on will make the dataset unreadable for older versions
+   * of LanceDB (prior to 0.10.0). To migrate an existing dataset, instead
+   * use the {@link LocalTable#migrateManifestPathsV2} method.
+   */
+  enableV2ManifestPaths?: boolean;
+
   /**
    * If true then data files will be written with the legacy format
    *
@@ -270,6 +279,7 @@ export class LocalConnection extends Connection {
       mode,
       cleanseStorageOptions(options?.storageOptions),
       dataStorageVersion,
+      options?.enableV2ManifestPaths,
     );
 
     return new LocalTable(innerTable);
@@ -308,6 +318,7 @@ export class LocalConnection extends Connection {
       mode,
       cleanseStorageOptions(options?.storageOptions),
       dataStorageVersion,
+      options?.enableV2ManifestPaths,
     );
     return new LocalTable(innerTable);
   }

nodejs/lancedb/table.ts

@@ -697,4 +697,31 @@ export class LocalTable extends Table {
     on = Array.isArray(on) ? on : [on];
     return new MergeInsertBuilder(this.inner.mergeInsert(on));
   }
+
+  /**
+   * Check if the table uses the new manifest path scheme.
+   *
+   * This function will return true if the table uses the V2 manifest
+   * path scheme.
+   */
+  async usesV2ManifestPaths(): Promise<boolean> {
+    return await this.inner.usesV2ManifestPaths();
+  }
+
+  /**
+   * Migrate the table to use the new manifest path scheme.
+   *
+   * This function will rename all V1 manifests to V2 manifest paths.
+   * These paths provide more efficient opening of datasets with many versions
+   * on object stores.
+   *
+   * This function is idempotent, and can be run multiple times without
+   * changing the state of the object store.
+   *
+   * However, it should not be run while other concurrent operations are happening.
+   * And it should also run until completion before resuming other operations.
+   */
+  async migrateManifestPathsV2(): Promise<void> {
+    await this.inner.migrateManifestPathsV2();
+  }
 }