fix: only use util function to build filesystem (#339)

python/lancedb/db.py

@@ -327,6 +327,6 @@ class LanceDBConnection(DBConnection):
         name: str
             The name of the table.
         """
-        filesystem, path = pa.fs.FileSystem.from_uri(self.uri)
+        filesystem, path = fs_from_uri(self.uri)
         table_path = os.path.join(path, name + ".lance")
         filesystem.delete_dir(table_path)

python/lancedb/table.py

@@ -23,12 +23,12 @@ import numpy as np
 import pandas as pd
 import pyarrow as pa
 import pyarrow.compute as pc
-import pyarrow.fs
 from lance import LanceDataset
 from lance.vector import vec_to_table
 
 from .common import DATA, VEC, VECTOR_COLUMN_NAME
 from .query import LanceFtsQueryBuilder, LanceQueryBuilder, Query
+from .util import fs_from_uri
 
 
 def _sanitize_data(data, schema, on_bad_vectors, fill_value):
@@ -527,7 +527,7 @@ class LanceTable(Table):
     @classmethod
     def open(cls, db, name):
         tbl = cls(db, name)
-        fs, path = pa.fs.FileSystem.from_uri(tbl._dataset_uri)
+        fs, path = fs_from_uri(tbl._dataset_uri)
         file_info = fs.get_file_info(path)
         if file_info.type != pa.fs.FileType.Directory:
             raise FileNotFoundError(

python/lancedb/util.py

@@ -71,7 +71,8 @@ def fs_from_uri(uri: str) -> Tuple[pa_fs.FileSystem, str]:
     Get a PyArrow FileSystem from a URI, handling extra environment variables.
     """
     if get_uri_scheme(uri) == "s3":
-        if os.environ["AWS_ENDPOINT"]:
-            uri += "?endpoint_override=" + os.environ["AWS_ENDPOINT"]
+        fs = pa_fs.S3FileSystem(endpoint_override=os.environ.get("AWS_ENDPOINT"))
+        path = get_uri_location(uri)
+        return fs, path
 
     return pa_fs.FileSystem.from_uri(uri)