diff --git a/python/lancedb/db.py b/python/lancedb/db.py index 4af40d8a..d8063442 100644 --- a/python/lancedb/db.py +++ b/python/lancedb/db.py @@ -327,6 +327,6 @@ class LanceDBConnection(DBConnection): name: str The name of the table. """ - filesystem, path = pa.fs.FileSystem.from_uri(self.uri) + filesystem, path = fs_from_uri(self.uri) table_path = os.path.join(path, name + ".lance") filesystem.delete_dir(table_path) diff --git a/python/lancedb/table.py b/python/lancedb/table.py index 2f8dd5d8..bb6e2854 100644 --- a/python/lancedb/table.py +++ b/python/lancedb/table.py @@ -23,12 +23,12 @@ import numpy as np import pandas as pd import pyarrow as pa import pyarrow.compute as pc -import pyarrow.fs from lance import LanceDataset from lance.vector import vec_to_table from .common import DATA, VEC, VECTOR_COLUMN_NAME from .query import LanceFtsQueryBuilder, LanceQueryBuilder, Query +from .util import fs_from_uri def _sanitize_data(data, schema, on_bad_vectors, fill_value): @@ -527,7 +527,7 @@ class LanceTable(Table): @classmethod def open(cls, db, name): tbl = cls(db, name) - fs, path = pa.fs.FileSystem.from_uri(tbl._dataset_uri) + fs, path = fs_from_uri(tbl._dataset_uri) file_info = fs.get_file_info(path) if file_info.type != pa.fs.FileType.Directory: raise FileNotFoundError( diff --git a/python/lancedb/util.py b/python/lancedb/util.py index 88f85d27..1d844aef 100644 --- a/python/lancedb/util.py +++ b/python/lancedb/util.py @@ -71,7 +71,8 @@ def fs_from_uri(uri: str) -> Tuple[pa_fs.FileSystem, str]: Get a PyArrow FileSystem from a URI, handling extra environment variables. """ if get_uri_scheme(uri) == "s3": - if os.environ["AWS_ENDPOINT"]: - uri += "?endpoint_override=" + os.environ["AWS_ENDPOINT"] + fs = pa_fs.S3FileSystem(endpoint_override=os.environ.get("AWS_ENDPOINT")) + path = get_uri_location(uri) + return fs, path return pa_fs.FileSystem.from_uri(uri)