rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-01-13 15:22:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: pass AWS_ENDPOINT environment variable down (#330)

Browse Source 
Tested locally against minio.
This commit is contained in:
Will Jones
2023-07-18 15:07:26 -07:00
committed by GitHub
parent 141b6647a8
commit fb97b03a51
2 changed files with 18 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
python/lancedb/util.py
View File

@@ -11,8 +11,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from typing import Tuple
from urllib.parse import urlparse
import pyarrow as pa
import pyarrow.fs as pa_fs
def get_uri_scheme(uri: str) -> str:
"""
@@ -59,3 +64,14 @@ def get_uri_location(uri: str) -> str:
return parsed.path
else:
return parsed.netloc + parsed.path
def fs_from_uri(uri: str) -> Tuple[pa_fs.FileSystem, str]:
"""
Get a PyArrow FileSystem from a URI, handling extra environment variables.
"""
if get_uri_scheme(uri) == "s3":
if os.environ["AWS_ENDPOINT"]:
uri += "?endpoint_override=" + os.environ["AWS_ENDPOINT"]
return pa_fs.FileSystem.from_uri(uri)