mirror of
https://github.com/lancedb/lancedb.git
synced 2025-12-25 22:29:58 +00:00
39cc2fd62b
This PR refactors how we handle read consistency: does the `LanceTable` class always pick up modifications to the table made by other instance or processes. Users have three options they can set at the connection level: 1. (Default) `read_consistency_interval=None` means it will not check at all. Users can call `table.checkout_latest()` to manually check for updates. 2. `read_consistency_interval=timedelta(0)` means **always** check for updates, giving strong read consistency. 3. `read_consistency_interval=timedelta(seconds=20)` means check for updates every 20 seconds. This is eventual consistency, a compromise between the two options above. There is now an explicit difference between a `LanceTable` that tracks the current version and one that is fixed at a historical version. We now enforce that users cannot write if they have checked out an old version. They are instructed to call `checkout_latest()` before calling the write methods. Since `conn.open_table()` doesn't have a parameter for version, users will only get fixed references if they call `table.checkout()`. The difference between these two can be seen in the repr: Table that are fixed at a particular version will have a `version` displayed in the repr. Otherwise, the version will not be shown. ```python >>> table LanceTable(connection=..., name="my_table") >>> table.checkout(1) >>> table LanceTable(connection=..., name="my_table", version=1) ``` I decided to not create different classes for these states, because I think we already have enough complexity with the Cloud vs OSS table references. Based on #812
69 lines
2.0 KiB
TOML
69 lines
2.0 KiB
TOML
[project]
|
|
name = "lancedb"
|
|
version = "0.5.3"
|
|
dependencies = [
|
|
"deprecation",
|
|
"pylance==0.9.12",
|
|
"ratelimiter~=1.0",
|
|
"retry>=0.9.2",
|
|
"tqdm>=4.27.0",
|
|
"pydantic>=1.10",
|
|
"attrs>=21.3.0",
|
|
"semver>=3.0",
|
|
"cachetools",
|
|
"overrides>=0.7"
|
|
]
|
|
description = "lancedb"
|
|
authors = [{ name = "LanceDB Devs", email = "dev@lancedb.com" }]
|
|
license = { file = "LICENSE" }
|
|
readme = "README.md"
|
|
requires-python = ">=3.8"
|
|
keywords = [
|
|
"data-format",
|
|
"data-science",
|
|
"machine-learning",
|
|
"arrow",
|
|
"data-analytics"
|
|
]
|
|
classifiers = [
|
|
"Development Status :: 3 - Alpha",
|
|
"Environment :: Console",
|
|
"Intended Audience :: Science/Research",
|
|
"License :: OSI Approved :: Apache Software License",
|
|
"Operating System :: OS Independent",
|
|
"Programming Language :: Python",
|
|
"Programming Language :: Python :: 3",
|
|
"Programming Language :: Python :: 3 :: Only",
|
|
"Programming Language :: Python :: 3.8",
|
|
"Programming Language :: Python :: 3.9",
|
|
"Programming Language :: Python :: 3.10",
|
|
"Programming Language :: Python :: 3.11",
|
|
"Topic :: Scientific/Engineering",
|
|
]
|
|
|
|
[project.urls]
|
|
repository = "https://github.com/lancedb/lancedb"
|
|
|
|
[project.optional-dependencies]
|
|
tests = ["aiohttp", "pandas>=1.4", "pytest", "pytest-mock", "pytest-asyncio", "duckdb", "pytz", "polars>=0.19"]
|
|
dev = ["ruff", "pre-commit"]
|
|
docs = ["mkdocs", "mkdocs-jupyter", "mkdocs-material", "mkdocstrings[python]"]
|
|
clip = ["torch", "pillow", "open-clip"]
|
|
embeddings = ["openai>=1.6.1", "sentence-transformers", "torch", "pillow", "open-clip-torch", "cohere", "huggingface_hub",
|
|
"InstructorEmbedding", "google.generativeai", "boto3>=1.28.57", "awscli>=1.29.57", "botocore>=1.31.57"]
|
|
|
|
[build-system]
|
|
requires = ["setuptools", "wheel"]
|
|
build-backend = "setuptools.build_meta"
|
|
|
|
[tool.ruff]
|
|
select = ["F", "E", "W", "I", "G", "TCH", "PERF"]
|
|
|
|
[tool.pytest.ini_options]
|
|
addopts = "--strict-markers --ignore-glob=lancedb/embeddings/*.py"
|
|
|
|
markers = [
|
|
"slow: marks tests as slow (deselect with '-m \"not slow\"')",
|
|
"asyncio"
|
|
]
|