mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-13 15:22:30 +00:00
docs: correct table and namespace claims the runtime audit contradicted
- Python's table_exists was said to work on namespace connections; on
one from connect_namespace("dir" or "rest") it fails with an
AttributeError. The page now shows the supported checks: open the
table and handle the error each connection raises, or read every page
of a listing.
- Dropping a missing table was said to raise by default. A local
database raises; a deployment may report success, depending on its
catalog, and its Python connection takes no ignore_missing.
- A deployment's catalog may refuse cloning altogether; where cloning
works, it is shallow.
- A dropped column was called unrecoverable. Restoring a version from
before the drop brings it back until cleanup prunes that version.
- The namespace page stored data in a `data/` directory that does not
exist, presented a recommended character set as enforced, and its
Python listing comments showed lists where the calls return response
objects. Its namespace examples now run as tests.
- The storage tip offered the table's reported options as the effective
ones; they omit what the table inherits from its connection.
This commit is contained in:
Brendan Clement
@@ -116,14 +116,26 @@ def namespace_table_ops_example():
|
||||
mode="create", # use "overwrite" only if you want to replace existing table
|
||||
)
|
||||
|
||||
# Verify
|
||||
print(db.list_namespaces()) # ['prod']
|
||||
print(db.list_namespaces(namespace_path=["prod"])) # ['recommendations', 'search']
|
||||
print(db.list_tables(namespace_path=["prod", "search"])) # ['user']
|
||||
print(db.list_tables(namespace_path=["prod", "recommendations"])) # ['user']
|
||||
# Verify: listings return a response with the names and a page token
|
||||
print(db.list_namespaces().namespaces) # ['prod']
|
||||
print(db.list_namespaces(namespace_path=["prod"]).namespaces)
|
||||
# ['recommendations', 'search']
|
||||
print(db.list_tables(namespace_path=["prod", "search"]).tables) # ['user']
|
||||
print(db.list_tables(namespace_path=["prod", "recommendations"]).tables) # ['user']
|
||||
# --8<-- [end:namespace_table_ops]
|
||||
|
||||
|
||||
def test_namespace_table_ops(tmp_path, monkeypatch, capsys):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
namespace_table_ops_example()
|
||||
assert capsys.readouterr().out.splitlines() == [
|
||||
"['prod']",
|
||||
"['recommendations', 'search']",
|
||||
"['user']",
|
||||
"['user']",
|
||||
]
|
||||
|
||||
|
||||
def namespace_admin_ops_example():
|
||||
# --8<-- [start:namespace_admin_ops]
|
||||
import lancedb
|
||||
@@ -135,8 +147,8 @@ def namespace_admin_ops_example():
|
||||
db.create_namespace(["prod", "search"])
|
||||
|
||||
child_namespaces = db.list_namespaces(namespace_path=["prod"]).namespaces
|
||||
print(f"Child namespaces under {namespace}: {child_namespaces}")
|
||||
# Child namespaces under ['prod', 'search']: ['search']
|
||||
print(f"Child namespaces under ['prod']: {child_namespaces}")
|
||||
# Child namespaces under ['prod']: ['search']
|
||||
|
||||
metadata = db.describe_namespace(["prod", "search"])
|
||||
print(f"Metadata for namespace {namespace}: {metadata}")
|
||||
@@ -145,7 +157,17 @@ def namespace_admin_ops_example():
|
||||
db.drop_namespace(["prod", "search"], mode="skip")
|
||||
db.drop_namespace(["prod"], mode="skip")
|
||||
# --8<-- [end:namespace_admin_ops]
|
||||
return child_namespaces, metadata
|
||||
return db, child_namespaces, metadata
|
||||
|
||||
|
||||
def test_namespace_admin_ops(tmp_path, monkeypatch, capsys):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
db, child_namespaces, metadata = namespace_admin_ops_example()
|
||||
assert child_namespaces == ["search"]
|
||||
assert capsys.readouterr().out.splitlines()[0] == (
|
||||
"Child namespaces under ['prod']: ['search']"
|
||||
)
|
||||
assert db.list_namespaces().namespaces == []
|
||||
|
||||
async def connect_object_storage_config_async():
|
||||
# --8<-- [start:connect_object_storage_async]
|
||||
|
||||
@@ -189,9 +189,9 @@ Table-level `storage_options` inherit every key from the connection and override
|
||||
</CodeGroup>
|
||||
|
||||
<Tip>
|
||||
**Inspect the effective options**
|
||||
**Inspect a table's own options**
|
||||
|
||||
On `AsyncTable`, `await table.initial_storage_options()` returns the options the table was opened with, and `await table.latest_storage_options()` returns the current options after any provider-driven refresh. The deprecated `table.storage_options()` method will be removed in a future release.
|
||||
On `AsyncTable`, `await table.initial_storage_options()` returns the options passed when the table was opened, and `await table.latest_storage_options()` returns them after any provider-driven refresh. Neither includes what the table inherits from its connection's `storage_options`, which still apply. The deprecated `table.storage_options()` method will be removed in a future release.
|
||||
</Tip>
|
||||
|
||||
#### General object store options {#general-object-store-options}
|
||||
|
||||
@@ -352,5 +352,8 @@ Use the `drop_table()` method on the database to remove a table.
|
||||
</CodeGroup>
|
||||
|
||||
This permanently removes the table and is not recoverable, unlike deleting rows.
|
||||
By default, if the table does not exist an exception is raised. To suppress this
|
||||
in Python on a local database, you can pass in `ignore_missing=True`.
|
||||
On a local database, dropping a table that does not exist raises an error; in
|
||||
Python, pass `ignore_missing=True` to suppress it. A deployment may report
|
||||
success instead, depending on how it stores its catalog, so do not rely on the
|
||||
error to tell you a table was missing. Python's deployment connection does not
|
||||
take `ignore_missing`.
|
||||
|
||||
@@ -53,14 +53,15 @@ If you want to go deeper, see the Lance format [namespace documentation](https:/
|
||||
A namespace path is a list of components. For example, `["prod", "search"]` means the `search`
|
||||
namespace inside the `prod` namespace. The empty path, `[]`, means the root namespace.
|
||||
|
||||
Each component is a name, not a filesystem path segment. Namespace names can't be empty, and each
|
||||
component can contain only letters, numbers, underscores, hyphens, and periods. That keeps the same
|
||||
identifier usable across local directory namespaces and REST namespace identifiers.
|
||||
Each component is a name, not a filesystem path segment, and can't be empty. Keep each one to
|
||||
letters, numbers, underscores, hyphens, and periods, so the same identifier is usable everywhere: a
|
||||
local directory namespace accepts other characters too, but other namespace implementations and
|
||||
deployments can refuse them.
|
||||
|
||||
## Directory namespaces {#directory-namespaces}
|
||||
|
||||
The simplest namespace model in LanceDB is a single root namespace, often represented by one
|
||||
directory:
|
||||
directory. Namespaces can nest inside it, as in this hierarchy:
|
||||
|
||||
```bash
|
||||
./local_lancedb (root)
|
||||
@@ -70,7 +71,12 @@ directory:
|
||||
└── data (table)
|
||||
```
|
||||
|
||||
As a user of LanceDB OSS, you might never notice namespaces at first, because LanceDB exposes the single-level hierarchy shown above, with the data stored in the `data/` directory, where the root namespace is implicit. Connecting to this namespace is as simple as connecting to the catalog root:
|
||||
The hierarchy is logical, not a tree of folders: a directory namespace records it in a
|
||||
`__manifest` table in the root directory, and stores each table in a directory of its own there.
|
||||
|
||||
As a user of LanceDB OSS, you might never notice namespaces at first, because a plain connection
|
||||
uses only the root namespace, which is implicit. Connecting to this namespace is as simple as
|
||||
connecting to the catalog root:
|
||||
|
||||
<CodeGroup>
|
||||
```python Python icon="python"
|
||||
@@ -95,7 +101,7 @@ let db = connect("./local_lancedb").execute().await?;
|
||||
```
|
||||
</CodeGroup>
|
||||
|
||||
This creates the default namespace directory (`data/`) under the specified root path.
|
||||
A table created through this connection is stored directly under that path, as `<name>.lance`.
|
||||
|
||||
You can also explicitly connect to a namespace using `lancedb.connect_namespace(...)` with the directory namespace implementation:
|
||||
|
||||
@@ -286,8 +292,8 @@ The SDK methods expose the namespace path in the idiom of each language:
|
||||
|
||||
<Note>
|
||||
Using namespaces is **optional** in LanceDB, and most basic use cases do not require to work with them.
|
||||
An empty namespace (`[]`), which is the default, means "root namespace", and the data will be stored in
|
||||
the `data/` directory under the specified root path.
|
||||
An empty namespace (`[]`), which is the default, means the root namespace, whose tables are stored
|
||||
directly under the specified root path.
|
||||
</Note>
|
||||
|
||||
## Namespace management APIs {#namespace-management-apis}
|
||||
@@ -322,12 +328,13 @@ doesn't exist, or contains data:
|
||||
- Drop mode: `fail` reports an error when the namespace doesn't exist, and `skip` treats a missing namespace as a successful no-op.
|
||||
- Drop behavior: `restrict` keeps non-empty namespaces from being dropped, and `cascade` drops child namespaces and tables first.
|
||||
|
||||
Namespace path components can't be empty. Each component can contain only letters, numbers,
|
||||
underscores, hyphens, and periods.
|
||||
Namespace path components can't be empty, and are best kept to letters, numbers, underscores,
|
||||
hyphens, and periods, as [above](#namespace-paths-and-names).
|
||||
|
||||
Listing APIs return the immediate children of the requested namespace path. Use `limit` with the
|
||||
returned `page_token` to page through large catalogs; pass an empty namespace path (`[]`) when you
|
||||
want to list from the root namespace.
|
||||
want to list from the root namespace. In Python, a listing returns a response: the names are in its
|
||||
`namespaces` or `tables`, and `page_token` is set when there is another page.
|
||||
|
||||
## Namespaces in LanceDB Enterprise {#namespaces-in-lancedb-enterprise}
|
||||
|
||||
|
||||
@@ -185,10 +185,16 @@ await db.tableNames();
|
||||
</CodeGroup>
|
||||
|
||||
Each accepts `namespace_path` to act inside a [namespace](/management/namespace)
|
||||
rather than at the database root. Python's `table_exists` works only on a
|
||||
namespace connection, where it takes the table's full path as a list; local
|
||||
directory and deployment connections raise `NotImplementedError`. TypeScript
|
||||
has no `tableExists`.
|
||||
rather than at the database root.
|
||||
|
||||
To find out whether a table exists, open it and handle the error, or look for
|
||||
its name in a listing. A missing table raises `ValueError` on a local directory
|
||||
or a deployment, and `lance_namespace.TableNotFoundError` on a connection from
|
||||
`connect_namespace`. A listing returns one page of names at a time, so check
|
||||
every page. Python's `table_exists` is no substitute: a local directory or a
|
||||
deployment raises `NotImplementedError` for it, and on a connection from
|
||||
`connect_namespace("dir", ...)` or `connect_namespace("rest", ...)` it fails with
|
||||
an `AttributeError`. TypeScript has no `tableExists`.
|
||||
|
||||
## Renaming and cloning {#rename-clone}
|
||||
|
||||
@@ -204,8 +210,8 @@ db.clone_table(
|
||||
```
|
||||
|
||||
Renaming is not supported on a local directory connection. On a deployment,
|
||||
whether a table can be renamed depends on how the deployment stores its
|
||||
catalog.
|
||||
whether a table can be renamed or cloned at all depends on how the deployment
|
||||
stores its catalog.
|
||||
|
||||
A **shallow** clone -- the default -- references the source's data files rather
|
||||
than copying them, so it is fast and cheap regardless of table size. It is the
|
||||
@@ -218,7 +224,8 @@ is [compacted](/maintenance/compaction) past the retention window, the version
|
||||
the clone points at can be pruned. Local OSS connections support shallow clones
|
||||
only; `is_shallow=False` is not implemented. If the copy must outlive the
|
||||
source's history, copy its data into an independent table instead. A deployment
|
||||
also makes shallow clones only.
|
||||
that supports cloning also makes shallow clones only, and one whose catalog
|
||||
does not support it refuses every clone.
|
||||
</Warning>
|
||||
|
||||
<Note>
|
||||
@@ -896,8 +903,13 @@ Remove several columns at once for efficiency:
|
||||
</CodeBlock>
|
||||
</CodeGroup>
|
||||
|
||||
<Warning title="Irreversible Column Deletion">
|
||||
Dropping columns cannot be undone. Make sure you have backups or are certain before removing columns.
|
||||
<Warning title="Dropping a column is undone only by restoring a version">
|
||||
Dropping a column changes the latest version only. Until
|
||||
[cleanup](/maintenance/cleanup) prunes the versions from before the drop,
|
||||
[restoring](#versioning) one of them brings the column and its values back --
|
||||
along with the rest of the table as it was then. Once those versions are
|
||||
pruned, the drop cannot be undone, so check before dropping a column you may
|
||||
need.
|
||||
</Warning>
|
||||
|
||||
## Versioning {#versioning}
|
||||
|
||||
@@ -10,9 +10,9 @@ export const PyConnectObjectStorage = "import lancedb\n\nuri = \"s3://your-bucke
|
||||
|
||||
export const PyConnectObjectStorageAsync = "import lancedb\n\nuri = \"s3://your-bucket/path\"\n# You can also use \"gs://your-bucket/path\" or \"az://your-container/path\".\nasync_db = await lancedb.connect_async(uri)\n";
|
||||
|
||||
export const PyNamespaceAdminOps = "import lancedb\n\ndb = lancedb.connect_namespace(\"dir\", {\"root\": \"./local_lancedb\"})\nnamespace = [\"prod\", \"search\"]\n\ndb.create_namespace([\"prod\"])\ndb.create_namespace([\"prod\", \"search\"])\n\nchild_namespaces = db.list_namespaces(namespace_path=[\"prod\"]).namespaces\nprint(f\"Child namespaces under {namespace}: {child_namespaces}\")\n# Child namespaces under ['prod', 'search']: ['search']\n\nmetadata = db.describe_namespace([\"prod\", \"search\"])\nprint(f\"Metadata for namespace {namespace}: {metadata}\")\n# Metadata for namespace ['prod', 'search']: properties=None\n\ndb.drop_namespace([\"prod\", \"search\"], mode=\"skip\")\ndb.drop_namespace([\"prod\"], mode=\"skip\")\n";
|
||||
export const PyNamespaceAdminOps = "import lancedb\n\ndb = lancedb.connect_namespace(\"dir\", {\"root\": \"./local_lancedb\"})\nnamespace = [\"prod\", \"search\"]\n\ndb.create_namespace([\"prod\"])\ndb.create_namespace([\"prod\", \"search\"])\n\nchild_namespaces = db.list_namespaces(namespace_path=[\"prod\"]).namespaces\nprint(f\"Child namespaces under ['prod']: {child_namespaces}\")\n# Child namespaces under ['prod']: ['search']\n\nmetadata = db.describe_namespace([\"prod\", \"search\"])\nprint(f\"Metadata for namespace {namespace}: {metadata}\")\n# Metadata for namespace ['prod', 'search']: properties=None\n\ndb.drop_namespace([\"prod\", \"search\"], mode=\"skip\")\ndb.drop_namespace([\"prod\"], mode=\"skip\")\n";
|
||||
|
||||
export const PyNamespaceTableOps = "import lancedb\n\ndb = lancedb.connect_namespace(\"dir\", {\"root\": \"./local_lancedb\"})\n\n# Create namespace tree: prod/search\ndb.create_namespace([\"prod\"], mode=\"exist_ok\")\ndb.create_namespace([\"prod\", \"search\"], mode=\"exist_ok\")\ndb.create_namespace([\"prod\", \"recommendations\"], mode=\"exist_ok\")\n\ndb.create_table(\n \"user\",\n data=[{\"id\": 1, \"vector\": [0.1, 0.2], \"name\": \"alice\"}],\n namespace_path=[\"prod\", \"search\"],\n mode=\"create\", # use \"overwrite\" only if you want to replace existing table\n)\n\ndb.create_table(\n \"user\",\n data=[{\"id\": 2, \"vector\": [0.3, 0.4], \"name\": \"bob\"}],\n namespace_path=[\"prod\", \"recommendations\"],\n mode=\"create\", # use \"overwrite\" only if you want to replace existing table\n)\n\n# Verify\nprint(db.list_namespaces()) # ['prod']\nprint(db.list_namespaces(namespace_path=[\"prod\"])) # ['recommendations', 'search']\nprint(db.list_tables(namespace_path=[\"prod\", \"search\"])) # ['user']\nprint(db.list_tables(namespace_path=[\"prod\", \"recommendations\"])) # ['user']\n";
|
||||
export const PyNamespaceTableOps = "import lancedb\n\ndb = lancedb.connect_namespace(\"dir\", {\"root\": \"./local_lancedb\"})\n\n# Create namespace tree: prod/search\ndb.create_namespace([\"prod\"], mode=\"exist_ok\")\ndb.create_namespace([\"prod\", \"search\"], mode=\"exist_ok\")\ndb.create_namespace([\"prod\", \"recommendations\"], mode=\"exist_ok\")\n\ndb.create_table(\n \"user\",\n data=[{\"id\": 1, \"vector\": [0.1, 0.2], \"name\": \"alice\"}],\n namespace_path=[\"prod\", \"search\"],\n mode=\"create\", # use \"overwrite\" only if you want to replace existing table\n)\n\ndb.create_table(\n \"user\",\n data=[{\"id\": 2, \"vector\": [0.3, 0.4], \"name\": \"bob\"}],\n namespace_path=[\"prod\", \"recommendations\"],\n mode=\"create\", # use \"overwrite\" only if you want to replace existing table\n)\n\n# Verify: listings return a response with the names and a page token\nprint(db.list_namespaces().namespaces) # ['prod']\nprint(db.list_namespaces(namespace_path=[\"prod\"]).namespaces)\n# ['recommendations', 'search']\nprint(db.list_tables(namespace_path=[\"prod\", \"search\"]).tables) # ['user']\nprint(db.list_tables(namespace_path=[\"prod\", \"recommendations\"]).tables) # ['user']\n";
|
||||
|
||||
export const TsConnect = "import * as lancedb from \"@lancedb/lancedb\";\n\nasync function connectExample(uri: string) {\n const db = await lancedb.connect(uri);\n return db;\n}\n";
|
||||
|
||||
|
||||
Reference in New Issue
Block a user