diff --git a/docs/web-tests/py/test_curation_sampling.py b/docs/web-tests/py/test_curation_sampling.py new file mode 100644 index 000000000..d50b8a43f --- /dev/null +++ b/docs/web-tests/py/test_curation_sampling.py @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + + +def test_sample_dataset(): + # --8<-- [start:sample_dataset] + from tempfile import TemporaryDirectory + + import lancedb + from lancedb.permutation import Permutation, permutation_builder + + with TemporaryDirectory(prefix="lancedb-sampling-") as uri: + db = lancedb.connect(uri) + table = db.create_table("records", [{"id": i} for i in range(10)]) + splits = ( + permutation_builder(table).split_random(counts=[3, 7], seed=42).execute() + ) + sample = Permutation.from_tables(table, splits, split=0).with_format("arrow") + selected_ids = [] + for batch in sample.iter(batch_size=3): + selected_ids.extend(batch.column("id").to_pylist()) + print(f"Selected {len(selected_ids)} rows: {selected_ids}") + # --8<-- [end:sample_dataset] + + assert len(selected_ids) == 3 + assert len(set(selected_ids)) == 3 + assert all(0 <= value < 10 for value in selected_ids) diff --git a/docs/web-tests/py/test_function.py b/docs/web-tests/py/test_function.py index 629ab89bc..f66a50aec 100644 --- a/docs/web-tests/py/test_function.py +++ b/docs/web-tests/py/test_function.py @@ -4,16 +4,18 @@ """Examples for docs/web/management/function.mdx. Registering a Function and filling its column need a LanceDB Enterprise -deployment, so `test_function_enterprise` runs only when the connection -variables the Enterprise examples already use are set: LANCEDB_URI, -LANCEDB_API_KEY and LANCEDB_HOST_OVERRIDE, plus LANCEDB_REGION if the -deployment needs one. The walkthrough calls its table `function_demo` and its -Function `double`; the test creates both under names unique to its run, never -replaces an existing object, and drops only what it created. +deployment, so `test_function_enterprise` runs only when +LANCEDB_DOCS_TEST_DEPLOYMENT=1 asks for one, with the connection variables the +Enterprise examples already use: LANCEDB_URI, LANCEDB_API_KEY and +LANCEDB_HOST_OVERRIDE, plus LANCEDB_REGION if the deployment needs one. The +walkthrough calls its table `function_demo` and its Function `double`; the test +creates both under names unique to its run, never replaces an existing object, +and drops only what it created. The other tests run locally and check the client side of the same code: what -`@udf` sends for registration, how a registered version binds a column, and -where a local connection stops. They do not run a Function remotely. +`@udf` sends for registration, how a registered version binds a column, where a +local connection stops, and when the Enterprise test connects. They do not run +a Function remotely. """ import json @@ -70,12 +72,17 @@ def registered_version(definition) -> FunctionVersion: return FunctionVersion.from_json(json.dumps(reply)) -def require_env(var_name: str) -> str: - """Skip the test unless the required environment variable is provided.""" - value = os.environ.get(var_name) - if not value: - pytest.skip(f"Set {var_name} to run this Enterprise example") - return value +DEPLOYMENT = ("LANCEDB_URI", "LANCEDB_HOST_OVERRIDE", "LANCEDB_API_KEY") +# The connection settings alone never send this test to a deployment. +OPT_IN = "LANCEDB_DOCS_TEST_DEPLOYMENT" + + +def require_deployment(): + if os.environ.get(OPT_IN) != "1": + pytest.skip(f"set {OPT_IN}=1 and {', '.join(DEPLOYMENT)} to use a deployment") + missing = [name for name in DEPLOYMENT if not os.environ.get(name)] + if missing: + pytest.fail(f"{OPT_IN}=1 needs {', '.join(missing)}") def test_function_runs_locally_as_python(): @@ -160,9 +167,7 @@ def test_function_version_binds_a_column(tmp_path): def test_function_enterprise(monkeypatch): - require_env("LANCEDB_URI") - require_env("LANCEDB_API_KEY") - require_env("LANCEDB_HOST_OVERRIDE") + require_deployment() # --8<-- [start:function_connect] import os @@ -240,3 +245,91 @@ def test_function_enterprise(monkeypatch): db.drop_table(table_name) if version is not None: db.drop_function(version.name, version=version.version) + + +class ReachedConnection(Exception): + pass + + +SETTINGS = { + "LANCEDB_URI": "db://docs-test", + "LANCEDB_HOST_OVERRIDE": "https://deployment.invalid", + "LANCEDB_API_KEY": "docs-test-key", +} + + +@pytest.fixture +def connect_attempts(monkeypatch): + """`lancedb.connect`, replaced so that it records each call and stops.""" + attempts = [] + + def connect(**kwargs): + attempts.append(kwargs) + raise ReachedConnection + + monkeypatch.setattr(lancedb, "connect", connect) + for name in (OPT_IN, *DEPLOYMENT, "LANCEDB_REGION"): + monkeypatch.delenv(name, raising=False) + return attempts + + +def enterprise_outcome(monkeypatch): + try: + test_function_enterprise(monkeypatch) + except (pytest.skip.Exception, pytest.fail.Exception, ReachedConnection) as outcome: + return outcome + + +@pytest.mark.parametrize( + "opt_in", + [None, "", "0", "true", "TRUE", "yes", "on", " 1", "1 ", "01", "1.0", "11"], +) +def test_function_enterprise_needs_the_exact_opt_in( + monkeypatch, connect_attempts, opt_in +): + for name, value in SETTINGS.items(): + monkeypatch.setenv(name, value) + if opt_in is not None: + monkeypatch.setenv(OPT_IN, opt_in) + outcome = enterprise_outcome(monkeypatch) + assert isinstance(outcome, pytest.skip.Exception), outcome + assert str(outcome).startswith(f"set {OPT_IN}=1 ") + assert connect_attempts == [] + + +@pytest.mark.parametrize("blank", ["unset", "empty"]) +@pytest.mark.parametrize( + "missing", [*([name] for name in DEPLOYMENT), list(DEPLOYMENT)] +) +def test_function_enterprise_fails_when_asked_without_settings( + monkeypatch, connect_attempts, missing, blank +): + monkeypatch.setenv(OPT_IN, "1") + for name, value in SETTINGS.items(): + if name not in missing: + monkeypatch.setenv(name, value) + elif blank == "empty": + monkeypatch.setenv(name, "") + outcome = enterprise_outcome(monkeypatch) + assert isinstance(outcome, pytest.fail.Exception), outcome + assert str(outcome) == f"{OPT_IN}=1 needs {', '.join(missing)}" + assert connect_attempts == [] + + +@pytest.mark.parametrize("region", [None, "eu-west-1"]) +def test_function_enterprise_connects_when_asked(monkeypatch, connect_attempts, region): + monkeypatch.setenv(OPT_IN, "1") + for name, value in SETTINGS.items(): + monkeypatch.setenv(name, value) + if region: + monkeypatch.setenv("LANCEDB_REGION", region) + outcome = enterprise_outcome(monkeypatch) + assert isinstance(outcome, ReachedConnection), outcome + assert connect_attempts == [ + { + "uri": "db://docs-test", + "api_key": "docs-test-key", + "region": region or "us-east-1", + "host_override": "https://deployment.invalid", + } + ] diff --git a/docs/web-tests/py/test_materialized_view.py b/docs/web-tests/py/test_materialized_view.py index ec48e1033..414d6dc1e 100644 --- a/docs/web-tests/py/test_materialized_view.py +++ b/docs/web-tests/py/test_materialized_view.py @@ -12,7 +12,9 @@ def test_materialized_view_create_and_refresh(tmp_path, monkeypatch): db = lancedb.connect("./.lancedb") table = db.create_table( - "people", [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}] + "people", + [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}], + storage_options={"new_table_enable_stable_row_ids": "true"}, ) view = db.create_materialized_view( @@ -29,12 +31,11 @@ def test_materialized_view_create_and_refresh(tmp_path, monkeypatch): # --8<-- [start:materialized_view_refresh] table.add([{"name": "bea", "age": 41}]) - result = view.refresh() - print(result.rows_written) + view.refresh() + print(view.table.count_rows()) # 2 # --8<-- [end:materialized_view_refresh] - assert result.mode == "rebuild" - assert result.rows_written == 2 + assert view.table.count_rows() == 2 assert sorted(row["shout"] for row in view.table.to_arrow().to_pylist()) == [ "ADA", "BEA", @@ -45,7 +46,11 @@ def test_materialized_view_with_no_data(tmp_path): import lancedb db = lancedb.connect(tmp_path) - db.create_table("people", [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}]) + db.create_table( + "people", + [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}], + storage_options={"new_table_enable_stable_row_ids": "true"}, + ) view = db.create_materialized_view( "adults", "people", where="age >= 18", with_no_data=True ) @@ -58,11 +63,16 @@ def test_materialized_view_refresh_to_source_version(tmp_path): import lancedb db = lancedb.connect(tmp_path) - table = db.create_table("people", [{"name": "ada", "age": 36}]) + table = db.create_table( + "people", + [{"name": "ada", "age": 36}], + storage_options={"new_table_enable_stable_row_ids": "true"}, + ) view = db.create_materialized_view("adults", "people", where="age >= 18") first = table.version table.add([{"name": "bea", "age": 41}]) - assert view.refresh().rows_written == 2 + view.refresh() + assert view.table.count_rows() == 2 result = view.refresh(source_version=first) assert (result.source_version, result.rows_written) == (first, 1) diff --git a/docs/web-tests/py/test_multimodal.py b/docs/web-tests/py/test_multimodal.py index bf7ec1bc8..cb4a647c3 100644 --- a/docs/web-tests/py/test_multimodal.py +++ b/docs/web-tests/py/test_multimodal.py @@ -215,28 +215,18 @@ async def test_query_to_pandas_kwargs(db_path_factory): .to_pandas(blob_mode="bytes") ) - # Vector / FTS / hybrid queries return "bytes" or "descriptions". - # Lazy BlobFile handles need a plain scan. + # Vector / FTS / hybrid queries can't materialize blob columns, + # so omit them from the projection df_vec = ( tbl.search(query_vector) .limit(10) - .select(["id", "video"]) - .to_pandas(blob_mode="bytes", split_blocks=True, self_destruct=True) + .select(["id", "vector"]) + .to_pandas(split_blocks=True, self_destruct=True) ) - - # to_arrow and to_list take the same blob_mode - hits = tbl.search(query_vector).limit(2).select(["id", "video"]) - arrow_bytes = hits.to_arrow(blob_mode="bytes") - rows = hits.to_list(blob_mode="bytes") # --8<-- [end:query_to_pandas_kwargs] assert len(df_lazy) == 1 assert len(df_bytes) == 1 assert df_bytes["video"].iloc[0] == b"fake_video_bytes_1" assert len(df_vec) == 10 - assert all( - video == f"fake_video_bytes_{i}".encode() - for i, video in zip(df_vec["id"], df_vec["video"]) - ) - assert arrow_bytes["video"].to_pylist() == [row["video"] for row in rows] - assert all(isinstance(row["video"], bytes) for row in rows) + assert "video" not in df_vec.columns diff --git a/docs/web-tests/py/test_tables.py b/docs/web-tests/py/test_tables.py index b3a456adc..d17344aec 100644 --- a/docs/web-tests/py/test_tables.py +++ b/docs/web-tests/py/test_tables.py @@ -835,31 +835,31 @@ def test_update_optimize_cleanup_snippet(tmp_db): # ============================================================================ -def _setup_schema_add_table(tmp_db, data=None): +def _setup_schema_add_table(db): # --8<-- [start:schema_add_setup] - table_name = "schema_evolution_add_example" - if data is None: - data = [ - { - "id": 1, - "name": "Laptop", - "price": 1200.00, - "vector": np.random.random(128).tolist(), - }, - { - "id": 2, - "name": "Smartphone", - "price": 800.00, - "vector": np.random.random(128).tolist(), - }, - { - "id": 3, - "name": "Headphones", - "price": 150.00, - "vector": np.random.random(128).tolist(), - }, - ] - table = tmp_db.create_table(table_name, data, mode="overwrite") + import numpy as np + + data = [ + { + "id": 1, + "name": "Laptop", + "price": 1200.00, + "vector": np.random.random(128).tolist(), + }, + { + "id": 2, + "name": "Smartphone", + "price": 800.00, + "vector": np.random.random(128).tolist(), + }, + { + "id": 3, + "name": "Headphones", + "price": 150.00, + "vector": np.random.random(128).tolist(), + }, + ] + table = db.create_table("schema_evolution_add_example", data, mode="overwrite") # --8<-- [end:schema_add_setup] return table @@ -953,17 +953,7 @@ def test_add_columns_default_values(tmp_db): def test_add_columns_nullable(tmp_db): - table = _setup_schema_add_table( - tmp_db, - data=[ - { - "id": 1, - "name": "Laptop", - "price": 1200.00, - "vector": np.random.random(128).tolist(), - } - ], - ) + table = _setup_schema_add_table(tmp_db) # --8<-- [start:add_columns_nullable] # Add a nullable timestamp column diff --git a/docs/web-tests/rs/embedding.rs b/docs/web-tests/rs/embedding.rs index 33bdec997..67f6bbc80 100644 --- a/docs/web-tests/rs/embedding.rs +++ b/docs/web-tests/rs/embedding.rs @@ -17,7 +17,7 @@ use lancedb::{ #[tokio::main] async fn main() -> Result<()> { let db = connect("./mydb").execute().await?; - let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set"); + let api_key = "your-openai-api-key".to_string(); let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model( api_key, "text-embedding-3-large", @@ -54,7 +54,7 @@ use std::sync::Arc; use lancedb::embeddings::openai::OpenAIEmbeddingFunction; -let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set"); +let api_key = "your-openai-api-key".to_string(); let embedding = Arc::new( OpenAIEmbeddingFunction::new_with_model(api_key, "text-embedding-3-small") .expect("failed to create OpenAI embedding function"), @@ -77,7 +77,7 @@ use lancedb::{ #[tokio::main] async fn main() -> Result<()> { let db = connect("./mydb").execute().await?; - let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY is not set"); + let api_key = "your-openai-api-key".to_string(); let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model( api_key, "text-embedding-3-large", diff --git a/docs/web/.mintignore b/docs/web/.mintignore new file mode 100644 index 000000000..b8da6c17a --- /dev/null +++ b/docs/web/.mintignore @@ -0,0 +1,6 @@ +# Functions are documented before their release. At release, remove these +# entries, add the pages back to the navigation in docs.json, restore the +# /feature-engineering/functions redirect to /management/function, and publish +# the drafts marked "Draft for the Functions release". +management/function.mdx +sql/ddl/function.mdx diff --git a/docs/web/api-reference/rest/index.mdx b/docs/web/api-reference/rest/index.mdx index 805f73cda..b2dd10016 100644 --- a/docs/web/api-reference/rest/index.mdx +++ b/docs/web/api-reference/rest/index.mdx @@ -35,10 +35,13 @@ To authenticate to the Enterprise REST API, you need the endpoint for your deplo ```bash export LANCEDB_API_KEY="" -export LANCEDB_URI="https://your-enterprise-endpoint.com" +export LANCEDB_HOST_OVERRIDE="https://your-enterprise-endpoint.com" export LANCEDB_DATABASE="your-database-name" ``` +`LANCEDB_HOST_OVERRIDE` is the endpoint the SDKs take as `host_override`; the +database name is the part of its `db://` URI after `db://`. + 3. If your Enterprise deployment is private, connect through the private network endpoint provided for your deployment. For example, Azure Private Link deployments commonly use a private IP or an internal DNS name as the endpoint. ### Verify authentication {#verify-authentication} @@ -46,30 +49,37 @@ export LANCEDB_DATABASE="your-database-name" 4. Check that you can reach the deployment and list tables: ```bash -curl -X GET "$LANCEDB_URI/v1/management/table" \ - -H "Content-Type: application/json" \ +curl -X GET "$LANCEDB_HOST_OVERRIDE/v1/table/" \ -H "x-api-key: $LANCEDB_API_KEY" \ -H "x-lancedb-database: $LANCEDB_DATABASE" ``` +The response lists one page of table names, with a token for the next page when +there are more. + If your deployment endpoint already includes the database host name, you can omit the `x-lancedb-database` header. -5. Create a table to confirm write access. Let's call it `words`. +5. Create a table to confirm write access. Let's call it `words`. A table is +created from an Arrow IPC stream, which this writes with PyArrow: ```bash -curl -X POST "$LANCEDB_URI/v1/tables/words" \ +python -c "import pyarrow as pa, sys; t = pa.table({'word': ['hello', 'world']}); w = pa.ipc.new_stream(sys.stdout.buffer, t.schema); w.write_table(t); w.close()" > words.arrow + +curl -X POST "$LANCEDB_HOST_OVERRIDE/v1/table/words/create" \ -H "Content-Type: application/vnd.apache.arrow.stream" \ -H "x-api-key: $LANCEDB_API_KEY" \ - -H "x-lancedb-database: $LANCEDB_DATABASE" + -H "x-lancedb-database: $LANCEDB_DATABASE" \ + --data-binary @words.arrow ``` 6. Check that the table has been created: ```bash -curl -X GET "$LANCEDB_URI/v1/management/table" \ +curl -X POST "$LANCEDB_HOST_OVERRIDE/v1/table/words/describe" \ -H "Content-Type: application/json" \ -H "x-api-key: $LANCEDB_API_KEY" \ - -H "x-lancedb-database: $LANCEDB_DATABASE" + -H "x-lancedb-database: $LANCEDB_DATABASE" \ + -d '{}' ``` That's it -- you're connected! Now, you can start adding data and querying it. diff --git a/docs/web/curation/aggregate.mdx b/docs/web/curation/aggregate.mdx index 91c32fdd4..b6d02d7fb 100644 --- a/docs/web/curation/aggregate.mdx +++ b/docs/web/curation/aggregate.mdx @@ -1,113 +1,144 @@ --- title: "Profiling a dataset" sidebarTitle: "Aggregate" -description: "Counting, grouping and summarizing — finding out what your data actually contains." +description: "Measure coverage, missing values and class balance before selecting a dataset." icon: chart-column keywords: ["aggregate", "group by", "class balance", "count", "profiling", "distinct"] +availability: + oss: unavailable + enterprise: available + summary: >- + Enterprise supports SQL aggregation. --- -Enterprise Aggregation is SQL, and -[SQL needs a deployment](/basics/sql#sql-is-remote). +Use SQL aggregates to summarize a dataset: count records, find missing values, +and compare groups. These summaries help you understand the data before +selecting records for search, evaluation or training. -Before curating anything, find out what you have. Most of the surprises in a -training set — a class that is 90% of the rows, a column that is half null, a -"unique" id that is not — are one `GROUP BY` away from being visible. +## Example data {#example-data} -## How big, and how complete {#size} +Connect with an [Enterprise SQL client](/basics/sql). The examples use a small +document table and a separate label table. Expand the setup below to create them. + + + +`curation_documents` has six records describing four documents. +`curation_labels` has one label for each of three documents. `row_id` identifies +a record; `doc_id` identifies the document it describes. + +Run each statement separately in a scratch database or namespace. Create these +tables once for the whole section, using fresh names if they already exist. ```sql -SELECT - count(*) AS rows, - count(embedding) AS embedded, - count(caption) AS captioned -FROM images +CREATE TABLE curation_documents ( + row_id BIGINT, doc_id BIGINT, version BIGINT, + source VARCHAR, caption VARCHAR, score DOUBLE +); +INSERT INTO curation_documents VALUES + (1, 10, 1, 'web', NULL, 0.80), + (2, 10, 2, 'web', 'A cat on a chair', 0.95), + (3, 20, 1, 'books', 'A dog in a field', 0.90), + (4, 30, 1, 'web', NULL, 0.70), + (5, 30, 1, 'web', 'A cat by a window', 0.85), + (6, 40, 1, 'books', 'A bird in a tree', 0.60); + +CREATE TABLE curation_labels (doc_id BIGINT, label VARCHAR); +INSERT INTO curation_labels VALUES + (10, 'cat'), (20, 'dog'), (30, 'cat'); ``` -`count(*)` counts rows; `count(col)` counts **non-null values**. The gap between -them is your coverage, which is the fastest way to check how far a -[column refresh](/feature-engineering/refresh-column) has got without waiting -for the job to report. +`score` is a sample quality score, not a search result score. Document 10 has +two versions. Document 30 has two records at the same version. Document 40 +has no label. These cases let you check the effect of each operation. -## Class balance {#balance} + -The single most useful curation query: +## Count records and missing values {#size} + +Use `count(*)` to count rows and `count(column)` to count non-null values. +Add `DISTINCT` to count unique values: ```sql -SELECT label, count(*) AS n -FROM images +SELECT count(*) AS records, + count(DISTINCT doc_id) AS documents, + count(caption) AS captioned, + count(*) FILTER (WHERE caption IS NULL) AS missing_caption +FROM curation_documents; +``` + +The example returns **6 records, 4 documents, 4 populated captions and 2 missing +captions**. `count(column)` excludes null values; it does not check whether a +string is empty or whether a value is useful for the task. + +## Group records by label {#balance} + +Use `GROUP BY` to calculate a summary for each label: + +```sql +SELECT label, count(*) AS documents +FROM curation_labels GROUP BY label -ORDER BY n DESC +ORDER BY documents DESC, label; ``` -If the top class is an order of magnitude larger than the rest, sampling -uniformly will reproduce that imbalance. Two ways out: draw a fixed number per -class with [windowing](/curation/windowing#per-group), or oversample the rare -ones and accept the duplication. +There are **2 cats and 1 dog** in the label table. Document 40 is absent, so this +query does not measure labeling coverage across all source documents. Use a +[left join](/curation/join#join-type) to find documents without labels. -## Distinct values {#distinct} +Decide whether the distribution matches your intended use. A uniform random +sample tends to preserve the source distribution; it does not balance classes. +To cap the number selected per label, use +[ranking within a group](/curation/windowing#per-group). + +## Count records that match a condition {#conditional} ```sql -SELECT count(DISTINCT source) FROM images +SELECT source, + count(*) AS records, + count(*) FILTER (WHERE score >= 0.9) AS high_score, + count(*) FILTER (WHERE caption IS NULL) AS missing_caption +FROM curation_documents +GROUP BY source +ORDER BY source; ``` -On a high-cardinality column this is expensive, because exact distinctness means -holding every value seen. When an estimate will do: +`FILTER` applies a condition to one aggregate. Here, `books` has 2 records, +1 high-score record and no missing captions. `web` has 4 records, 1 high-score +record and 2 missing captions. Counts are per record, so repeated document IDs +still contribute more than once. + + +## Group numeric values into ranges {#buckets} + +Use `count(DISTINCT column)` for the number of distinct non-null values. +For numeric fields, group by a range or summarize their distribution. For +example, this query counts records in score buckets of width 0.1: ```sql -SELECT approx_distinct(source) FROM images +SELECT floor(score * 10) / 10 AS score_bucket, count(*) AS records +FROM curation_documents +GROUP BY score_bucket +ORDER BY score_bucket; ``` -Cheap enough to run on a table where the exact version is not worth the wait. -The other approximate functions — `approx_median`, `approx_percentile_cont` — -are useful for the same reason when profiling a numeric column. +For time-based data, group by the time interval relevant to the task. The +[aggregation reference](/sql/dql/aggregation) and +[scalar function reference](/sql/scalar-functions) cover additional summaries, +including approximate counts and percentiles. Use an approximate aggregate when an estimate is sufficient; use an exact +count when the result must be precise. -## Conditional counts {#conditional} +## Profile a sample {#sampling} -`FILTER` narrows one aggregate without narrowing the query, which is how you get -several slices in one pass rather than one query each: +A [sample](/curation/sampling) can help you explore a large dataset before a +full scan. Report the sample size and method alongside the result. A sample can miss rare classes, and fragment sampling can reflect the order +in which records were written. -```sql -SELECT - label, - count(*) AS total, - count(*) FILTER (WHERE score > 0.9) AS confident, - count(*) FILTER (WHERE caption IS NULL) AS uncaptioned -FROM images -GROUP BY label -``` +Before accepting a curated dataset, check its required fields and key +constraints directly rather than inferring them from a small sample. -## Grouping by a bucket {#buckets} +## Continue curating {#reference} -Raw values are rarely the grouping you want. `date_bin` buckets timestamps, and -ordinary arithmetic buckets numbers: - -```sql -SELECT date_bin(INTERVAL '1 day', created_at) AS day, count(*) -FROM images GROUP BY day ORDER BY day -``` - -A histogram of a score column is the same trick: - -```sql -SELECT floor(score * 10) / 10 AS bucket, count(*) -FROM images GROUP BY bucket ORDER BY bucket -``` - -## Profiling a sample {#sampling} - -On a very large table, profile a [sample](/curation/sampling) and scale: - -```sql -SELECT label, count(*) * 100 AS estimated -FROM images TABLESAMPLE (1) -GROUP BY label -``` - -Treat the result as an estimate. A 1% sample will miss classes rarer than about -1 in 10,000 entirely, which is exactly the tail you are usually looking for — so -use this to check the shape of the head, not the existence of the tail. - -## Reference {#reference} - -The complete clause and function list is in -[Aggregation](/sql/dql/aggregation). +Use these measurements to choose a [sample](/curation/sampling), +[attach missing metadata](/curation/join), or +[select one record per document](/curation/windowing). +See [Aggregation](/sql/dql/aggregation) for complete SQL syntax. diff --git a/docs/web/curation/index.mdx b/docs/web/curation/index.mdx index 0bfc89b2d..fd8ea73aa 100644 --- a/docs/web/curation/index.mdx +++ b/docs/web/curation/index.mdx @@ -1,54 +1,125 @@ --- title: "Data curation" sidebarTitle: "Overview" -description: "Shaping a dataset before you train on it: profiling, sampling, joining, and ranking." +description: "Prepare datasets for search, evaluation and training by profiling, selecting and combining records." icon: filter keywords: ["curation", "dataset", "training set", "sampling", "deduplication", "class balance"] --- -Curation is the work between having data and having a *dataset*: finding out -what you actually have, drawing a subset you can afford to train on, attaching -labels that live elsewhere, and picking one row per thing when you have several. +Data curation turns a collection of records into a dataset suited to a task. +You might select labeled examples for training, prepare a fixed evaluation +set, or remove duplicate documents from a search corpus. -These pages are task-oriented. Each shows the operation in the context you would -reach for it, and links to the [SQL reference](/sql) for the complete syntax. +Start with the result you need. Then inspect the source data, choose which +records belong, and save the selected data or the rules used to produce it. - -Most curation is analytical -- grouping, joining, ranking -- which the query -builders do not do. It is therefore SQL, and SQL needs a deployment. See -[SQL](/basics/sql#sql-is-remote). - +## Choose a curation task {#questions} -## The four questions {#questions} - -| | | +| Task | What it helps you decide | | --- | --- | -| [What do I have?](/curation/aggregate) | counts, class balance, null coverage, distinct values | -| [What can I afford?](/curation/sampling) | a random subset, reproducible when you need it fixed | -| [What else do I know?](/curation/join) | labels, metadata and scores that live in another table | -| [Which one do I keep?](/curation/windowing) | one row per key, ranking within a group, splits | +| [Profile a dataset](/curation/aggregate) | Whether coverage, missing values and class balance suit the task. | +| [Sample records](/curation/sampling) | Which subset to inspect or use for an experiment. | +| [Join datasets](/curation/join) | How to attach labels, metadata or scores stored elsewhere. | +| [Rank and deduplicate](/curation/windowing) | Which records to keep when several describe the same item. | -## A worked shape {#shape} +The analytical examples in this section use **Enterprise SQL**. Connect with +the [SQL guide](/basics/sql). For sampling through the Python SDK in OSS or +Enterprise, see [Sample without SQL](/curation/sampling#training). -The four compose into the same pipeline most curation ends up being: +## Prepare a labeled document dataset {#shape} + +Suppose you need one labeled record per document for a training experiment. +Inspect the source records, choose the latest version of each document, then +join its label. Finally, check the class balance of the selected records. + +### Example data {#example-data} + +Connect with an [Enterprise SQL client](/basics/sql). The examples use a small +document table and a separate label table. Expand the setup below to create them. + + + +`curation_documents` has six records describing four documents. +`curation_labels` has one label for each of three documents. `row_id` identifies +a record; `doc_id` identifies the document it describes. + +Run each statement separately in a scratch database or namespace. Create these +tables once for the whole section, using fresh names if they already exist. + +```sql +CREATE TABLE curation_documents ( + row_id BIGINT, doc_id BIGINT, version BIGINT, + source VARCHAR, caption VARCHAR, score DOUBLE +); +INSERT INTO curation_documents VALUES + (1, 10, 1, 'web', NULL, 0.80), + (2, 10, 2, 'web', 'A cat on a chair', 0.95), + (3, 20, 1, 'books', 'A dog in a field', 0.90), + (4, 30, 1, 'web', NULL, 0.70), + (5, 30, 1, 'web', 'A cat by a window', 0.85), + (6, 40, 1, 'books', 'A bird in a tree', 0.60); + +CREATE TABLE curation_labels (doc_id BIGINT, label VARCHAR); +INSERT INTO curation_labels VALUES + (10, 'cat'), (20, 'dog'), (30, 'cat'); +``` + +`score` is a sample quality score, not a search result score. Document 10 has +two versions. Document 30 has two records at the same version. Document 40 +has no label. These cases let you check the effect of each operation. + + + +The following query chooses the latest record for each document, then keeps +only documents with a label: ```sql WITH ranked AS ( - SELECT d.*, c.label, - row_number() OVER (PARTITION BY d.doc_id ORDER BY d.version DESC) AS rn - FROM documents AS d - JOIN labels AS c ON c.doc_id = d.doc_id + SELECT *, row_number() OVER ( + PARTITION BY doc_id ORDER BY version DESC, row_id DESC + ) AS rn + FROM curation_documents ) -SELECT * FROM ranked WHERE rn = 1 +SELECT d.row_id, d.doc_id, d.caption, l.label +FROM ranked AS d +JOIN curation_labels AS l ON l.doc_id = d.doc_id +WHERE d.rn = 1 +ORDER BY d.doc_id; ``` -Join to bring in what you know, window to pick one row per document, filter to -keep it. Add [`TABLESAMPLE`](/curation/sampling) when the result is still larger -than you want, and [aggregate](/curation/aggregate) to check the balance of what -you ended up with. +The result contains documents **10, 20 and 30**, with labels **cat, dog and +cat**. Document 40 is excluded because this particular task requires a label. +The `row_id` tie-breaker makes the choice for document 30 deterministic; +choose a tie-breaker that fits your own data. -## Then what {#next} +The example assumes one label per document. Duplicate keys in the label table +would multiply the result rows. The [joining guide](/curation/join#labels) +shows how to check that assumption. -A curated result you want to keep is a -[materialized view](/management/materialized-view) -- stored, refreshed, and -readable by a [data loader](/training/data-loading) like any other table. +## Check the selected dataset {#check-result} + +Compare the result with the task's requirements: are the expected documents +present, are required fields populated, and is the class distribution suitable? +This example has two cats and one dog. That distribution may be appropriate +for exploration, but it does not establish a representative evaluation set. + +[Profiling](/curation/aggregate) provides the counts to inspect. +[Sampling](/curation/sampling) and [ranking](/curation/windowing) help you +adjust the selection without changing the original records. + +## Save and reuse the result {#next} + +Choose the output according to how you will use it: + +- **A fixed dataset:** save the query result in a new table with + [`CREATE TABLE AS SELECT`](/sql/ddl/table#create). Keep that output unchanged + when comparing experiments. +- **A reusable selection:** save the query as a [view](/management/view). + A view stores the query definition, so it is not a frozen evaluation set. +- **Stored results that you refresh:** use a + [materialized view](/management/materialized-view) when its supported query + and refresh behavior fit the workflow. + +Record the source versions and selection rules alongside the output. Continue +with [data loading](/training/data-loading) to use the curated data in a +training pipeline. diff --git a/docs/web/curation/join.mdx b/docs/web/curation/join.mdx index b4c198ef9..03a45326e 100644 --- a/docs/web/curation/join.mdx +++ b/docs/web/curation/join.mdx @@ -1,91 +1,133 @@ --- title: "Joining datasets" sidebarTitle: "Join" -description: "Bringing labels, metadata and scores together with the rows they describe." +description: "Combine records with labels or metadata, and check for missing or duplicate matches." icon: link keywords: ["join", "labels", "metadata", "enrichment", "namespace"] +availability: + oss: unavailable + enterprise: available + summary: >- + Enterprise supports SQL joins. --- -Enterprise Joins are SQL, and -[SQL needs a deployment](/basics/sql#sql-is-remote). +A SQL join combines records from two tables using a shared key. Use joins to +attach labels, metadata or model scores to your documents. Choose an inner join +to keep matching records, or a left join to retain records without a match. -Curation usually means combining things that were produced separately — images -in one table, human labels in another, model scores in a third. A join is how -they become one dataset. +## Example data {#example-data} -## Attaching labels {#labels} +Connect with an [Enterprise SQL client](/basics/sql). The examples use a small +document table and a separate label table. Expand the setup below to create them. + + + +`curation_documents` has six records describing four documents. +`curation_labels` has one label for each of three documents. `row_id` identifies +a record; `doc_id` identifies the document it describes. + +Run each statement separately in a scratch database or namespace. Create these +tables once for the whole section, using fresh names if they already exist. ```sql -SELECT i.id, i.embedding, l.label -FROM images AS i -JOIN labels AS l ON l.image_id = i.id +CREATE TABLE curation_documents ( + row_id BIGINT, doc_id BIGINT, version BIGINT, + source VARCHAR, caption VARCHAR, score DOUBLE +); +INSERT INTO curation_documents VALUES + (1, 10, 1, 'web', NULL, 0.80), + (2, 10, 2, 'web', 'A cat on a chair', 0.95), + (3, 20, 1, 'books', 'A dog in a field', 0.90), + (4, 30, 1, 'web', NULL, 0.70), + (5, 30, 1, 'web', 'A cat by a window', 0.85), + (6, 40, 1, 'books', 'A bird in a tree', 0.60); + +CREATE TABLE curation_labels (doc_id BIGINT, label VARCHAR); +INSERT INTO curation_labels VALUES + (10, 'cat'), (20, 'dog'), (30, 'cat'); ``` -An inner join keeps only rows that matched, which for curation is often exactly -what you want: unlabelled rows are not training data. +`score` is a sample quality score, not a search result score. Document 10 has +two versions. Document 30 has two records at the same version. Document 40 +has no label. These cases let you check the effect of each operation. -When you need to *see* what did not match, use a left join and filter: + + +## Basic join {#join-type} + +An inner join keeps matching records: ```sql -SELECT i.id -FROM images AS i -LEFT JOIN labels AS l ON l.image_id = i.id -WHERE l.image_id IS NULL +SELECT d.row_id, d.doc_id, l.label +FROM curation_documents AS d +JOIN curation_labels AS l ON l.doc_id = d.doc_id +ORDER BY d.row_id; ``` -That is your unlabelled backlog, which is the thing worth counting before you -plan an annotation round. +This returns **5 rows**: both records for document 10, one for document 20, +and both records for document 30. To produce one row per document, +[deduplicate the source records](/curation/windowing#dedup). -## Across namespaces {#namespaces} +### Find records without a match -Tables in different [namespaces](/management/namespace) join by their dotted -paths — the catalog is one catalog, so there is nothing to federate: +A left join keeps every record from the left table and fills missing right-side +values with `NULL`. Add `WHERE l.doc_id IS NULL` to return only unlabeled documents: ```sql -SELECT i.id, u.team -FROM prod.media.images AS i -JOIN prod.reference.uploaders AS u ON u.id = i.uploader_id +SELECT DISTINCT d.doc_id +FROM curation_documents AS d +LEFT JOIN curation_labels AS l ON l.doc_id = d.doc_id +WHERE l.doc_id IS NULL +ORDER BY d.doc_id; ``` -## Joining a search result {#search} +The result is **document 40**. You might send it for annotation, retain it for +an unlabeled-data workflow, or exclude it from a task that requires labels. -Full-text search is a table function, so its result is a relation and joins like -any other. This is how you ask which categories a query matches, rather than -just which rows: +## Handle duplicate keys {#labels} + +This example joins `curation_documents.doc_id` to `curation_labels.doc_id`. +First check whether the label table contains more than one row per document: ```sql -SELECT c.name, count(*) AS hits -FROM fts('documents', 'puppy') AS m -JOIN categories AS c ON c.id = m.category_id -GROUP BY c.name -ORDER BY hits DESC +SELECT doc_id, count(*) AS labels +FROM curation_labels +GROUP BY doc_id +HAVING count(*) > 1; ``` - -Vector search has no SQL form, so a similarity result cannot be joined this way. -Run the search in an SDK, then join over its output — or write the result to a -table first. - +The example returns no rows. If your data returns duplicates, decide whether +they represent valid multiple labels or records that need consolidation. +A join does not enforce one-to-one matching. -## Making it fast {#performance} +## Join tables across namespaces {#namespaces} -Two things dominate join cost on a lakehouse table. +Use a table's qualified path to address it in another +[namespace](/management/namespace). For example, a table named `images` in +`prod.media` is addressed as `prod.media.images`. The same matching-key and +join-type rules apply; your identity needs access to the source objects. -**Index the join key.** A [scalar index](/indexing/scalar-index) on the key lets -the planner look rows up instead of scanning for them. Check with -[`EXPLAIN`](/sql/dql/explain): an indexed key shows a `ScalarIndexQuery`, an -unindexed one shows a `LanceScan` feeding the join. +## Combine search results with metadata {#search} -**Do not carry wide columns through.** Select the columns you need before -joining, not after. Embeddings and blobs are the expensive part of a multimodal -table, and a join can materialize them many more times than the result has rows. +A [SQL full-text search](/sql/dql/fts) returns rows that can be joined to other +tables. Join using an identifier retained in the search result, then check +whether the metadata has one or several matches per result. -```sql -SELECT i.id, l.label -FROM (SELECT id, uploader_id FROM images) AS i -JOIN labels AS l ON l.image_id = i.id -``` +For an SDK search, retain the matching IDs and combine them with metadata in +your application or save the selected records to a table for further SQL work. +See [SQL search interfaces](/basics/sql#no-vector-sql) for supported query forms. -## Reference {#reference} +## Join performance {#performance} -The join types and their semantics are in [Joins](/sql/dql/join). +Check the row count, unmatched keys and duplicate matches before using a +joined dataset. Select only the columns needed for the next step, especially +when source records include large media or embedding columns. + +Use [Explain a query plan](/search/explain) to see how the query reads and +combines data. Check the plan to see whether the query uses indexes and how it performs the +join. An index on a key does not guarantee that the join will use it. + +## Save the result or continue curating {#reference} + +Keep the query for reuse or [save the selected data](/curation#next). +See [Joins](/sql/dql/join) for join types and exact SQL behavior. diff --git a/docs/web/curation/sampling.mdx b/docs/web/curation/sampling.mdx index 4e8b000cd..643b06bdf 100644 --- a/docs/web/curation/sampling.mdx +++ b/docs/web/curation/sampling.mdx @@ -1,98 +1,172 @@ --- -title: "Sampling a dataset" -sidebarTitle: "Sampling" -description: "Drawing a subset you can afford to work with, and fixing it when it has to be reproducible." +title: "SQL sampling" +sidebarTitle: "SQL Sampling" +description: "Choose a sample for exploration or evaluation and preserve the selected records when reproducibility matters." icon: dice keywords: ["sampling", "tablesample", "subset", "eval set", "reproducible", "shuffle"] +availability: + oss: unavailable + enterprise: available + summary: >- + Enterprise supports SQL `TABLESAMPLE`. For sampling in OSS and Enterprise through the Python SDK, see [Sample without SQL](#training). --- -Most curation starts with a subset. A question you can answer in ten seconds on -1% of the data is worth asking twenty times; the same question on the whole -table is worth asking once. +import { PySampleDataset } from '/snippets/curation_sampling.mdx'; -## A quick look {#quick} +Sampling selects records from a larger table. Use it to explore unfamiliar +data, run a smaller experiment, or prepare an evaluation dataset. Enterprise +SQL provides `TABLESAMPLE` for selecting a percentage or number of rows. -Enterprise +This page covers Enterprise SQL sampling and a complete +[Python SDK example](#training) that runs locally. + +## Example data {#example-data} + +Connect with an [Enterprise SQL client](/basics/sql). The examples use a small +document table and a separate label table. Expand the setup below to create them. + + + +`curation_documents` has six records describing four documents. +`curation_labels` has one label for each of three documents. `row_id` identifies +a record; `doc_id` identifies the document it describes. + +Run each statement separately in a scratch database or namespace. Create these +tables once for the whole section, using fresh names if they already exist. ```sql -SELECT * FROM images TABLESAMPLE (1) -- 1 percent -SELECT * FROM images TABLESAMPLE (500 ROWS) -- 500 rows +CREATE TABLE curation_documents ( + row_id BIGINT, doc_id BIGINT, version BIGINT, + source VARCHAR, caption VARCHAR, score DOUBLE +); +INSERT INTO curation_documents VALUES + (1, 10, 1, 'web', NULL, 0.80), + (2, 10, 2, 'web', 'A cat on a chair', 0.95), + (3, 20, 1, 'books', 'A dog in a field', 0.90), + (4, 30, 1, 'web', NULL, 0.70), + (5, 30, 1, 'web', 'A cat by a window', 0.85), + (6, 40, 1, 'books', 'A bird in a tree', 0.60); + +CREATE TABLE curation_labels (doc_id BIGINT, label VARCHAR); +INSERT INTO curation_labels VALUES + (10, 'cat'), (20, 'dog'), (30, 'cat'); ``` -A bare number is a percentage; add `ROWS` for a count. Use it while you are -exploring — the cost of the query drops with the sample, and so does the cost -of being wrong about what you wanted to ask. +`score` is a sample quality score, not a search result score. Document 10 has +two versions. Document 30 has two records at the same version. Document 40 +has no label. These cases let you check the effect of each operation. -## Rows or fragments {#methods} + + +## Basic sampling {#quick} + +A percentage selects a fraction of the source; `ROWS` requests a row count: ```sql -SELECT * FROM images TABLESAMPLE BERNOULLI (10) -- rows -SELECT * FROM images TABLESAMPLE SYSTEM (10) -- fragments +SELECT row_id, doc_id FROM curation_documents TABLESAMPLE (50); +SELECT row_id, doc_id FROM curation_documents TABLESAMPLE (3 ROWS); ``` -`BERNOULLI` considers each row independently and reads across the whole table. -`SYSTEM` picks whole Lance fragments and takes the rows in them — far less I/O, -because it skips files rather than rows. +A percentage sample does not guarantee an exact number of rows. Use a count +when the desired sample size is the main constraint, and inspect the result +rather than assuming it contains every category you need. -The trade is uniformity. A fragment sample is only as representative as your -write order. If rows arrived grouped by source, by date, or by label — which is -usual — then a fragment sample inherits that grouping. +`TABLESAMPLE` requires a table that is not WAL-backed. See the +[sampling reference](/sql/dql/sampling#refused) for this restriction; do not +change a production table's write configuration just to try an example. - -Reach for `SYSTEM` when you want a cheap look, and `BERNOULLI` when the number -you compute has to be defensible. Omitting the method gives `BERNOULLI`. - +## Sampling methods {#methods} -## A fixed evaluation set {#fixed} +Choose a method according to how records should be selected: -Without a seed, every execution resamples. That is fine for exploring and wrong -for evaluation — a metric computed on different rows each run is not comparable -with itself. +| Method | Selection | When to use it | +| --- | --- | --- | +| `BERNOULLI` (default) | Select individual rows | When records should be sampled independently of their storage fragments | +| `SYSTEM` | Select whole Lance fragments | When reducing reads matters and grouping by fragment is acceptable | ```sql -SELECT * FROM images TABLESAMPLE (10) REPEATABLE (42) +SELECT row_id, doc_id +FROM curation_documents TABLESAMPLE BERNOULLI (50); + +SELECT row_id, doc_id +FROM curation_documents TABLESAMPLE SYSTEM (50); ``` -The same seed against the same data gives the same rows, which is what makes a -sample usable as a held-out set. If the underlying data changes, so does the -draw — when you need the set frozen against that too, materialize it: +If write order groups records by label, date or source, a fragment sample can +reflect that grouping. On a tiny table stored in one fragment, it may include +all records or none. Row sampling also needs an adequate sample size to +represent rare groups; check that the resulting sample covers the groups you need. + +## Repeatable samples {#fixed} + +A seed requests a repeatable draw: ```sql -CREATE TABLE eval_set AS -SELECT * FROM images TABLESAMPLE (10) REPEATABLE (42) +SELECT row_id, doc_id +FROM curation_documents TABLESAMPLE (50) REPEATABLE (42); ``` -A [materialized view](/management/materialized-view) is the other option when -you want it to track the source on a refresh rather than stay frozen. - -## Sampling both sides of a join {#joins} +Keep the source data and sampling configuration fixed when comparing draws. +For a lasting evaluation set, save the selected records rather than relying +on a seed to preserve membership as the source changes: ```sql -SELECT * FROM a TABLESAMPLE (10) JOIN b TABLESAMPLE (10) ON a.id = b.id +CREATE TABLE curation_eval AS +SELECT * FROM curation_documents TABLESAMPLE (50) REPEATABLE (42); ``` -Each side samples independently, so a row survives only if both of its sides -were drawn. Sampling 10% of each side does **not** give 10% of the join — it -gives roughly 1%. Sample once, after the join, unless you specifically want the -smaller result. +Create this output once with a fresh name and keep it unchanged. Record the +source version and selection policy. Sampling records independently can place +revisions of the same document into different splits; decide the +[grouping policy](/curation/windowing#splits) before using the result for +training or evaluation. -## Shuffling for training {#training} +## Sampling with joins {#joins} -For a training loop rather than a query, the SDKs read a table in a shuffled -order without SQL at all: +Sampling both input tables independently changes which pairs can match. +For a one-to-one match, independent 10% row samples retain a given pair with +about 1% probability, not 10%. -```python -from lancedb.permutation import Permutation +To sample documents while retaining their available labels, sample only the +document table: -permutation = Permutation.identity(table) -dataloader = torch.utils.data.DataLoader(permutation) +```sql +SELECT curation_documents.row_id, curation_documents.doc_id, l.label +FROM curation_documents TABLESAMPLE (50) +JOIN curation_labels AS l ON l.doc_id = curation_documents.doc_id; ``` -This is the right tool for epoch-level shuffling, where you want every row -eventually and only the order varies. `TABLESAMPLE` is for when you want fewer -rows. See [Training](/training/data-loading). +This samples source records, not distinct documents. If you need to sample the +result of a join or deduplication, first save that result as a table suitable +for sampling. `TABLESAMPLE` cannot be applied directly to a subquery result. -## Reference {#reference} +## Sample without SQL {#training} -The full clause, its options and what it refuses are in -[Sampling](/sql/dql/sampling). +The Python SDK provides a permutation builder for selecting and ordering rows +in OSS and Enterprise. A split selects a subset; a shuffle changes the order +in which records are read. `Permutation.identity(table)` retains stored order. + + + +Install the Python SDK with `pip install lancedb`, save the program as +`sample_dataset.py`, and run `python sample_dataset.py`. It creates a temporary +local database with ten rows and reads a random split containing three rows. +The temporary database is removed when the program finishes. + + +{PySampleDataset} + + +The program prints three distinct IDs. It uses a fixed seed, but does not +promise the same IDs across SDK versions or changes to the source dataset. + + + +For shuffled training reads, worker configuration and data loaders, see +[Loading data for model training](/training/data-loading). + +## Related guides {#reference} + +Count the sampled rows and [inspect their distribution](/curation/aggregate) +before using them. Preserve the selected data when the exact membership matters. +See [SQL sampling](/sql/dql/sampling) for syntax and restrictions. diff --git a/docs/web/curation/windowing.mdx b/docs/web/curation/windowing.mdx index fbb3cbbf2..fa84ed5fe 100644 --- a/docs/web/curation/windowing.mdx +++ b/docs/web/curation/windowing.mdx @@ -1,124 +1,162 @@ --- title: "Ranking and deduplication" sidebarTitle: "Windowing" -description: "Picking one row per thing, ranking within a group, and splitting a dataset." +description: "Choose records with explicit ranking rules and deterministic tie handling." icon: chart-simple keywords: ["window", "deduplication", "row_number", "rank", "train test split", "per class"] +availability: + oss: unavailable + enterprise: available + summary: >- + Enterprise supports SQL window functions. --- -Enterprise Window functions are SQL, and -[SQL needs a deployment](/basics/sql#sql-is-remote). +Window functions calculate values across related rows while keeping each row +in the result. Use them to rank documents, keep one record per item, or compare +a record with the one before it. -A window function computes across a set of related rows and returns a value -**per row** — nothing is collapsed. That is what makes it the tool for "which of -these do I keep", where an aggregate would throw away the rows you are choosing -between. +## Example data {#example-data} -## One row per thing {#dedup} +Connect with an [Enterprise SQL client](/basics/sql). The examples use a small +document table and a separate label table. Expand the setup below to create them. -The most common curation query there is. You have several rows per document — -revisions, re-uploads, repeated annotations — and want one. + + +`curation_documents` has six records describing four documents. +`curation_labels` has one label for each of three documents. `row_id` identifies +a record; `doc_id` identifies the document it describes. + +Run each statement separately in a scratch database or namespace. Create these +tables once for the whole section, using fresh names if they already exist. + +```sql +CREATE TABLE curation_documents ( + row_id BIGINT, doc_id BIGINT, version BIGINT, + source VARCHAR, caption VARCHAR, score DOUBLE +); +INSERT INTO curation_documents VALUES + (1, 10, 1, 'web', NULL, 0.80), + (2, 10, 2, 'web', 'A cat on a chair', 0.95), + (3, 20, 1, 'books', 'A dog in a field', 0.90), + (4, 30, 1, 'web', NULL, 0.70), + (5, 30, 1, 'web', 'A cat by a window', 0.85), + (6, 40, 1, 'books', 'A bird in a tree', 0.60); + +CREATE TABLE curation_labels (doc_id BIGINT, label VARCHAR); +INSERT INTO curation_labels VALUES + (10, 'cat'), (20, 'dog'), (30, 'cat'); +``` + +`score` is a sample quality score, not a search result score. Document 10 has +two versions. Document 30 has two records at the same version. Document 40 +has no label. These cases let you check the effect of each operation. + + + +## Keep one record per document {#dedup} + +`row_number()` numbers rows within each group. `PARTITION BY` defines the group, +and `ORDER BY` determines which record comes first. This query selects the +latest version of each document, using `row_id` to break ties: ```sql WITH ranked AS ( - SELECT *, - row_number() OVER (PARTITION BY doc_id ORDER BY version DESC) AS rn - FROM revisions + SELECT *, row_number() OVER ( + PARTITION BY doc_id ORDER BY version DESC, row_id DESC + ) AS rn + FROM curation_documents ) -SELECT * FROM ranked WHERE rn = 1 +SELECT row_id, doc_id, version, caption +FROM ranked +WHERE rn = 1 +ORDER BY doc_id; ``` -`PARTITION BY` says what "per thing" means; `ORDER BY` says which one wins. The -outer query is not optional — a window function cannot appear in `WHERE`, -because windows are computed after filtering, so the predicate would have -nothing to read. +This returns row IDs **2, 3, 5 and 6**. For document 30, rows 4 and 5 have the +same version; the example deliberately chooses the larger `row_id`. +Without a complete ordering, the selected row among ties is not determined by +the query. - -Swap `row_number()` for `rank()` when ties should both survive. `row_number()` -always picks exactly one; `rank()` gives tied rows the same rank, so `rank = 1` -can return several. - +Use `rank()` instead of `row_number()` when tied records should all survive, +and order only by the fields that define a tie. Adding a unique tie-breaker +would make every rank distinct. The outer query filters the calculated rank +because window functions are evaluated after the inner query's `WHERE` clause. -## A fixed number per class {#per-group} +## Select the highest-scoring records per class {#per-group} -Class imbalance found by [profiling](/curation/aggregate#balance) is fixed here: -rank within each class, then take the top N. +Deduplicate first, then rank the labeled documents within each class: ```sql -WITH ranked AS ( - SELECT *, row_number() OVER (PARTITION BY label ORDER BY score DESC) AS rn - FROM images +WITH latest AS ( + SELECT *, row_number() OVER ( + PARTITION BY doc_id ORDER BY version DESC, row_id DESC + ) AS version_rank + FROM curation_documents +), ranked AS ( + SELECT d.doc_id, l.label, d.score, + row_number() OVER ( + PARTITION BY l.label ORDER BY d.score DESC, d.doc_id + ) AS class_rank + FROM latest AS d + JOIN curation_labels AS l ON l.doc_id = d.doc_id + WHERE d.version_rank = 1 ) -SELECT * FROM ranked WHERE rn <= 1000 +SELECT doc_id, label, score +FROM ranked +WHERE class_rank <= 1 +ORDER BY label; ``` -That gives at most 1,000 rows per label, taking the highest-scoring ones. Order -by something arbitrary but stable if you want an unbiased draw instead, or -combine with [sampling](/curation/sampling) to randomize within each class. +The result is **document 10 for cat and document 20 for dog**. Change the cap +from `1` to the maximum number you want per class. A class with fewer records +cannot supply the full cap. -## Splitting a dataset {#splits} +This selects by score, not at random. For a random subset, use +[sampling](/curation/sampling) instead of ordering by score or ID. -`ntile` divides each partition into equal buckets, which is a train/test split -that stays balanced per class: +## Training and evaluation splits {#splits} + +Ranking can divide ordered records into groups, but ordering by ID does not +make those groups random or representative. `ntile()` produces approximately +equal-sized buckets within a partition; small classes may not populate every +bucket. + +Choose the split policy for the task. Keep related records, such as revisions +of one document or frames from one video, in the same split to avoid leakage. +A time-based evaluation may instead require a cutoff date. Save the final +assignments or resulting datasets so later updates do not silently change the +evaluation population. + +## Compare neighboring records {#offset} + +`lag()` and `lead()` read neighboring rows in the specified order. For example, +inspect how scores change across a document's records: ```sql -WITH split AS ( - SELECT *, ntile(10) OVER (PARTITION BY label ORDER BY id) AS bucket - FROM images -) -SELECT * FROM split WHERE bucket <= 8 -- train +SELECT doc_id, row_id, score, + lag(score) OVER ( + PARTITION BY doc_id ORDER BY version, row_id + ) AS previous_score +FROM curation_documents +ORDER BY doc_id, version, row_id; ``` -Buckets 9 and 10 are your validation and test sets. Because the partition is by -label, each split keeps the class proportions of the whole. +The first record in each document has no previous score. In this dataset, +row 2 follows a score of 0.80 and row 5 follows a score of 0.70. - -Order by something stable. Ordering by a value that changes — a score you -recompute, a timestamp that gets updated — means the split changes with it, and -a row can move from train to test between runs. - + +## Window query performance {#cost} -## Comparing a row to its neighbours {#offset} +Window operations can require sorting. Filter out records you do not need +before ranking, and inspect the +[query plan](/search/explain) when cost matters. -`lag` and `lead` read the previous and next row in the window, which is how you -find gaps and changes without a self-join: +For cumulative counts, sums and other running summaries, use an aggregate +with an explicit window order and frame. The +[window reference](/sql/dql/window#window) covers those expressions. -```sql -SELECT - id, - created_at, - created_at - lag(created_at) OVER (ORDER BY created_at) AS gap -FROM events -``` +## Continue with the curated data {#reference} -Useful for spotting ingestion gaps, duplicate timestamps, and runs of identical -values that suggest a broken upstream job. - -## Running totals {#running} - -Any [aggregate](/curation/aggregate) becomes a window function with `OVER`, -which is how cumulative coverage is expressed: - -```sql -SELECT - label, - count(*) AS n, - sum(count(*)) OVER (ORDER BY count(*) DESC) AS cumulative -FROM images -GROUP BY label -``` - -Reading down the `cumulative` column tells you how many classes cover most of -your data — usually far fewer than you expect. - -## Cost {#cost} - -Window functions sort within each partition, so on a large table they are among -the more expensive things you can ask for. Filter first, partition on something -selective, and check the plan with -[Explain Analyze](/search/explain). - -## Reference {#reference} - -The full window syntax, framing, and function list are in -[Window functions](/sql/dql/window). +[Profile the selected records](/curation/aggregate) and +[save the result](/curation#next) for downstream use. +See [Window functions](/sql/dql/window) for the full syntax and function list. diff --git a/docs/web/deployment/monitoring.mdx b/docs/web/deployment/monitoring.mdx index 6a66eb0ed..5ad56ce1c 100644 --- a/docs/web/deployment/monitoring.mdx +++ b/docs/web/deployment/monitoring.mdx @@ -11,10 +11,10 @@ LanceDB emits internal metrics (currently object store request counts, bytes tra The bridge is available in the Python and TypeScript SDKs. It is a thin wrapper over LanceDB's `metrics` recorder; your application supplies and configures the OpenTelemetry SDK. -This page covers LanceDB OSS instrumentation. LanceDB Enterprise clusters emit their own metrics from the server side — see [Enterprise monitoring](#enterprise) below. +This page covers LanceDB OSS instrumentation. LanceDB Enterprise clusters emit their own metrics from the server side. See [Enterprise monitoring](#enterprise) below. -## What you get {#what-you-get} +## Available metrics {#what-you-get} Once instrumented, LanceDB registers one observable instrument per metric on your `MeterProvider`. The current catalog covers the object store layer: @@ -22,8 +22,10 @@ Once instrumented, LanceDB registers one observable instrument per metric on you |--------|------|-------------| | `lance_object_store_requests_total` | Counter | Total object store requests, labelled by `operation` and `base` (store scheme). | | `lance_object_store_request_duration_seconds` | Histogram | Request latency in seconds. | -| `lance_object_store_bytes_transferred_total` | Counter | Bytes read from or written to the store. | -| `lance_object_store_retryable_responses_total` | Counter | Requests that returned a retryable error (throttles, transient failures). | +| `lance_object_store_request_bytes_total` | Counter | Bytes read from or written to the store. | +| `lance_object_store_errors_total` | Counter | Object store requests that failed. | +| `lance_object_store_retryable_responses_total` | Counter | Retryable HTTP responses (5xx, 429 and 408) from a store reached over HTTP. | +| `lance_object_store_throttle_total` | Counter | Throttle responses (429 and 503) from a store reached over HTTP. | | `lance_object_store_in_flight_requests` | Gauge | Currently outstanding object store requests. | The recorder is process-global and pull-based: your configured `MetricReader` collects on its own schedule, so there is no hot-path overhead beyond the atomic aggregation that LanceDB does anyway. @@ -37,7 +39,7 @@ The recorder is process-global and pull-based: your configured `MetricReader` co Install LanceDB with the `otel` extra to pull in the OpenTelemetry API, plus an OpenTelemetry SDK of your choice. The SDK is intentionally not bundled, so you configure it and its readers and exporters however your platform expects. ```bash -pip install "lancedb[otel]" opentelemetry-sdk +pip install "lancedb[otel]" opentelemetry-sdk opentelemetry-exporter-otlp-proto-grpc ``` Call `instrument_lancedb_metrics()` once at startup, before opening any tables. It returns `True` when the recorder is installed and instruments are registered. @@ -70,7 +72,7 @@ If you omit `meter_provider`, LanceDB uses the global provider returned by `open ## TypeScript {#typescript} -The Node SDK depends on `@opentelemetry/api` directly, so no extra install step is needed to expose the entry point. You still need an OpenTelemetry SDK to actually export. +Install the OpenTelemetry SDK and exporter in your application: ```bash npm install @opentelemetry/sdk-metrics @opentelemetry/exporter-metrics-otlp-grpc @@ -93,7 +95,7 @@ const db = await connect("s3://my-bucket/lancedb"); `instrumentLanceDbMetrics()` also accepts no arguments, in which case it uses the global provider from `@opentelemetry/api`. Calling it more than once is safe: instruments are created only on the first successful call. -## What to watch {#what-to-watch} +## Build dashboards and alerts {#what-to-watch} A few starting points for dashboards and alerts: @@ -116,7 +118,7 @@ export them with data enrichment -- no client-side instrumentation needed. [Contact support](/deployment/support) for the comprehensive metric list and help integrating them with your monitoring dashboard. -## Where to go next {#where-to-go-next} +## Related guides {#where-to-go-next} diff --git a/docs/web/deployment/storage.mdx b/docs/web/deployment/storage.mdx index fd8550ea6..4994e8f0c 100644 --- a/docs/web/deployment/storage.mdx +++ b/docs/web/deployment/storage.mdx @@ -1,11 +1,24 @@ --- -title: "Storage" -sidebarTitle: "Storage" -description: "Choosing and configuring where LanceDB stores its data -- how you deploy an embedded database." +title: "Embedded storage configuration" +sidebarTitle: "Embedded Storage" +description: "Choose and configure local disk or object storage for tables accessed by the embedded LanceDB OSS library." icon: database keywords: ["storage", "s3", "gcs", "azure", "object store", "configuration", "deploy"] +availability: + oss: available + enterprise: unavailable + summary: >- + The embedded library chooses and configures its storage backend from the + connection URI and `storage_options`. In LanceDB Enterprise, storage is part of + the deployment's own configuration instead. --- +This page is about the embedded library, where your connection chooses and +configures the storage backend. A LanceDB Enterprise deployment sets up its own +storage when it is deployed, so a `db://` connection passes none of these +options except an Azure storage account name -- see +[Deployment](/enterprise/deployment) for how a deployment is set up. + LanceDB's storage layer is built on modular, disk-first components. That design makes it flexible enough to run across local NVMe, EBS, EFS, and any object store that exposes an S3-compatible API. It also supports region-specific backends such as [Tencent COS](/storage/configuration#tencent-cos) and cache-acceleration layers such as [GooseFS](/storage/configuration#goosefs). Choosing a backend is a balance between latency, scalability, cost, and operational complexity. Use this guide to pick the right fit for your workload. @@ -176,9 +189,9 @@ Table-level `storage_options` inherit every key from the connection and override -**Inspect the effective options** +**Inspect a table's own options** -On `AsyncTable`, `await table.initial_storage_options()` returns the options the table was opened with, and `await table.latest_storage_options()` returns the current options after any provider-driven refresh. The deprecated `table.storage_options()` method will be removed in a future release. +On `AsyncTable`, `await table.initial_storage_options()` returns the options passed when the table was opened, and `await table.latest_storage_options()` returns them after any provider-driven refresh. Neither includes what the table inherits from its connection's `storage_options`, which still apply. The deprecated `table.storage_options()` method will be removed in a future release. #### General object store options {#general-object-store-options} diff --git a/docs/web/deployment/support.mdx b/docs/web/deployment/support.mdx index f758cab00..5e35d9131 100644 --- a/docs/web/deployment/support.mdx +++ b/docs/web/deployment/support.mdx @@ -9,11 +9,10 @@ keywords: ["support", "help", "discord", "community"] ## LanceDB OSS {#oss} For a quick question, the -[Discord community](https://discord.gg/AUEWnJ7Txb) is fastest -- ask under the -relevant language channel for Python, TypeScript, or Rust, and you reach both -the community and the engineering team. +[Discord community](https://discord.gg/AUEWnJ7Txb) has channels for +Python, TypeScript and Rust where you can ask the community for help. -For anything worth keeping searchable for the next person, use GitHub instead: +Use GitHub for discussions and issues that others can search and follow: | | | | --- | --- | diff --git a/docs/web/docs.json b/docs/web/docs.json index 38c2bc6cd..a8fb44007 100644 --- a/docs/web/docs.json +++ b/docs/web/docs.json @@ -63,10 +63,6 @@ "source": "/feature-engineering/computed-columns", "destination": "/feature-engineering/add-computed-column" }, - { - "source": "/feature-engineering/functions", - "destination": "/management/function" - }, { "source": "/tables/create", "destination": "/ingestion" @@ -332,8 +328,7 @@ "management/table", "tables/branching", "management/materialized-view", - "management/view", - "management/function" + "management/view" ] }, { @@ -383,7 +378,6 @@ "sql/ddl/view", "sql/ddl/materialized-view", "sql/ddl/index", - "sql/ddl/function", "sql/ddl/job" ], "icon": "sitemap" diff --git a/docs/web/embedding/index.mdx b/docs/web/embedding/index.mdx index e3568604b..9f1531c05 100644 --- a/docs/web/embedding/index.mdx +++ b/docs/web/embedding/index.mdx @@ -3,6 +3,13 @@ title: "Managing Embeddings" sidebarTitle: "Managing Embeddings" description: "Use the embedding API in LanceDB -- registry, functions, schemas, and multi-language SDK support." icon: "bars" +availability: + oss: available + enterprise: available + summary: >- + Embedding functions run in your process, whether the table is local or on a + deployment, which stores their configuration but never runs them. Python + and TypeScript also embed query text; in Rust you embed queries yourself. --- import { @@ -30,8 +37,9 @@ the data, so vectors that are close to each other are considered similar. LanceDB provides an embedding function registry in OSS as well as its Enterprise versions ([see below](#embeddings-in-lancedb-enterprise)) -that automatically generates vector embeddings during data ingestion. Automatic query-time embedding -generation is available in LanceDB OSS, with SDK-specific query ergonomics. The API abstracts +that automatically generates vector embeddings during data ingestion. Query text is embedded +automatically too, in the client, for local and Enterprise tables alike, with SDK-specific query +ergonomics. The API abstracts embedding generation, allowing you to focus on your application logic. ## Embedding Registry {#embedding-registry} @@ -41,6 +49,9 @@ Once configured, the embedding function will automatically generate embeddings w into the table. Query-time behavior depends on SDK: Python/TypeScript can query with text directly, while Rust examples typically compute query embeddings explicitly before vector search. +In the Rust examples, replace `your-openai-api-key` with your OpenAI API key. In +application code, load credentials from environment variables or a secrets manager. + {PyOpenaiEmbeddings} diff --git a/docs/web/enterprise/authentication.mdx b/docs/web/enterprise/authentication.mdx index f41603fdf..4d04120fc 100644 --- a/docs/web/enterprise/authentication.mdx +++ b/docs/web/enterprise/authentication.mdx @@ -4,14 +4,15 @@ sidebarTitle: "Authentication" description: "Authentication modes for LanceDB Enterprise: API keys and OAuth credentials." icon: key keywords: ["authentication", "oauth", "api key", "azure managed identity", "client credentials"] +availability: + oss: unavailable + enterprise: available + summary: >- + Requests to a deployment carry an API key or an OAuth token, whichever it is + configured to accept. Embedded LanceDB has no server to sign in to; see + [Architecture](/architecture). --- -Enterprise - -This page describes a LanceDB Enterprise capability. The embedded form -of LanceDB has no server to provide it -- see -[Architecture](/architecture) for what differs between the two. - A deployment authenticates every request before [authorization](/governance/authorization) decides what it may do, against a `db://` remote table, in one of two ways: @@ -50,6 +51,10 @@ from an OIDC issuer and attach them to every request. Token acquisition, caching, and refresh are handled inside the client -- your application code only provides the configuration. +A deployment accepts these tokens only when its administrator has configured +it with your identity provider. Ask them for the issuer URL, client ID and +scopes to use. + In Python, OAuth is supported through `lancedb.connect_async`. The synchronous `connect` entry point continues to use API key authentication for `db://` diff --git a/docs/web/enterprise/index.mdx b/docs/web/enterprise/index.mdx index ed4d232c1..a601e850d 100644 --- a/docs/web/enterprise/index.mdx +++ b/docs/web/enterprise/index.mdx @@ -21,9 +21,13 @@ What it adds over the embedded form: | | | | --- | --- | | [SQL](/sql) | an Arrow Flight SQL endpoint | -| [Functions](/management/function) | Python running on workers | | [Jobs](/basics/jobs) | server-side work with ids, status and events | | [Authorization](/governance/authorization) | per-object grants | | Scale | a cluster rather than your process | +{/* +Draft for the Functions release. Add this row to the table above, after SQL: +| [Functions](/management/function) | Python running on workers | +*/} + See [Architecture](/architecture) for the two shapes side by side. diff --git a/docs/web/feature-engineering/add-computed-column.mdx b/docs/web/feature-engineering/add-computed-column.mdx index 571797d4e..331918715 100644 --- a/docs/web/feature-engineering/add-computed-column.mdx +++ b/docs/web/feature-engineering/add-computed-column.mdx @@ -4,13 +4,18 @@ sidebarTitle: "Computed Columns" description: "Declare a column from a SQL expression. Nothing is evaluated until you refresh it." icon: table-columns keywords: ["computed columns", "add_columns", "transforms", "schema"] +availability: + oss: available + enterprise: available + summary: >- + Both offerings support computed columns. Enterprise requires the deployment to enable them; the embedded library evaluates them in your process. --- A computed column is declared with an expression and starts with no values. Declaring it is cheap on any table size, because nothing is evaluated until [a refresh](/feature-engineering/refresh-column) asks for it. -## Declaring one {#declare} +## Add a computed column {#declare} ```python import lancedb @@ -26,14 +31,17 @@ That commits a new table version with the column present and empty. It is an ordinary table version, so the usual [versioning](/management/table#versioning) applies -- including that you can travel back to before the column existed. -## `computed` and `transforms` are different {#computed-vs-transforms} +## Compute now or on refresh {#computed-vs-transforms} `add_columns` takes either, and the difference matters: - **SQL expressions in `transforms`** are evaluated *now*, for every row, as part of the call. Good for a small table, or a column you want populated - immediately. A `FunctionApplication` passed through `transforms` instead - declares empty Function-bound columns; a refresh fills them. + immediately. + {/* Draft for the Functions release: + A `FunctionApplication` passed through `transforms` instead + declares empty Function-bound columns, which a refresh fills; that needs a + deployment with Functions enabled, and the embedded library refuses it. */} - **`computed`** stores the expression and evaluates nothing. The column is committed empty and filled later by [`refresh_column`](/feature-engineering/refresh-column). @@ -48,8 +56,8 @@ the table, including another computed column: ```python table.add_columns(computed={ - "power_score": "(stats.magic + stats.strength) / 2.0", - "is_strong": "power_score > 3.0", + "power_score": "cast(x as double) / 2.0", + "is_strong": "power_score > 0.5", }) ``` @@ -59,12 +67,22 @@ declared earlier in the same mapping, but not one declared after it. The functions available are the ones the [SQL dialect](/sql/functions) documents. -## Its inputs become load-bearing {#input-constraints} +A literal in the expression has to fit the column it meets: `x / 2.0` is +refused when `x` is an integer column, so write `cast(x as double) / 2.0`. + + +In LanceDB Enterprise, keep the expression to top-level columns. A column +refresh can fail on an expression that reads a field inside a struct, such as +`stats.magic + stats.strength`, which the embedded library computes. Try a new +expression on a small table first. + + +## Change columns used by an expression {#input-constraints} While a computed column's declaration references another column, that column -cannot be renamed, retyped, or dropped. The declaration is part of the schema, -and LanceDB will not let you invalidate it out from under itself. +cannot be renamed, retyped, or dropped. Drop the computed column first +if you need to change one of its inputs. This is worth knowing before deriving a column from something you expect to @@ -74,18 +92,33 @@ reshape later. Dropping the computed column releases the constraint. Enterprise +When the value cannot be expressed in SQL -- an embedding, a model call, an +image decode -- [Geneva](/geneva), the feature engineering package for LanceDB +Enterprise, fills the column with a Python function instead. + +{/* +Draft for the Functions release. Replaces the paragraph above: + When the value cannot be expressed in SQL -- an embedding, a model call, an image decode -- the column is filled by a registered [function](/management/function) instead of an expression. Declaring it looks -the same; what differs is what fills it. +the same; what differs is what fills it. It needs a deployment with Functions +enabled. +*/} -## What to read next {#next} +## Related guides {#next} Fill the rows that lack values. + + Fill a column with Python on LanceDB Enterprise. + + {/* + Draft for the Functions release. Replaces the Geneva card above: Fill a column with Python. + */} diff --git a/docs/web/feature-engineering/data-evolution.mdx b/docs/web/feature-engineering/data-evolution.mdx index 5f50bdcbb..f70612cf9 100644 --- a/docs/web/feature-engineering/data-evolution.mdx +++ b/docs/web/feature-engineering/data-evolution.mdx @@ -4,6 +4,12 @@ sidebarTitle: "Data Evolution" description: "Derive new columns from data already in the table, without rewriting what is already there." icon: wand-magic-sparkles keywords: ["data evolution", "derived columns", "add_columns", "zero copy", "feature engineering"] +availability: + oss: available + enterprise: available + summary: >- + Adding columns computed from SQL expressions works in the embedded library + and in LanceDB Enterprise. --- import { @@ -18,20 +24,23 @@ import { RsAddFeatureColumnsSql as RsAddFeatureColumnsSql, } from '/snippets/tables.mdx'; -Data evolution is adding **values** derived from what a table already holds. -[Schema evolution](/management/table#schema) changes the table's shape; this fills the -shape with something new. +Add derived columns to calculate features from data already in a table. +For example, compute a product score from existing fields without reading +and rewriting the table in your application. -The reason it is cheap is the format. Adding a derived column writes only that -column -- the existing data, which is the expensive part on a multimodal table, -is never rewritten. See [why that works](/basics/lance#evolution). +Lance can write the new columns without rewriting the existing columns. +This is useful when the table also contains large images, video or embeddings. +See [Lance format](/basics/lance) for the storage model. -In SQL this is [`ALTER TABLE ... ADD COLUMN ... AS`](/sql/ddl/table#computed-column). +In SQL, on an Enterprise cluster that enables computed columns, this is [`ALTER TABLE ... ADD COLUMN ... AS`](/sql/ddl/table#computed-column). ## Set up the example table {#set-up-the-example-table} +The examples use `db`, a connection opened as in +[Connect to LanceDB](/basics/connect), and create a small product table in it: + {PySchemaAddSetup} @@ -96,16 +105,24 @@ If your transformation cannot be expressed in SQL, compute the values outside ## When SQL is not enough {#beyond-sql} -The expressions above are evaluated once, as part of the call. Two other shapes -exist when that does not fit: +The expressions above are evaluated once, as part of the call. For other workflows: - A [computed column](/feature-engineering/add-computed-column) declares the expression and leaves the column empty, so you choose when the work happens. - That is the right default on a large table. + Use it when you want to schedule the computation separately. +- On LanceDB Enterprise, [Geneva](/geneva) fills a column with Python, for + values SQL cannot express -- an embedding, a model call, an image decode. + +{/* +Draft for the Functions release. Replaces the Geneva item in the list above: - A [function](/management/function) fills a column with Python, for values SQL cannot express -- an embedding, a model call, an image decode. +*/} -## What to read next {#next} +A deployment has to enable computed columns before you use them. +{/* Draft for the Functions release: Functions need enabling there too. */} + +## Related guides {#next} diff --git a/docs/web/feature-engineering/index.mdx b/docs/web/feature-engineering/index.mdx index 4b8200d40..7fe699648 100644 --- a/docs/web/feature-engineering/index.mdx +++ b/docs/web/feature-engineering/index.mdx @@ -3,52 +3,90 @@ title: "Feature engineering" sidebarTitle: "Overview" description: "Derive columns and tables from data already in the lakehouse, without moving it somewhere else to compute." icon: layer-group -keywords: ["feature engineering", "computed columns", "materialized views", "functions", "lakehouse", "jobs"] +keywords: ["feature engineering", "computed columns", "materialized views", "lakehouse", "jobs"] --- -Feature engineering happens where the data already is. Rather than exporting a -table, computing somewhere else and writing the result back, you declare what a -column or a table should contain and let LanceDB fill it. +Use feature engineering to turn raw data into features for search, analysis +and model training. LanceDB stores derived columns and materialized views +alongside the source data, so you can reuse the results across those workflows. + +[Materialized views](/feature-engineering/materialized-views) are ordinary +[catalog objects](/basics/catalog). Create them through the SDK or SQL, +then refresh the derived data when its inputs change. + +{/* +Draft for the Functions release. Replaces the paragraph above: [Functions](/management/function) and [materialized views](/feature-engineering/materialized-views) are ordinary -[catalog objects](/basics/catalog), alongside tables and indexes -- created, -versioned, and refreshed like anything else, not a separate system bolted -alongside. +[catalog objects](/basics/catalog). Create them through the SDK or SQL, +then refresh the derived data when its inputs change. +*/} -Every operation here has a SQL form -- see [DDL](/sql/ddl/table#alter) to declare a column and [REFRESH](/sql/dml/refresh) to fill it. +In LanceDB Enterprise, every operation here also has a SQL form -- see [DDL](/sql/ddl/table#alter) to declare a column and [REFRESH](/sql/dml/refresh) to fill it. -## The shape everything shares {#the-shape} +## Computed columns and materialized views {#the-shape} -Every mechanism here separates **declaring** from **filling**, and that -separation is the point: +Computed columns and materialized views both keep derived data in the +lakehouse, but they fill it differently, and the difference decides what a +refresh costs: -- Declaring is cheap and immediate. It records intent in the schema and commits - a new table version, with no data written. Declaring a column on a - billion-row table costs what declaring one on an empty table costs. -- Filling is a separate call you control, can retry, and -- against a deployment - -- can hand to the server as a [job](/basics/jobs) rather than waiting on. +| | Computed column | Materialized view | +| --- | --- | --- | +| Declaring or creating it | records the expression; no values are written | runs the query and stores its rows, unless created empty | +| A refresh | fills rows with no value and recomputes rows whose inputs changed | applies source changes or rebuilds the view, depending on the query and SDK version | +| A refresh when nothing changed | fills nothing | can be a no-op when incremental refresh is supported | +| In LanceDB Enterprise | runs as a [job](/basics/jobs) | runs as a server-side materialization job | -**Filling does not redo current work.** A refresh gives values to rows that -lack them and recomputes rows whose inputs changed since they were computed. It +### Computed columns fill incrementally {#incremental} + +A [computed column](/feature-engineering/add-computed-column) separates +**declaring** from **filling**: + +- Declaring is cheap and immediate. It records the expression in the schema + and commits a new table version, with no data written. It does not evaluate the expression over existing rows. +- Fill the column with a separate refresh call that you can retry. In LanceDB + Enterprise, submit the refresh as a [job](/basics/jobs) and track its progress. + +**Refresh new and changed rows.** A +[refresh](/feature-engineering/refresh-column) gives values to rows that lack +them and recomputes rows whose inputs changed since they were computed. It leaves the rest alone, so calling it twice is not twice the work, and appending rows means the next refresh handles only the new ones. -The corollary is worth stating plainly: **a filled value catches up with its -inputs only when you refresh.** If a column derives from `price` and a row's +**Computed values update when you refresh.** If a column derives from `price` and a row's `price` is later updated, the derived value keeps its old value until the next -refresh recomputes it. That is a deliberate trade -- it is what makes refreshes -idempotent and cheap -- but it will surprise you if you expect a spreadsheet. +refresh recomputes it. Schedule refreshes after changing the source data when consumers need +up-to-date features. -## Which one to reach for {#which-one} +### Materialized views store query results {#views} + +A [materialized view](/feature-engineering/materialized-views) has no separate +declaration step: creating one runs its query and stores the result, so the view +is ready to read. A refresh updates those stored results. Python SDK 0.40.0 can apply +changes incrementally for supported queries; other queries require a full rebuild. +See [refresh behavior](/feature-engineering/materialized-views#incremental) for +version-specific details. + +Enterprise runs materialization on remote compute. Use the SDK and refresh +options supported by your server version; see +[Enterprise refreshes](/feature-engineering/materialized-views#availability). + +## Choose a feature engineering workflow {#which-one} | | What it produces | Filled by | Available | | --- | --- | --- | --- | -| [Computed column](/feature-engineering/add-computed-column) | a new column on an existing table | a SQL expression | everywhere | -| [Materialized view](/feature-engineering/materialized-views) | a new table derived by a query | a refresh | everywhere | +| [Computed column](/feature-engineering/add-computed-column) | a new column on an existing table | a SQL expression | OSS and Enterprise | +| [Materialized view](/feature-engineering/materialized-views) | a new table derived by a query | a refresh | OSS and Enterprise | +| [Geneva](/geneva) | columns filled by Python | the `geneva` package | Enterprise | + +{/* +Draft for the Functions release. Replace the Geneva row of the table above with: | [Function](/management/function) | a computed column filled by Python | registered code | Enterprise | +| [Geneva](/geneva) | columns filled by Python, for existing pipelines | the `geneva` package | Enterprise | +*/} **Computed columns** are the common case: a column declared with a SQL expression, filled by [a refresh](/feature-engineering/refresh-column). @@ -57,23 +95,34 @@ expression, filled by [a refresh](/feature-engineering/refresh-column). aggregated view of a source -- and bring it up to date on each refresh. The result is an ordinary table: queryable, indexable, searchable. +**Geneva** is LanceDB Enterprise's Python package for work SQL cannot express: +embedding a column, calling a model, decoding an image. It defines features as +Python functions and backfills them as distributed jobs. + +{/* +Draft for the Functions release. Replaces the Geneva paragraph above: + **Functions** run Python where the data is, for work SQL cannot express: embedding a column, calling a model, decoding an image. Registered once, -versioned, then applied as a column. +versioned, then applied as a column. They are the way to do that on a +deployment; [Geneva](/geneva) remains available for existing pipelines. +*/} -They compose. A materialized view can select computed columns, and a computed -column can be filled by a registered function. +A materialized view can select computed columns. +{/* Draft for the Functions release: A computed column can be filled by a registered function. */} ## Where the work runs {#where} Enterprise -Locally, a refresh runs in your process. Against a deployment it runs across the +Locally, a refresh runs in your process. With LanceDB Enterprise it runs across the cluster, and the API reflects that: `refresh_column` raises, and -`refresh_column_async` returns a [job](/basics/jobs) handle instead. +`refresh_column_async` returns a [job](/basics/jobs) handle instead. Computed +columns are off by default in LanceDB Enterprise, and its administrator has to enable +them before a column can be declared there. ```python -job = table.refresh_column_async("embedding") +job = table.refresh_column_async("power_score") result = job.wait() result.rows_assigned ``` @@ -81,24 +130,28 @@ result.rows_assigned A backfill over a large table can therefore be started, left, and checked on from a different process -- the connection does not have to stay open. -That relocation is the main thing to understand about Enterprise feature -engineering. The API is the same; what changes is that the code runs somewhere -else, which is also why a function must declare the environment it needs. +Enterprise manages the compute for the refresh. You still control when to +submit it; declaring a computed column does not schedule recurring refreshes. +{/* Draft for the Functions release: That is also why a function must declare the environment it needs. */} ## Recovering from a failure {#failures} +{/* +Draft for the Functions release. + A function that raises fails the job. There is no per-row retry or skip policy in the function definition, so a function calling a flaky external service should handle that itself, inside the function body. +*/} -The alternative is to let the job fail, fix the cause, and run the refresh +When a refresh fails, fix the cause and run the refresh again. The new refresh fills what is still empty or stale. How much of the failed run was kept depends on where it ran (see [Refresh a column](/feature-engineering/refresh-column#failures)), and a retry can recompute values that were already written, so do not count on it costing only the rows that did not complete. -## Where to go next {#next} +## Related guides {#next} @@ -110,7 +163,10 @@ only the rows that did not complete. Bring a derived table up to date with its source. + {/* + Draft for the Functions release: Register Python to fill a column with. + */} diff --git a/docs/web/feature-engineering/materialized-views.mdx b/docs/web/feature-engineering/materialized-views.mdx index 7b00a6ce4..ab949adb1 100644 --- a/docs/web/feature-engineering/materialized-views.mdx +++ b/docs/web/feature-engineering/materialized-views.mdx @@ -1,9 +1,14 @@ --- title: "Refresh a materialized view" sidebarTitle: "Materialized View Refresh" -description: "A lakehouse object: a table defined by a query over another table, brought up to date by each refresh." +description: "Create a materialized view and refresh it when the source data changes." icon: table keywords: ["materialized view", "create_materialized_view", "refresh", "full refresh"] +availability: + oss: available + enterprise: available + summary: >- + Both offerings support materialized views. OSS refreshes run in your process; Enterprise runs the work on remote compute. See [Enterprise refreshes](#availability) for server compatibility requirements. --- import { @@ -11,12 +16,9 @@ import { PyMaterializedViewRefresh as MaterializedViewRefresh, } from '/snippets/materialized_view.mdx'; -A materialized view is a lakehouse object: a table whose contents are defined by -a query over another table. The query is recorded in the view's schema; -creating the view computes its rows, and a refresh brings them up to date. - -The result is an ordinary table. It can be queried, indexed, and searched, and -it appears in `table_names()` — nothing about reading it is special. +A materialized view stores the result of a query as a table. Use it to keep a +filtered or transformed dataset that you can query, index and search repeatedly. +Refresh the view after its source data changes. In SQL this is [`REFRESH MATERIALIZED VIEW`](/sql/refresh#refresh-materialized-view). @@ -36,89 +38,80 @@ reach the view through a refresh: {MaterializedViewRefresh} - -These results were verified with the development SDK 0.40.0b13, built from -lancedb main. The published 0.39.0 differs: the source must be created with -stable row ids (the `new_table_enable_stable_row_ids` storage option), -`create_materialized_view` creates the view empty and has no `with_no_data`, -and `refresh()` still accepts `full`. Its first `refresh()` writes every -matching row, two in this example. - +The example works with Python SDK 0.40.0. It creates the source with stable +row IDs, which that release requires for ordinary materialized-view sources. `select` takes column names, `(alias, expression)` pairs, or a dict of the same. Omitting it selects every source column, expanded against the schema as it was at creation time. `where` filters source rows; `limit` caps the view. -## Local sources no longer need stable row ids {#stable-row-ids} +## Prepare the source table {#stable-row-ids} -A local refresh does not track which source rows the view came from, so a local -view's source does not need stable row ids (`new_table_enable_stable_row_ids`): -the example above creates its source with the defaults. Earlier development -builds and the published 0.39.0 required them, and a deployment may still -require them; see [Against a deployment](#availability). +In Python SDK 0.40.0, create the source with +`storage_options={"new_table_enable_stable_row_ids": "true"}`, as the example +above does. Stable row IDs let the view track source rows across updates and +compaction. Set this option when creating the table; it cannot be added later. -The source is still checked, when the view is created and on every refresh: one -with an LSM write spec, or with retained write-ahead-log rows not yet compacted, -is refused. +A source with an LSM write specification or retained, uncompacted write-ahead-log +rows is not accepted by the local materialized-view implementation. -## Every local refresh rebuilds the view {#incremental} +## Refresh behavior {#incremental} -Locally, `refresh()` runs the view's query against the source and replaces -everything the view holds with the result, in one commit. It does not reconcile -changes: `rows_written` is the size of the whole result — two after adding Bea, -not one — and a refresh with nothing new to read still rewrites the view and -commits a new table version. +With Python SDK 0.40.0, a refresh applies source changes incrementally when the +query supports it, and rebuilds the view otherwise. In the example, adding Bea +writes one new view row. Refreshing again without source changes writes none. +Use `refresh(full=True)` to force a rebuild. -`source_version=N` refreshes against that source version instead of the latest, -which is how a view reproduces a known point in time. There is no `full` -argument; every local refresh is a full one. +`rows_written` counts rows written by that refresh, not all rows in the view. +Use `view.table.count_rows()` to get the total. Pass `source_version=N` to +refresh against a particular source version instead of the latest. -## Against a deployment {#availability} + +Refresh behavior has changed between SDK versions. The earlier 0.40.0b13 +development build rebuilt every local view on refresh and did not accept +`full`; 0.39.0 created views empty. Use the reference for your installed SDK +when maintaining an existing pipeline. + + +## Enterprise refreshes {#availability} Enterprise -`create_materialized_view` and `refresh` also work on a `db://` connection. -There the SDK sends them to the deployment as SQL -- `CREATE MATERIALIZED VIEW` -and `REFRESH MATERIALIZED VIEW` -- and the work runs on the cluster rather than -in your process. `source_version` is for local views only; the SDK refuses it -on a remote refresh. +Enterprise runs materialized-view creation and refresh on remote compute. +For SQL, see [`CREATE MATERIALIZED VIEW`](/sql/ddl/materialized-view) and +[`REFRESH MATERIALIZED VIEW`](/sql/dml/refresh#refresh-materialized-view). -Materialized views are off by default on a deployment and have to be enabled -there. How a deployment refreshes a view -- rebuilding the whole view, as above, -or reconciling what changed in the source -- depends on its LanceDB Enterprise -version and configuration, and so do the options -[`REFRESH MATERIALIZED VIEW`](/sql/dml/refresh#refresh-materialized-view) -accepts and whether the source table needs stable row ids. +Use an SDK version supported by your Enterprise deployment. Materialized-view +APIs and stored definitions have changed across releases; a local SDK example +alone does not establish compatibility with an older server. Before upgrading +an existing pipeline, confirm the supported refresh options and whether its +views need to be recreated with your administrator. -Stable row ids are set up on the deployment, not by the SDK: a `db://` -connection does not send `new_table_enable_stable_row_ids`, and no SDK call adds -them to an existing table. Depending on its version and configuration, a -deployment may give new tables stable row ids, may refuse a view created through -SQL -- as the SDK creates one -- over a source without them, or may convert an -existing source when a view is created through its REST API. Check with the -deployment's administrator what it requires before relying on a source. +Stable row IDs and feature configuration for Enterprise sources are managed on +the server. The source-table storage option in the local example does not +configure the Enterprise deployment, and Python SDK 0.40.0's `create_table` on a +`db://` connection does not accept `storage_options`: leave it out, and confirm +with your administrator that the deployment gives new tables stable row IDs. -`SHOW MATERIALIZED VIEWS` lists them, which is cheaper than -`list_materialized_views` -- that method finds views by reading every table's -schema, so it costs one open per table. +`list_materialized_views` requests the server's view listing. On a local +database it reads table schemas to find views. ## Working with the view {#working-with-the-view} `view.table` is the underlying table, for querying, indexing, or search. -`view.definition` is the recorded query — the select list, predicate, and -limit — which is how you check what a view actually contains without inferring -it from the rows. Both are properties; on an async view, the definition is read +`view.definition` is the recorded query: the select list, predicate, and +limit. Use it to inspect the query without inferring it from the rows. Both are properties; on an async view, the definition is read with `await view.definition()`. An existing view is reopened with `db.open_materialized_view(name)`. -## What to read next {#next} +## Related guides {#next} Derive a column instead of a whole table. - Every refresh commits a table version. + Inspect the table history and read a particular version. diff --git a/docs/web/feature-engineering/refresh-column.mdx b/docs/web/feature-engineering/refresh-column.mdx index d0f9f06a3..ae430e5f8 100644 --- a/docs/web/feature-engineering/refresh-column.mdx +++ b/docs/web/feature-engineering/refresh-column.mdx @@ -1,13 +1,19 @@ --- title: "Refresh a column" sidebarTitle: "Column Refresh" -description: "Fill the rows of a computed column that lack values — in process, or as a server-side job." +description: "Populate a computed column and update it when its inputs change." icon: arrows-rotate keywords: ["refresh_column", "backfill", "job", "idempotent", "REFRESH COLUMN"] +availability: + oss: available + enterprise: available + summary: >- + Both offerings support column refresh. The embedded library runs it in your process; Enterprise runs it as a server job that must be enabled for your Enterprise cluster. --- -A [declared column](/feature-engineering/add-computed-column) starts empty. A -refresh gives values to the rows that lack them. +Refresh a [computed column](/feature-engineering/add-computed-column) to fill +new rows and recompute values whose inputs changed. The example below continues +with the `doubled` column from the computed-column guide. ```python Python icon="python" @@ -37,8 +43,7 @@ The SQL form is [`REFRESH COLUMN`](/sql/refresh#refresh-column), with the same r A refresh fills rows that have no value and recomputes rows whose inputs changed since they were computed. Every other row is left alone, so a second -refresh with nothing changed does nothing. Two consequences follow, and both -are load-bearing. +refresh with nothing changed does nothing. This supports two common workflows. **Appending is cheap.** Rows added since the last refresh are filled by the next one; filled rows whose inputs are unchanged are skipped. Backfilling a @@ -63,16 +68,16 @@ REFRESH COLUMN embedding ON documents WHERE created_at > '2026-01-01' ``` -**Enterprise job type:** `computed_refresh` for a SQL-expression column, -`function_refresh` for one filled by a registered -[function](/management/function). +**Enterprise job type:** `computed_refresh` for a SQL-expression column. +{/* Draft for the Functions release: `function_refresh` for one filled by a registered [function](/management/function). */} -## As a job {#async} +## Run a refresh on Enterprise {#async} Enterprise -Against a deployment the refresh runs on the cluster. `refresh_column` raises +With an Enterprise cluster that enables computed columns, the refresh runs on the +cluster. `refresh_column` raises there; use `refresh_column_async`, which returns a [job](/basics/jobs) you can wait on or poll instead of holding a connection open: @@ -92,7 +97,7 @@ does not raise, which is what makes it safe in a loop; `wait()` raises A long backfill can therefore be started, left, and checked on from a different process. -### Resourcing the job {#resources} +### Configure workers and resources {#resources} SQL exposes the worker budget directly, which is the lever when a backfill is too slow or too expensive: @@ -111,7 +116,7 @@ rather than failing the job. What a failure keeps depends on where the refresh ran. `refresh_column` in your process writes the column in one commit, so a failure leaves the column as it -was. A refresh job on a deployment commits finished batches as it goes, so some +was. A refresh job in LanceDB Enterprise commits finished batches as it goes, so some values can remain after a failure. Either way, recover by fixing the cause and refreshing again. The new refresh @@ -121,12 +126,16 @@ that were already written -- for example when the record of the inputs they were computed from was not saved -- so do not count on a retry costing only the rows that did not complete. +{/* +Draft for the Functions release. + For a column filled by a [function](/management/function), the usual cause is the worker environment rather than the data. See [environment mismatches](/management/function#mismatch-symptoms) for the symptoms, which rarely name their cause. +*/} -## What to read next {#next} +## Related guides {#next} diff --git a/docs/web/governance/authorization.mdx b/docs/web/governance/authorization.mdx index 044db158e..9ef978c38 100644 --- a/docs/web/governance/authorization.mdx +++ b/docs/web/governance/authorization.mdx @@ -4,15 +4,13 @@ sidebarTitle: "Authorization" description: "Who may do what: privileges granted on catalog objects to principals, groups, roles, and API keys." icon: shield-halved keywords: ["authorization", "authz", "privileges", "rbac", "access control"] +availability: + oss: unavailable + enterprise: available + summary: >- + Enterprise supports catalog privileges when its administrator configures enforcement. Embedded LanceDB has no catalog authorization layer: access is whatever the object store allows. --- -Enterprise - -This page describes a LanceDB Enterprise capability. Embedded LanceDB has no -such layer -- access is whatever the object store enforces on the bucket or -directory, which is coarser and has no concept of a table. See -[Architecture](/architecture) for what differs between the two. - [Authentication](/enterprise/authentication) establishes who is calling. Authorization decides what they may do. @@ -28,15 +26,16 @@ A grant is held by one of four kinds of subject, distinguished by a prefix: | `p:` | a principal, by ID -- `P:` by name | | `g:` | a group, by ID -- `G:` by name | | `r:` | a role | -| `a:` | an API key | +| `a:` | the principal holding an API key, named by the key | Granting to a group or role rather than a principal is what keeps the entry count manageable: a principal's effective privileges are the union of what it holds directly and what it inherits. -Granting to an API key is narrower than granting to the principal that owns -it -- useful when a key is issued for one job and should not carry that -principal's full authority. +`P:`, `G:` and `a:` are ways of naming a subject when you make a request. The +entry is stored against the ID they resolve to, so a grant to `a:` belongs to +the principal holding that key and carries that principal's full authority, +not a narrower one. ## Objects {#objects} @@ -45,10 +44,28 @@ Objects are addressed by kind and path: ``` system database:db -namespace:db/ns -table:db/ns/table +namespace:db:ns +table:db:ns$table +view:db:ns$view ``` +{/* +Draft for the Functions release. Replaces the object kinds above: + +``` +system +database:db +namespace:db:ns +table:db:ns$table +view:db:ns$view +function:db:ns$function +secret:db:ns$secret +``` +*/} + +A nested namespace joins its path with `$`, as in `namespace:db:parent$child` +or `table:db:parent$child$table`. + The hierarchy is not decorative. Reaching a table requires `USAGE` on the namespace containing it and on the database containing that -- a grant on the table alone is not enough to traverse to it. @@ -57,10 +74,11 @@ table alone is not enough to traverse to it. | Privilege | Granted on | Allows | | --- | --- | --- | -| `OWNERSHIP` | any object | ownership, conveying full authority | +| `OWNERSHIP` | any object but `system` | ownership, conveying full authority | +| `USAGE` | system | listing jobs and their events, and looking up other identities | | `USAGE` | database | accessing the namespaces inside | | `USAGE` | namespace | accessing the tables inside | -| `SELECT` | table | reading rows | +| `SELECT` | table, view | reading rows | | `INSERT` | table | appending rows | | `UPDATE` | table | modifying existing rows | | `DELETE` | table | deleting rows | @@ -69,7 +87,19 @@ table alone is not enough to traverse to it. | `CREATE_TABLE` | namespace | creating a table in it | | `CREATE_MATERIALIZED_VIEW` | namespace | creating a materialized view in it | | `CREATE_VIEW` | namespace | creating a non-materialized view in it | -| `OPERATE` | system | reading server configuration | +| `OPERATE` | system | describing and controlling jobs, managing caches, and reading server configuration | + +{/* +Draft for the Functions release. Add these rows to the table above. +After the `USAGE` row for namespace: +| `USAGE` | function | binding it to a column, and refreshing a column it computes | +| `USAGE` | secret | binding and resolving it | +After the `UPDATE` row for table: +| `UPDATE` | secret | rotating it, without the right to bind or read it | +After the `CREATE_VIEW` row: +| `CREATE_FUNCTION` | namespace | registering a function in it | +| `CREATE_SECRET` | namespace | creating a secret in it | +*/} The four data privileges map onto the four [ingestion operations](/ingestion), so a principal that may append but not @@ -87,28 +117,32 @@ grant the other. A subtlety worth knowing, because it explains grants that appear to work on objects that do not exist yet. -When `USAGE` is checked on a database and **no entries exist for it at all**, -the database is assumed not to exist yet, and `CREATE_DATABASE` on `system` is -checked instead. The same applies to the default namespace: absent any -entries, the check falls back to `CREATE_NAMESPACE` on the containing -database. +When you create an object, the deployment checks the database and each +namespace that will contain it. One that **has no owner** -- because it does +not exist yet, or predates per-object authorization -- is treated as one you are +creating: the check falls back to `CREATE_DATABASE` on `system` for the +database, or to `CREATE_NAMESPACE` on its parent for a namespace, and you become +its owner. One that has an owner needs `USAGE` as usual. -So a principal holding `CREATE_DATABASE` can act on a database that has not -been created -- which is what makes creation possible at all, and why an empty -ACL set is not the same as a deny. +So a principal holding `CREATE_DATABASE` can create a table in a database that +has not been created -- which is what makes creation possible at all, and why +an empty ACL set is not the same as a deny. The fallback applies only when +creating; other operations on a database or namespace need `USAGE` on it. ## Listing grants {#listing} Entries can be listed filtered by object, by subject, or both: ``` -object = table:db/ns/documents +object = table:db:ns$documents subject = G:data-science ``` -Results paginate through a continuation token. One stored `(object, subject)` -tuple carrying several privileges expands into several entries in the -response, so the count reflects privileges rather than tuples. +Results paginate through a continuation token. A page holds at most 100 stored +`(object, subject)` tuples, and a tuple carrying several privileges expands +into several entries in the response, so a page can return more than 100 +entries: the count reflects privileges rather than tuples. Keep requesting with +the returned token until a page comes back without one. Listing by subject answers "what can this team reach"; listing by object answers "who can reach this table". Both are worth running before changing a diff --git a/docs/web/maintenance/background-jobs.mdx b/docs/web/maintenance/background-jobs.mdx index ba88b7137..151828e5d 100644 --- a/docs/web/maintenance/background-jobs.mdx +++ b/docs/web/maintenance/background-jobs.mdx @@ -1,22 +1,29 @@ --- title: "Background jobs" sidebarTitle: "Background Jobs" -description: "Jobs a deployment runs on its own — visible in SHOW JOBS, not started by a call you make." +description: "Understand the Enterprise background jobs that maintain tables, indexes and caches." icon: gears keywords: ["background jobs", "job type", "wal", "spfresh", "prewarm", "archive"] +availability: + oss: unavailable + enterprise: available + summary: >- + LanceDB Enterprise schedules background maintenance jobs according to your + cluster configuration. --- -Enterprise - -Every job documented elsewhere in these docs -- [Build & Repair](/indexing/reindexing), +The jobs documented elsewhere in these docs -- [Build & Repair](/indexing/reindexing), [Compaction](/maintenance/compaction), [Cleanup](/maintenance/cleanup), -[Column Refresh](/feature-engineering/refresh-column) -- is one you cause, -directly or by a schedule you set. This page is the rest: jobs the deployment -runs to keep itself correct and fast, which you may see named in -[`SHOW JOBS`](/sql/ddl/job#show) without having asked for any of them. +[Column Refresh](/feature-engineering/refresh-column) -- start from a call you +make or, for compaction, cleanup and index updates, from the deployment's own +maintenance. This page is the rest: jobs the deployment runs to keep itself +correct and fast, which you may see named in [`SHOW JOBS`](/sql/ddl/job#show) +without having asked for any of them. -There is no action to take on these beyond recognizing the name. None of them -has a parameter you set or a call you make. +These jobs are configured by the deployment's operators rather than through +the table SDK. If a maintenance job repeatedly fails or stops making progress, +share its job ID and errors with your Enterprise support team. Operators can +also start cache-warming jobs through the admin API. ## Write-ahead log maintenance {#wal} @@ -41,7 +48,7 @@ log from growing without bound: | --- | --- | | `prewarm_page_cache` | Loads table data into a query node's page cache ahead of a request needing it. | | `prewarm_index_cache` | The same, for index structures. | -| `startup_prewarm_idx` | Runs cache prewarming as part of a query node's own startup, rather than on a schedule. | +| `startup_prewarm_idx` | Runs cache prewarming as part of a plan executor's own startup, rather than on a schedule, when the deployment enables it. | ## Registry housekeeping {#registry} @@ -51,15 +58,16 @@ log from growing without bound: | `job_events_export` | Copies terminal jobs' event histories into a curated system table for metering. Registry-scoped, like the archive sweep. | These two are why an old job you completed weeks ago no longer shows in -`SHOW JOBS` -- it was archived, not deleted; its events survive in the exported -table. +`SHOW JOBS` -- it was archived, not deleted. Where the deployment enables the +export, its events also survive in the exported table. ## Object teardown {#teardown} | Job type | What it does | | --- | --- | -| `drop_object` | Deletes a dropped catalog object's content, then clears it from the database's cleanup backlog. Covers namespaces, views, and functions -- everything [`drop_table`](/management/table#drop) does not, since that one fences a directory with its own marker instead. | +| `drop_object` | Deletes a dropped catalog object's content, then clears it from the database's cleanup backlog. Whether a dropped table runs as this job or as [`drop_table`](/management/table#drop), which fences the table's directory with its own marker instead, depends on how the deployment stores its catalog. | -You trigger this indirectly, by dropping a [namespace](/management/namespace), -[view](/sql/ddl/view#drop), or [function](/management/function#drop) -- the teardown -itself runs as this job rather than inline with the call. +You trigger it indirectly, by dropping a table, a [view](/sql/ddl/view#drop) or +a [namespace](/management/namespace); the teardown runs as this job rather than +inline with the call. +{/* Draft for the Functions release: Dropping a [function](/management/function#drop) starts it the same way. */} diff --git a/docs/web/maintenance/cleanup.mdx b/docs/web/maintenance/cleanup.mdx index 71447a7f7..60a2f6bb0 100644 --- a/docs/web/maintenance/cleanup.mdx +++ b/docs/web/maintenance/cleanup.mdx @@ -4,6 +4,11 @@ sidebarTitle: "Cleanup" description: "Pruning old versions and the files only they referenced." icon: broom keywords: ["cleanup", "retention", "prune", "delete_unverified", "vacuum"] +availability: + oss: available + enterprise: available + summary: >- + Both offerings support cleanup. Embedded `optimize()` uses a retention window of 7 days by default; Enterprise uses configured background cleanup and retention, and does not support `optimize()`. --- Every write keeps the version before it, which is what makes @@ -14,7 +19,8 @@ those versions referenced. **Enterprise job type:** `cleanup`. Distinct from `compact` -- compaction merges files, cleanup removes them. `optimize()` in OSS runs both in one call; -Enterprise schedules them as separate jobs. +Enterprise schedules them as separate jobs, and runs cleanup only when the +deployment's configuration enables it. ## Retention {#retention} @@ -26,17 +32,43 @@ table.optimize(cleanup_older_than=timedelta(days=30)) ``` `cleanup_older_than` defaults to **7 days**. Versions older than that are -removed, along with the files only they referenced. +removed, along with the files only they referenced. In LanceDB Enterprise, the +retention period comes from its configuration instead. -The latest version is never removed, whatever you pass -- `timedelta(0)` prunes -everything except it. +The latest version is never removed, whatever you pass. `timedelta(0)` makes +every older version eligible for removal, but cleanup still keeps tagged +versions and the files branches still read, as the warning below describes. -Pruning is what makes [time travel](/management/table#versioning) finite. A -version outside the retention window is gone, and a tag or branch pointing at -it no longer resolves. +Pruning is what makes [time travel](/management/table#versioning) finite: an +eligible version outside the retention window is gone. Cleanup keeps a version, +whatever its age, when it is: -If you rely on reading old versions, set the window to cover the range you +- the **latest** version; +- a **tagged** version. + +Cleanup also keeps the files a **branch** still reads, and the version the +branch was forked from while the branch's remaining versions read the parent's +files. That does not keep the fork for good: once the branch has replaced the +data it inherited and its own older versions are pruned, cleanup prunes the +fork version like any other. To keep a version recoverable, tag it, subject to +the policy below, or keep the window wide enough to cover it. + +Under the default policy, a tag can also stop cleanup. When a tagged version is +older than the window and is not the latest, cleanup refuses to run: it raises +an error naming the tag and prunes nothing. To let it proceed, move the tag to a +version inside the window, delete the tag, or widen the window. + +A tag on the latest version does not stop cleanup. But `optimize()` compacts +before it prunes, and compacting commits a new version whenever it merges +files, so a tag on what was the latest version can become an older one by the +time cleanup runs. `optimize()` then raises after compacting, and its index +update does not run. + +A deployment's background cleanup follows the same rules. Its own background +jobs, such as compaction and index updates, also commit new versions, so a tag +on what is now the latest version can become an older one without a call from +you. If you rely on reading old versions, set the window to cover the range you need before running this, not after. @@ -46,8 +78,8 @@ Compaction writes new, merged files before the old ones are removed, so disk usage can rise before it falls. Space is reclaimed only when cleanup prunes the versions that referenced the superseded files. -If you need to reclaim space aggressively, shorten the retention window rather -than waiting for the default: +If you need to reclaim space aggressively in the embedded library, shorten the +retention window rather than waiting for the default: ```python table.optimize(cleanup_older_than=timedelta(days=1)) @@ -63,6 +95,9 @@ old, because it cannot tell them apart from a live write. table.optimize(delete_unverified=True) ``` +This option belongs to `optimize()` in the embedded library. A deployment's +cleanup jobs never set it. + `delete_unverified=True` removes them regardless of age. Only set it when you can guarantee **no other process is writing to the table** -- otherwise it can diff --git a/docs/web/maintenance/compaction.mdx b/docs/web/maintenance/compaction.mdx index b6697921c..757a344fc 100644 --- a/docs/web/maintenance/compaction.mdx +++ b/docs/web/maintenance/compaction.mdx @@ -3,18 +3,27 @@ title: "Compaction and cleanup" sidebarTitle: "Compaction" description: "Merge small files, prune old versions, and fold new rows into existing indexes." icon: broom +availability: + oss: available + enterprise: available + summary: >- + Tables are compacted in the embedded library and in LanceDB Enterprise. In the + embedded library you call `optimize()`; a deployment compacts tables in the + background, and `optimize()` is not available there. --- import { PyStorageRebuildVectorIndex } from '/snippets/doc_storage_examples.mdx'; Every write commits a new table version, and versions accumulate: small files from frequent appends, superseded files from updates and deletes, and rows added -since an index was built. `optimize()` addresses all three. +since an index was built. In the embedded library, `optimize()` addresses all +three; a deployment handles them in the background and does not support +`optimize()`. -It is modelled after `VACUUM` in PostgreSQL — routine housekeeping that a table -needs periodically rather than a repair for something broken. +Run optimization as regular table maintenance, particularly after many small +writes or a large ingest. -## What it does {#what-it-does} +## Optimize a table {#what-it-does} ```python table.optimize() @@ -60,7 +69,9 @@ and can help after a large ingest that changed the data distribution. ## When to run it {#when} -There is no background compactor: `optimize()` runs when you call it. +In the embedded library there is no background compactor: `optimize()` runs +when you call it. A deployment compacts a table automatically once enough small +fragments accumulate, unless its operator turns this off. Frequent small appends are what make it necessary, because each one commits files. A table written in a few large batches needs it rarely; one taking @@ -92,4 +103,4 @@ table.optimize() # 7-day default retention table.optimize(cleanup_older_than=timedelta(days=1)) # reclaim space sooner ``` -The best practice is to run `optimize()` after large writes or on a schedule. It also bundles incremental index updates — see [Build & Repair](/indexing/reindexing) for the breakdown. Updates also move rows out of the vector index, so they remain searchable but unindexed — rebuild the index after large update batches. +The best practice is to run `optimize()` after large writes or on a schedule. It also bundles incremental index updates. See [Build & Repair](/indexing/reindexing) for the breakdown. Updates also move rows out of the vector index, so they remain searchable but unindexed. Update or rebuild the index after large update batches. diff --git a/docs/web/management/database.mdx b/docs/web/management/database.mdx index 1f77e5b10..a4b76f195 100644 --- a/docs/web/management/database.mdx +++ b/docs/web/management/database.mdx @@ -1,97 +1,73 @@ --- title: "Database" sidebarTitle: "Database" -description: "Connecting to LanceDB: the URI forms, their options, and what each one changes." +description: "Connect to LanceDB and configure storage, credentials and read consistency." icon: database keywords: ["connect", "database", "uri", "api key", "storage options", "consistency"] +availability: + oss: available + enterprise: available + summary: >- + OSS opens tables in a local directory or object storage. Enterprise connects + to remote compute with a database URI and credentials. --- -A database is the root of a [catalog](/basics/catalog). A connection opens -exactly one, and every other object is reached through it. +import { PyConnectEnterprisePlaceholders, TsConnectEnterprisePlaceholders } from '/snippets/connection.mdx'; -Connecting is the one decision that reaches furthest: the URI you pass decides -where queries execute, which features exist, and what a table handle actually -is. +A database connection provides access to tables and namespaces in a +[catalog](/basics/catalog). Use it to create and open tables, list catalog +objects, and configure how your application reaches the data. -## The three URI forms {#uris} +## Connection URIs {#uris} + +| URI | Offering | Where queries run | +| --- | --- | --- | +| `./.lancedb` | OSS, with local storage | In your application | +| `s3://bucket/path`, `gs://bucket/path`, `az://container/path` | OSS, with object storage | In your application | +| `db://my-database` | Enterprise | On remote compute | + +[Connect to LanceDB](/basics/connect) has examples for each connection type. +A connection to object storage still uses the embedded library: your process +reads and writes the bucket directly. Enterprise runs operations remotely and +manages the supporting compute and maintenance. + +## Connect to Enterprise {#deployment} + +Replace the placeholders with the database URI, key, region and endpoint +provided by your administrator. + + + + {PyConnectEnterprisePlaceholders} + + + { "import * as lancedb from \"@lancedb/lancedb\";\n\n" } + {TsConnectEnterprisePlaceholders} + + + +| Python option | TypeScript option | Purpose | +| --- | --- | --- | +| `api_key` | `apiKey` | Authenticate the connection | +| `region` | `region` | Select the database's region | +| `host_override` | `hostOverride` | Set the Enterprise endpoint | +| `client_config` | `clientConfig` | Configure retries, timeouts and connection pooling. A retry can [repeat an append](/ingestion/insert#performance). | + +Use your deployment's region rather than relying on the `us-east-1` default. +For application code, load credentials from environment variables or a secrets +manager. See [Authentication](/enterprise/authentication) for OAuth and +[SQL](/basics/sql) for the separate SQL connection settings. + +## Storage options {#storage-options} + +For OSS tables in object storage, `storage_options` configures credentials, +endpoints and options for new tables. This example enables stable row IDs for +tables created through the connection: ```python Python icon="python" import lancedb -db = lancedb.connect("./.lancedb") # local directory -db = lancedb.connect("s3://bucket/path") # object storage -db = lancedb.connect("db://my-database", api_key="...") # a deployment -``` - -```typescript TypeScript icon="square-js" -import * as lancedb from "@lancedb/lancedb"; - -const db = await lancedb.connect("./.lancedb"); -const db = await lancedb.connect("s3://bucket/path"); -const db = await lancedb.connect("db://my-database", { apiKey: "..." }); -``` - - -The first two are **embedded**: your process opens Lance tables directly from -storage and does the work itself. The third talks to a **deployment**. - -| | Embedded | Deployment | -| --- | --- | --- | -| Where queries run | in your process | on the server | -| Table handle | `LanceTable` | `RemoteTable` | -| [SQL](/basics/sql) | -- | yes | -| Server-side [jobs](/basics/jobs) | -- | yes | -| [Functions](/management/function) | -- | yes | -| [Authorization](/governance/authorization) | storage permissions | granted per object | - -![](/static/assets/images/overview/understanding-tables.png) - -Both handles expose the same table API -- create, open, mutate, query -- so -application code moves between them by changing the URI. What differs is where -resolution and execution happen, and therefore which operations exist at all. - -## Connecting to a deployment {#deployment} - -Enterprise - - -```python Python icon="python" -db = lancedb.connect( - "db://my-database", - api_key="...", - region="us-east-1", -) -``` - -```typescript TypeScript icon="square-js" -const db = await lancedb.connect("db://my-database", { - apiKey: "...", - region: "us-east-1", -}); -``` - - -| Option | Meaning | -| --- | --- | -| `api_key` / `apiKey` | authenticates the connection -- see [Authentication](/enterprise/authentication) | -| `region` | which region hosts the database. Defaults to `us-east-1` | -| `host_override` / `hostOverride` | point at a specific endpoint instead of the regional one | -| `sql_host_override` | the same, for the [SQL endpoint](/sql#connecting). Python only | -| `client_config` / `clientConfig` | retries, timeouts, and connection pooling | - -An API key belongs in the environment, not in source. `region` is easy to get -wrong silently -- the default is `us-east-1` whether or not that is where your -database lives, and a wrong region fails as a connection error rather than a -helpful one. - -## Storage options {#storage-options} - -`storage_options` configures the underlying object store -- credentials, -endpoints, and defaults for tables created through this connection: - - -```python Python icon="python" db = lancedb.connect( "s3://bucket/path", storage_options={"new_table_enable_stable_row_ids": "true"}, @@ -99,95 +75,89 @@ db = lancedb.connect( ``` ```typescript TypeScript icon="square-js" +import * as lancedb from "@lancedb/lancedb"; + const db = await lancedb.connect("s3://bucket/path", { storageOptions: { new_table_enable_stable_row_ids: "true" }, }); ``` - -`new_table_enable_stable_row_ids` is worth setting deliberately: it gives stable -row ids to the tables this connection creates, and no SDK call adds them to an -existing table. A `db://` connection does not send it; a deployment decides -stable row ids itself (see -[Against a deployment](/feature-engineering/materialized-views#availability)). - +Set stable row IDs before creating a table that needs them, such as an ordinary +materialized-view source in Python SDK 0.40.0. The option does not change tables +that already exist. Enterprise storage settings are managed on the server. -See [Storage](/deployment/storage) for the full set, including per-cloud credentials. +See [Storage](/deployment/storage) for provider credentials and other options. ## Read consistency {#consistency} -`read_consistency_interval` controls how often a connection checks for versions -committed by other writers. +For OSS connections, `read_consistency_interval` controls how often reads check +for table versions committed by other writers. For example, check at most once +every five seconds: ```python Python icon="python" from datetime import timedelta +import lancedb -db = lancedb.connect(uri, read_consistency_interval=timedelta(seconds=5)) +db = lancedb.connect( + "./.lancedb", read_consistency_interval=timedelta(seconds=5) +) ``` ```typescript TypeScript icon="square-js" -const db = await lancedb.connect(uri, { readConsistencyInterval: 5 }); +import * as lancedb from "@lancedb/lancedb"; + +const db = await lancedb.connect("./.lancedb", { readConsistencyInterval: 5 }); ``` -The default reads a snapshot taken when the table was opened, so a long-lived -connection **will not see other processes' writes** until it reopens the table. -That is the right default for a batch job and the wrong one for a service. +By default, an OSS table handle does not automatically pick up another +process's writes. Reopen the table, check out its latest version, or configure +a consistency interval when you need those updates. A zero interval checks on +every read, with the cost of a metadata check. -Set it when several processes write the same table and readers must observe -those writes. The cost is a metadata check on that interval. See -[Consistency](/tables/consistency). +Enterprise manages read freshness on the server. See +[Consistency](/tables/consistency) for its behavior and the distinction between +seeing your own writes and seeing another client's writes. -## Database-level operations {#operations} +## Database operations {#operations} -| | Python | TypeScript | SQL | -| --- | --- | --- | --- | -| list tables | `table_names`, `list_tables` | `tableNames`, `listTables` | -- | -| the URI it opened | `uri` | `display()` | -- | -| drop every table | `drop_all_tables` | `dropAllTables` | -- | -| drop the database | `drop_database` | -- | -- | -| close | -- | `close`, `isOpen` | -- | +The connection provides methods to create, open, list and drop tables. - -```python Python icon="python" -db.table_names() -tbl = db.open_table("documents") -db.drop_table("documents") -``` +| Operation | Python | TypeScript | +| --- | --- | --- | +| Create a table | `create_table` | `createTable` | +| Open a table | `open_table` | `openTable` | +| List tables | `table_names`, `list_tables` | `tableNames`, `listTables` | +| Drop a table | `drop_table` | `dropTable` | -```typescript TypeScript icon="square-js" -await db.tableNames(); -const tbl = await db.openTable("documents"); -await db.dropTable("documents"); -``` - -```sql SQL -DROP TABLE documents; -``` - - -Each takes a `namespace_path` to act inside a [namespace](/management/namespace) rather -than at the root. +Use the namespace and pagination options documented in +[Namespaces](/management/namespace) when working outside the root namespace +or listing a large catalog. [Tables](/management/table) covers table schemas, +versions and lifecycle operations. -`drop_database()` removes every table in the database. There is no confirmation -and no undo -- table [versioning](/management/table#versioning) does not survive the table -being dropped. +Dropping a table removes its history as well as its current data. You cannot +recover a dropped table through table versioning. The OSS `drop_database()` +and `drop_all_tables()` operations remove all tables without a confirmation prompt. ## Async connections {#async} -`connect_async` returns an `AsyncConnection` with the same surface as -coroutines. The TypeScript SDK is async throughout, so its `connect` already is -one. +Python's `connect_async` returns an `AsyncConnection`. Await its operations +when working in an asynchronous application: ```python -db = await lancedb.connect_async("db://my-database", api_key="...") -tbl = await db.open_table("documents") +import lancedb + +db = await lancedb.connect_async("./.lancedb") +table = await db.open_table("documents") ``` -This is a different axis from the `_async` operations that return a -[job](/basics/jobs): `connect_async` is about not blocking your event loop, -while `create_index_async` is about work that outlives the call. +TypeScript's connection and table operations already return promises. + +An async connection and a background job serve different purposes. Async APIs +let other tasks run while your application waits for an operation. A +[job](/basics/jobs) represents work that can continue on the server after the +submission call returns. diff --git a/docs/web/management/function.mdx b/docs/web/management/function.mdx index 40e5bcd00..5734d2545 100644 --- a/docs/web/management/function.mdx +++ b/docs/web/management/function.mdx @@ -4,27 +4,31 @@ sidebarTitle: "Function" description: "Register a Python function once and apply it as a column, for work SQL cannot express." icon: code keywords: ["udf", "functions", "create_function", "embedding", "feature engineering"] +availability: + oss: unavailable + enterprise: available + summary: >- + Enterprise supports Functions when enabled for your Enterprise cluster; the embedded library cannot register or run them. --- import { PyFunctionDefine as FunctionDefine, PyFunctionLocalCall as FunctionLocalCall, - PyFunctionConnect as FunctionConnect, PyFunctionCreateTable as FunctionCreateTable, PyFunctionRegister as FunctionRegister, PyFunctionApply as FunctionApply, PyFunctionRefresh as FunctionRefresh, PyFunctionInspect as FunctionInspect, } from '/snippets/function.mdx'; - -Enterprise +import { PyConnectEnterprisePlaceholders as FunctionConnect } from '/snippets/connection.mdx'; Some columns cannot be written as a SQL expression: an embedding, a model call, an image decode. A function is a Python callable registered with the database, versioned, and then applied as a column — the code runs where the data is rather than pulling rows to your process. -Registering and running a function needs a LanceDB Enterprise connection. On a +Registering and running a function needs a LanceDB Enterprise deployment with +Functions enabled, which they are not by default. On a local connection `create_function` raises `NotImplementedError`; to derive a column locally, declare a [SQL computed column](/feature-engineering/add-computed-column) instead. @@ -33,6 +37,10 @@ column locally, declare a In SQL, a function is registered with [`CREATE FUNCTION`](/sql/ddl/function#create) and listed with [`SHOW FUNCTIONS`](/sql/ddl/function#show). +Functions are the way to fill columns with Python in LanceDB Enterprise. If you +already use [Geneva](/geneva), it remains available; see +[Geneva and Functions](/geneva#functions) for how its concepts map onto these. + The steps below take a column of numbers to a filled column of their doubles. ## Define a function {#define} @@ -62,9 +70,9 @@ sibling columns, so a null field is a value rather than a missing row. ## Connect and create a table {#table} -The remaining steps run against your Enterprise deployment. This connection -reads its settings from environment variables, so no credential appears in the -code: +The remaining steps run against your Enterprise deployment. Replace the database +URI, API key, region and endpoint below with your deployment details. In application +code, load credentials from environment variables or a secrets manager: {FunctionConnect} @@ -75,7 +83,7 @@ A nullable column requires a nullable parameter, such as `Optional[float]`; a non-nullable column can bind to either. This example uses `value: float`, so the table declares `value` non-nullable. -Computed-column inputs depend on the deployment version. Where supported, +Computed-column inputs depend on the Enterprise server version. Where supported, they bind to non-nullable parameters even though the physical column is nullable: refresh skips rows whose computed input is still null. Check your deployment's support before using another computed column as an input. @@ -212,6 +220,10 @@ the name was already gone. Deleting the object's content can finish after the call returns, as a [background job](/maintenance/background-jobs). `drop_function_async` returns that job with the result, so you can wait for it. +A deployment refuses the drop unless it also allows changes to a function's +lifecycle, which is a separate setting, off by default even where Functions are +enabled. + ## What to read next {#next} diff --git a/docs/web/management/index.mdx b/docs/web/management/index.mdx index 1d4d4eeba..b6bc07e88 100644 --- a/docs/web/management/index.mdx +++ b/docs/web/management/index.mdx @@ -6,17 +6,17 @@ icon: folder-tree keywords: ["ddl", "create", "drop", "list", "catalog", "management"] --- -Every [catalog object](/basics/catalog) is created, listed, and dropped the same -few ways. This section is the reference for those operations; the pages linked -from it cover what each object is for. +Use the SDKs or SQL to manage [catalog objects](/basics/catalog), including +tables, namespaces and views. The tables below provide a quick reference to +the available methods; each object's guide covers its setup and behavior. -Operations that can take a while have an `_async` form returning a -[job](/basics/jobs). +Some long-running operations also provide a submission method that returns a +[job](/basics/jobs). See the operation's guide for how to track completion. -## The surface at a glance {#surface} +## Management APIs {#surface} -SQL requires a deployment -- a local database has no engine to plan a statement. +SQL is available through the Enterprise Flight SQL endpoint. See [SQL](/basics/sql#sql-is-remote). @@ -59,8 +59,8 @@ See [Tables](/management/table) for creating one and shaping its schema. | drop | `drop_table` | `dropTable` | `DROP TABLE` | -`list_materialized_views` reads every table's schema to find them, so it costs -one open per table. `SHOW MATERIALIZED VIEWS` asks the server instead. +Local `list_materialized_views` reads table schemas to find views. On +Enterprise, it requests the server's view listing. See [Materialized views](/management/materialized-view) for creating one, and @@ -71,13 +71,17 @@ it current. | | Python | TypeScript | SQL | | --- | --- | --- | --- | -| create | -- | -- | `CREATE VIEW` | -| drop | -- | -- | `DROP VIEW` | +| create | `create_view` | `createView` | `CREATE VIEW` | +| list | `list_views` | `listViews` | -- | +| drop | `drop_view`, `drop_view_async` | `dropView`, `dropViewAsync` | `DROP VIEW` | -Enterprise Views exist only in SQL -- there is no SDK -method for them, because the thing a view stores *is* a SQL statement. See +Enterprise Views exist only on an Enterprise cluster that +enables them; the SDK methods send the view's SQL query to the deployment. See [Views](/management/view). +{/* +Draft for the Functions release. + ### Functions {#functions} | | Python | TypeScript | SQL | @@ -87,9 +91,11 @@ method for them, because the thing a view stores *is* a SQL statement. See | list | `list_functions` | -- | `SHOW FUNCTIONS` | | drop | `drop_function` | -- | -- | -Enterprise A function runs on the server, so there is -no local form. The TypeScript SDK has no function methods; use Python or SQL. +Enterprise A function runs on the server, and only on +a deployment that enables functions, so there is no local form. The TypeScript +SDK has no function methods; use Python or SQL. See [Functions](/management/function). +*/} ### Jobs {#jobs} @@ -119,7 +125,7 @@ names it. For a multi-column or named index, use the SDK. Indexes are a performance decision rather than a structural one, so they are covered under [Maintenance](/indexing). -## Namespaces qualify everything {#namespace-paths} +## Work within a namespace {#namespace-paths} Most connection-level operations take a `namespace_path` (`namespacePath`) to act inside a namespace rather than at the database root: @@ -131,8 +137,8 @@ tbl = db.open_table("clicks", namespace_path=["prod", "events"]) ``` ```typescript TypeScript icon="square-js" -await db.tableNames({ namespacePath: ["prod", "events"] }); -const tbl = await db.openTable("clicks", { namespacePath: ["prod", "events"] }); +await db.tableNames(["prod", "events"]); +const tbl = await db.openTable("clicks", ["prod", "events"]); ``` ```sql SQL @@ -142,7 +148,7 @@ SELECT * FROM prod.events.clicks In SQL the path is written as dotted identifiers instead. -## Dropping is not undoable {#dropping} +## Delete catalog objects {#dropping} `drop_table` removes the table and every version of it. Table diff --git a/docs/web/management/materialized-view.mdx b/docs/web/management/materialized-view.mdx index cbdb703b1..1475bd93a 100644 --- a/docs/web/management/materialized-view.mdx +++ b/docs/web/management/materialized-view.mdx @@ -4,6 +4,11 @@ sidebarTitle: "Materialized View" description: "Create, open, list and drop a table whose contents are defined by a query." icon: table keywords: ["materialized view", "create_materialized_view", "catalog", "ddl"] +availability: + oss: available + enterprise: available + summary: >- + Both offerings support materialized views. See [Enterprise compatibility](#deployment) for connection and server requirements. --- A materialized view is a table whose contents are defined by a query over @@ -20,7 +25,7 @@ This page covers creating and managing one. Keeping it current is In SQL, see [`REFRESH MATERIALIZED VIEW`](/sql/dml/refresh#refresh-materialized-view) and [`SHOW MATERIALIZED VIEWS`](/sql/ddl/materialized-view#show). -## Creating one {#create} +## Create a materialized view {#create} ```python Python icon="python" @@ -28,24 +33,24 @@ view = db.create_materialized_view( "recent_docs", source="documents", select=["id", "title", "embedding"], - where="created_at > '2026-01-01'", + where="created_at > TIMESTAMP '2026-01-01 00:00:00'", ) ``` ```typescript TypeScript icon="square-js" const view = await db.createMaterializedView("recent_docs", "documents", { select: ["id", "title", "embedding"], - where: "created_at > '2026-01-01'", + where: "created_at > TIMESTAMP '2026-01-01 00:00:00'", }); ``` -`source` names the table it derives from. A local source does not need stable -row ids or any other storage option; a deployment may require stable row ids -(see [Against a deployment](#deployment)). `select`, `where` and `limit` record -the query -- there is no separate definition step. +`source` names the table the view reads. With Python SDK 0.40.0, create that +table with stable row IDs before creating the view. The +[complete example](/feature-engineering/materialized-views#create-and-refresh) +shows the setup. `select`, `where` and `limit` define the query. -## Opening, listing and dropping {#manage} +## Open, list and drop views {#manage} | | Python | TypeScript | SQL | | --- | --- | --- | --- | @@ -56,34 +61,30 @@ the query -- there is no separate definition step. | drop | `drop_table` | `dropTable` | `DROP TABLE` | -`list_materialized_views` finds views by reading every table's schema, so it -costs one open per table. `SHOW MATERIALIZED VIEWS` asks the server instead and -is cheaper on a catalog with many tables. +On a local database, `list_materialized_views` reads table schemas to find +views. On Enterprise, it asks the server for the view listing, as +`SHOW MATERIALIZED VIEWS` does. A view is dropped with `drop_table`, because a materialized view **is** a table. That also means dropping one removes stored rows -- unlike dropping a [view](/management/view), which removes only a saved query. -## Working with the view {#working} +## Read the view and its definition {#working} `view.table` is the underlying table, for querying, indexing, or search. -`view.definition` is the recorded query -- select list, predicate, limit --- which is how you check what a view contains without inferring it from the -rows. Both are properties; on an async view, the definition is read with +`view.definition` contains the query's select list, predicate and limit. Both are properties; on an async view, the definition is read with `await view.definition()`. -## Against a deployment {#deployment} +## Enterprise compatibility {#deployment} -Enterprise `create_materialized_view` works on a -`db://` connection as well as a local one. There the SDK creates and refreshes -the view through SQL, and the work runs on the cluster rather than in your -process. Materialized views are off by default on a deployment and have to be -enabled there, and a deployment may require a view's source to have stable row -ids; see -[Against a deployment](/feature-engineering/materialized-views#availability). +Enterprise supports creating and refreshing materialized views on remote +compute. Use the SDK version supported by your deployment, or the +[SQL interface](/sql/ddl/materialized-view). See +[Enterprise refreshes](/feature-engineering/materialized-views#availability) +for compatibility considerations when upgrading an existing pipeline. -## What to read next {#next} +## Related guides {#next} diff --git a/docs/web/management/namespace.mdx b/docs/web/management/namespace.mdx index f1310b6da..a3485f46d 100644 --- a/docs/web/management/namespace.mdx +++ b/docs/web/management/namespace.mdx @@ -4,6 +4,12 @@ sidebarTitle: "Namespace" description: "Grouping tables into a hierarchy, and addressing them through it." icon: folder-tree keywords: ["namespace", "catalog", "hierarchy", "directory namespace", "rest namespace"] +availability: + oss: available + enterprise: available + summary: >- + Namespaces work in the embedded library, through directory or REST + catalogs, and in LanceDB Enterprise. --- import { @@ -15,8 +21,10 @@ import { RsNamespaceTableOps, } from '/snippets/connection.mdx'; -Despite its name, LanceDB is not a "database" in the traditional sense -- it is a **Multimodal Lakehouse** that builds on the table abstraction, -similar to many other lakehouse projects. LanceDB exposes a catalog-level abstraction over the Lance table format, via a *namespace spec*. +LanceDB organizes Lance tables through a catalog-level abstraction defined by +the *namespace spec*. LanceDB OSS is an embedded library for accessing those +tables, and LanceDB Enterprise is a multimodal lakehouse that provides shared +compute and catalog services on top of the Lance format. If you're coming from traditional databases or lakehouses, you can think of a namespace as the catalog path that says where a table name lives. Lance provides the **file** and **table** formats to store and manage your data and indexes. @@ -29,7 +37,7 @@ language: Python uses `namespace_path`, Rust uses builder methods like `.namespa TypeScript uses `namespacePath` arguments. -In SQL a namespaced table is addressed as a [dotted path](/sql/dql/select#namespaces). +In LanceDB Enterprise SQL, a namespaced table is addressed as a [dotted path](/sql/dql/select#namespaces). ## Namespace hierarchy {#namespace-hierarchy} @@ -47,14 +55,15 @@ If you want to go deeper, see the Lance format [namespace documentation](https:/ A namespace path is a list of components. For example, `["prod", "search"]` means the `search` namespace inside the `prod` namespace. The empty path, `[]`, means the root namespace. -Each component is a name, not a filesystem path segment. Namespace names can't be empty, and each -component can contain only letters, numbers, underscores, hyphens, and periods. That keeps the same -identifier usable across local directory namespaces and REST namespace identifiers. +Each component is a name, not a filesystem path segment, and can't be empty. Keep each one to +letters, numbers, underscores, hyphens, and periods, so the same identifier is usable everywhere: a +local directory namespace accepts other characters too, but other namespace implementations and +deployments can refuse them. [Naming rules](/basics/naming) has the lengths and the other limits. ## Directory namespaces {#directory-namespaces} The simplest namespace model in LanceDB is a single root namespace, often represented by one -directory: +directory. Namespaces can nest inside it, as in this hierarchy: ```bash ./local_lancedb (root) @@ -64,7 +73,12 @@ directory: └── data (table) ``` -As a user of LanceDB OSS, you might never notice namespaces at first, because LanceDB exposes the single-level hierarchy shown above, with the data stored in the `data/` directory, where the root namespace is implicit. Connecting to this namespace is as simple as connecting to the catalog root: +The hierarchy is logical, not a tree of folders: a directory namespace records it in a +`__manifest` table in the root directory, and stores each table in a directory of its own there. + +As a user of LanceDB OSS, you might never notice namespaces at first, because a plain connection +uses only the root namespace, which is implicit. Connecting to this namespace is as simple as +connecting to the catalog root: ```python Python icon="python" @@ -89,7 +103,7 @@ let db = connect("./local_lancedb").execute().await?; ``` -This creates the default namespace directory (`data/`) under the specified root path. +A table created through this connection is stored directly under that path, as `.lance`. You can also explicitly connect to a namespace using `lancedb.connect_namespace(...)` with the directory namespace implementation: @@ -152,7 +166,7 @@ println!("Created table: {}", table.name()); - For simple use cases in LanceDB OSS, you don't need to go too deep into namespaces. -- To integrate LanceDB with external catalogs and to use it as a true **multimodal lakehouse**, it's useful to understand the different namespace implementations and how to use them in your organization's setup. +- To integrate LanceDB with external catalogs, understand how each namespace implementation resolves table names and locations in your organization's setup. ## Remote or external catalog namespaces {#remote-or-external-catalog-namespaces} @@ -169,9 +183,11 @@ LanceDB Enterprise REST requests use the `x-api-key` header for API-key authenti that route multiple databases through the same endpoint can also use headers such as `x-lancedb-database` or `x-lancedb-database-prefix` for database context. +Replace the catalog URL and credential placeholders below with your values. In +application code, load credentials from environment variables or a secrets manager. + ```python Python icon="python" -import os import lancedb # Remote namespace-backed catalog root (RestNamespace) @@ -180,9 +196,9 @@ db = lancedb.connect_namespace( "rest", { "uri": "https://.internal..com", - "headers.x-api-key": os.environ["API_KEY"], + "headers.x-api-key": "your-api-key", # or: - # "headers.Authorization": f"Bearer {os.environ['REST_AUTH_TOKEN']}", + # "headers.Authorization": "Bearer your-access-token", }, ) ``` @@ -195,9 +211,9 @@ import * as lancedb from "@lancedb/lancedb"; const db = await lancedb.connectNamespace("rest", { uri: "https://.internal..com", headers: { - "x-api-key": process.env.API_KEY ?? "", + "x-api-key": "your-api-key", // or: - // Authorization: `Bearer ${process.env.REST_AUTH_TOKEN}`, + // Authorization: "Bearer your-access-token", }, }); ``` @@ -214,12 +230,12 @@ properties.insert( ); properties.insert( "headers.x-api-key".to_string(), - std::env::var("API_KEY")?, + "your-api-key".to_string(), ); // or: // properties.insert( // "headers.Authorization".to_string(), -// format!("Bearer {}", std::env::var("REST_AUTH_TOKEN")?), +// "Bearer your-access-token".to_string(), // ); let db = lancedb::connect_namespace("rest", properties) .execute() @@ -227,7 +243,8 @@ let db = lancedb::connect_namespace("rest", properties) ``` [LanceDB Enterprise](/enterprise) operates a REST namespace server on top of the Lance format, so any REST client that can speak the REST namespace API -contract can be used to interact with it. For authentication examples in LanceDB Enterprise, visit +contract can be used to interact with it. It does not serve the contract's `exists` routes, so checks such as `namespace_exists` +fail against it. For authentication examples in LanceDB Enterprise, visit the [Namespaces in SDKs](#namespaces-in-lancedb-enterprise) page. ## Best practices {#best-practices} @@ -279,8 +296,8 @@ The SDK methods expose the namespace path in the idiom of each language: Using namespaces is **optional** in LanceDB, and most basic use cases do not require to work with them. -An empty namespace (`[]`), which is the default, means "root namespace", and the data will be stored in -the `data/` directory under the specified root path. +An empty namespace (`[]`), which is the default, means the root namespace, whose tables are stored +directly under the specified root path. ## Namespace management APIs {#namespace-management-apis} @@ -315,12 +332,13 @@ doesn't exist, or contains data: - Drop mode: `fail` reports an error when the namespace doesn't exist, and `skip` treats a missing namespace as a successful no-op. - Drop behavior: `restrict` keeps non-empty namespaces from being dropped, and `cascade` drops child namespaces and tables first. -Namespace path components can't be empty. Each component can contain only letters, numbers, -underscores, hyphens, and periods. +Namespace path components can't be empty, and are best kept to letters, numbers, underscores, +hyphens, and periods, as [above](#namespace-paths-and-names). Listing APIs return the immediate children of the requested namespace path. Use `limit` with the returned `page_token` to page through large catalogs; pass an empty namespace path (`[]`) when you -want to list from the root namespace. +want to list from the root namespace. In Python, a listing returns a response: the names are in its +`namespaces` or `tables`, and `page_token` is set when there is another page. ## Namespaces in LanceDB Enterprise {#namespaces-in-lancedb-enterprise} diff --git a/docs/web/management/table.mdx b/docs/web/management/table.mdx index 447b59029..1b8ede070 100644 --- a/docs/web/management/table.mdx +++ b/docs/web/management/table.mdx @@ -4,6 +4,13 @@ sidebarTitle: "Table" description: "The object most operations act on: creating, opening, renaming, cloning and dropping one." icon: table keywords: ["table", "create_table", "open_table", "clone", "rename", "drop"] +availability: + oss: available + enterprise: available + summary: >- + Tables are created, evolved, versioned and dropped in the embedded library + and in LanceDB Enterprise. Renaming, cloning, dropping every table and pruning old + versions differ between the two. --- import { @@ -113,18 +120,20 @@ import { PyQueryToPandasKwargs as QueryToPandasKwargs, } from '/snippets/multimodal.mdx'; -A table is rows and columns, versioned. It is what searches read, what writes -commit to, and what indexes and branches belong to. +A table stores data in rows and columns and keeps a history of committed +versions. Queries read table data, writes update it, and indexes accelerate +searches over its columns. Branches let you make changes separately from the +main table history. -This page covers the table as a catalog object. Getting data into one is -[Ingest](/ingestion); changing its shape is -[Schema evolution](/management/table#schema). +This page covers table management, data types, schema changes and version +history. For loading records, see [Ingestion](/ingestion). For changes to +columns, see [Schema evolution](/management/table#schema). -Tables are created and dropped in SQL with [`CREATE TABLE` and `DROP TABLE`](/sql/ddl/table). +In LanceDB Enterprise, tables are also created and dropped in SQL with [`CREATE TABLE` and `DROP TABLE`](/sql/ddl/table). -## What a table carries {#carries} +## Table structure and metadata {#carries} | | | | --- | --- | @@ -133,7 +142,7 @@ Tables are created and dropped in SQL with [`CREATE TABLE` and `DROP TABLE`](/sq | **Indexes** | vector, full-text and scalar, built over its columns | | **Branches** | isolated lines of change you can [merge or discard](/tables/branching) | -## Creating {#create} +## Create a table {#create} ```python Python icon="python" @@ -156,19 +165,18 @@ frame, or Pydantic models. [Ingesting data](/ingestion) covers each, plus `exist_ok` and `mode="overwrite"` for re-runnable scripts. -An explicit schema is where nullability is decided. Creation is also the only -point at which the SDK can give a table **stable row ids**, which a deployment -may require on a [materialized view](/management/materialized-view) source. See -[Limits](/basics/limits#schema). +An explicit schema is where nullability is decided. In the embedded library, +creation is also the only point at which a table can be given **stable row +ids**; in LanceDB Enterprise, whether a new table gets them is the deployment's +setting. See [Limits](/basics/limits#schema). -## Opening and listing {#open} +## Open and list tables {#open} ```python Python icon="python" tbl = db.open_table("documents") db.table_names() -db.table_exists("documents") ``` ```typescript TypeScript icon="square-js" @@ -179,9 +187,18 @@ await db.tableNames(); Each accepts `namespace_path` to act inside a [namespace](/management/namespace) -rather than at the database root. TypeScript has no `tableExists`. +rather than at the database root. -## Renaming and cloning {#rename-clone} +To find out whether a table exists, open it and handle the error, or look for +its name in a listing. A missing table raises `ValueError` on a local directory +or a deployment, and `lance_namespace.TableNotFoundError` on a connection from +`connect_namespace`. A listing returns one page of names at a time, so check +every page. Python's `table_exists` is no substitute: a local directory or a +deployment raises `NotImplementedError` for it, and on a connection from +`connect_namespace("dir", ...)` or `connect_namespace("rest", ...)` it fails with +an `AttributeError`. TypeScript has no `tableExists`. + +## Rename and clone tables {#rename-clone} ```python db.rename_table("documents", "docs") @@ -194,6 +211,10 @@ db.clone_table( ) ``` +Renaming is not supported on a local directory connection. In LanceDB Enterprise, +whether a table can be renamed or cloned at all depends on how the deployment +stores its catalog. + A **shallow** clone -- the default -- references the source's data files rather than copying them, so it is fast and cheap regardless of table size. It is the practical way to branch an experiment off a known version, or to pin a training @@ -204,18 +225,20 @@ A shallow clone depends on the source's files continuing to exist. If the source is [compacted](/maintenance/compaction) past the retention window, the version the clone points at can be pruned. Local OSS connections support shallow clones only; `is_shallow=False` is not implemented. If the copy must outlive the -source's history, copy its data into an independent table instead. For remote -connections, confirm which clone modes your deployment supports. +source's history, copy its data into an independent table instead. A deployment +that supports cloning also makes shallow clones only, and one whose catalog +does not support it refuses every clone. -**Enterprise job type:** `drop_table`. Fences the table's directory with a -marker and deletes its content in the background -- see -[Background jobs](/maintenance/background-jobs#teardown) for the namespace, -view, and function equivalent. +**Enterprise job type:** on an Enterprise cluster that drops tables in the background, +the teardown runs as a job: `drop_table`, which fences the table's directory +with a marker and deletes its content, or `drop_object`, depending on how the +deployment stores its catalog. See +[Background jobs](/maintenance/background-jobs#teardown). -## Dropping {#drop} +## Drop a table {#drop} ```python Python icon="python" @@ -238,13 +261,16 @@ DROP TABLE documents Dropping removes the table and **every version of it**. [Versioning](/management/table#versioning) does not survive a drop, and neither -`drop_all_tables` nor `drop_database` asks for confirmation. +`drop_all_tables` nor `drop_database` asks for confirmation. Both work only on +a local database; on an Enterprise connection the SDK raises instead. On a large table the drop can take a while, so `drop_table_async` returns a -[job](/basics/jobs) rather than blocking. +[job](/basics/jobs) rather than blocking when the deployment drops tables in the +background. In the embedded library, and on an Enterprise cluster that drops inline, the +returned job has already finished. -## The full surface {#surface} +## Table management APIs {#surface} | | Python | TypeScript | SQL | | --- | --- | --- | --- | @@ -456,6 +482,8 @@ When you call `to_pandas()` on a local LanceDB table that contains Blob API colu - `"bytes"`: eagerly materializes each blob as `bytes`. Use this when you need the raw payload in the DataFrame, for example to decode an image or audio clip in-process. - `"descriptions"`: returns blob descriptors (offsets, sizes, and positions) instead of the data itself. Use this when you want to plan I/O without paying the cost of loading every blob. +`"bytes"` is not supported on in-memory tables. + Extra keyword arguments are forwarded to the underlying PyArrow / Lance pandas conversion, so you can also pass options like `split_blocks` or `self_destruct`: @@ -464,13 +492,10 @@ Extra keyword arguments are forwarded to the underlying PyArrow / Lance pandas c -Query builders also accept `blob_mode` on `to_pandas()`, `to_arrow()`, and `to_list()`: +Query builders also accept `blob_mode` on their `to_pandas()` method: - Plain scans support `"lazy"`, `"bytes"`, and `"descriptions"` with filters, projections, aliases, `limit`, and `offset`. -- Vector, FTS, hybrid, and take queries support `"bytes"` and `"descriptions"`. Lazy `BlobFile` handles need a plain scan. -- In `to_arrow()` and `to_list()`, `"lazy"` returns blob descriptors that carry the row id `fetch_blobs` reads, and `"descriptions"` returns them without it. -- A plain scan's `to_pandas()` without `timeout` reads the local dataset through Lance's scanner, so it also returns `"bytes"` for legacy blob columns and under namespace query pushdown. In-memory tables have no local dataset, so they take the `to_arrow()` route. -- Every other `"bytes"` route raises `ValueError` for legacy blob columns and blobs inside lists, and `NotImplementedError` on remote tables and namespace query pushdown, where the blob fetch could read a different table version than the query. +- Vector, FTS, hybrid, and ordered queries can't materialize blob columns through `to_pandas()`; omit blob columns from the projection for those query shapes. - This works on both sync and async query builders. Extra PyArrow kwargs like `split_blocks` and `self_destruct` are still forwarded. @@ -880,8 +905,13 @@ Remove several columns at once for efficiency: - -Dropping columns cannot be undone. Make sure you have backups or are certain before removing columns. + +Dropping a column changes the latest version only. Until +[cleanup](/maintenance/cleanup) prunes the versions from before the drop, +[restoring](#versioning) one of them brings the column and its values back -- +along with the rest of the table as it was then. Once those versions are +pruned, the drop cannot be undone, so check before dropping a column you may +need. ## Versioning {#versioning} @@ -1041,8 +1071,9 @@ Numeric table versions like `v3` or `v17` are precise but hard to remember. Tags let you attach human-readable labels (e.g., `"prod"`, `"baseline"`, `"q3-evaluation"`) to specific versions and check those out by name. They are conceptually similar to git tags, and unlike numeric versions, **tagged versions -are preserved when old versions are pruned** (see the cleanup note at the bottom -of this page). +are never pruned** -- a tagged version that is older than the retention window and +is not the latest stops cleanup instead (see the cleanup note at the bottom of this +page). The tags API supports the standard CRUD operations — create, list, update, delete — plus checking out by tag name. @@ -1138,10 +1169,17 @@ mixing in lower-level Lance file operations. **System Operations** System operations like `optimize()`, index updates, and table compaction also increment table version numbers. -In LanceDB OSS and Enterprise, `optimize()` can prune older versions based on its retention setting (`cleanup_older_than`, 7 days by default), -which is when old-version files are removed and disk space is reclaimed. +In LanceDB OSS, `optimize()` can prune older versions based on its retention setting (`cleanup_older_than`, 7 days by default), +which is when old-version files are removed and disk space is reclaimed. A deployment prunes them in the background +when its configuration enables [cleanup](/maintenance/cleanup); `optimize()` is not available there. -**Tagged versions are exempt from cleanup.** A version with a tag pointing at it is -retained regardless of age, and its files are not removed by `optimize()`. To make -a tagged version eligible for pruning, [delete the tag](#versioning-tag-based-versioning) first. +**Tagged versions are never pruned.** A version with a tag pointing at it is +retained regardless of age. When a tagged version is older than the retention window and +is not the latest, cleanup refuses to run instead of pruning around it: `optimize()` raises +an error naming the tag and prunes nothing. A tag on the latest version does not cause +this, but `optimize()` compacts before it prunes, and compacting commits a new version +whenever it merges files, so a tag on what was the latest version can become an older +one by the time cleanup runs; the error then comes after compacting, and the index +update does not run. To let cleanup proceed, [move the tag](#versioning-tag-based-versioning) +to a version inside the retention window, delete it, or widen the window. diff --git a/docs/web/management/view.mdx b/docs/web/management/view.mdx index bfb3a0543..de6cd38a5 100644 --- a/docs/web/management/view.mdx +++ b/docs/web/management/view.mdx @@ -4,29 +4,31 @@ sidebarTitle: "View" description: "A saved query, evaluated when read. The non-materialized counterpart to a materialized view." icon: eye keywords: ["view", "create view", "sql", "catalog"] +availability: + oss: unavailable + enterprise: available + summary: >- + Enterprise supports views when enabled for your Enterprise cluster; the embedded library cannot create them. --- -Enterprise - -This page describes a LanceDB Enterprise capability. The embedded form -of LanceDB has no server to provide it -- see -[Architecture](/architecture) for what differs between the two. - -A view stores a **query**, not rows. Reading it runs the query against current -data, so a view is never stale -- and never faster than the query it wraps. +A view saves a query that you can refer to by name. Reading the view executes +that query against the source tables, using their read-consistency behavior. +Views do not store a separate copy of the result. It is the counterpart to a [materialized view](/management/materialized-view), which stores rows and is brought up to date by a refresh. Both are [catalog objects](/basics/catalog) -defined by a query; the difference is entirely when that query runs. +defined by a query; choose a materialized view when you need to store and reuse the result. | | View | Materialized view | | --- | --- | --- | | Stores | the query | the rows | -| Freshness | always current | as of the last refresh | -| Read cost | the full query, every read | a table scan | +| Freshness | follows source-table reads | as of the last refresh | +| Read cost | executes the saved query | queries the stored result | | Write cost | none | a refresh | -Views are created in SQL, over the [Flight SQL endpoint](/sql#connecting) -- -there is no SDK method, because the thing a view stores *is* a SQL statement. -See [View](/sql/ddl/view) for `CREATE VIEW`, `DROP VIEW`, and listing. +Views are created in LanceDB Enterprise, either in SQL over the +[Flight SQL endpoint](/sql#connecting) or with the SDKs' `create_view` / +`createView`, which send the query to the deployment. Either way, the deployment +must have views enabled. See [View](/sql/ddl/view) for `CREATE VIEW`, +`DROP VIEW`, and listing. diff --git a/docs/web/snippets/curation_sampling.mdx b/docs/web/snippets/curation_sampling.mdx new file mode 100644 index 000000000..9c3d482b8 --- /dev/null +++ b/docs/web/snippets/curation_sampling.mdx @@ -0,0 +1,4 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const PySampleDataset = "from tempfile import TemporaryDirectory\n\nimport lancedb\nfrom lancedb.permutation import Permutation, permutation_builder\n\nwith TemporaryDirectory(prefix=\"lancedb-sampling-\") as uri:\n db = lancedb.connect(uri)\n table = db.create_table(\"records\", [{\"id\": i} for i in range(10)])\n splits = (\n permutation_builder(table).split_random(counts=[3, 7], seed=42).execute()\n )\n sample = Permutation.from_tables(table, splits, split=0).with_format(\"arrow\")\n selected_ids = []\n for batch in sample.iter(batch_size=3):\n selected_ids.extend(batch.column(\"id\").to_pylist())\n print(f\"Selected {len(selected_ids)} rows: {selected_ids}\")\n"; + diff --git a/docs/web/snippets/embedding.mdx b/docs/web/snippets/embedding.mdx index 8db4472dc..da3388ea7 100644 --- a/docs/web/snippets/embedding.mdx +++ b/docs/web/snippets/embedding.mdx @@ -34,13 +34,13 @@ export const TsRegisterModelFallback = "const registry = getRegistry();\nregistr export const TsRegisterSecret = "const registry = getRegistry();\nregistry.setVar(\"api_key\", \"sk-...\");\n\nconst func = registry.get(\"openai\")!.create({\n apiKey: \"$var:api_key\",\n});\n"; -export const RsCreateEmbeddingFunction = "use std::sync::Arc;\n\nuse lancedb::embeddings::openai::OpenAIEmbeddingFunction;\n\nlet api_key = std::env::var(\"OPENAI_API_KEY\").expect(\"OPENAI_API_KEY is not set\");\nlet embedding = Arc::new(\n OpenAIEmbeddingFunction::new_with_model(api_key, \"text-embedding-3-small\")\n .expect(\"failed to create OpenAI embedding function\"),\n);\n"; +export const RsCreateEmbeddingFunction = "use std::sync::Arc;\n\nuse lancedb::embeddings::openai::OpenAIEmbeddingFunction;\n\nlet api_key = \"your-openai-api-key\".to_string();\nlet embedding = Arc::new(\n OpenAIEmbeddingFunction::new_with_model(api_key, \"text-embedding-3-small\")\n .expect(\"failed to create OpenAI embedding function\"),\n);\n"; export const RsEmbeddingFunction = "use std::{borrow::Cow, sync::Arc};\n\nuse arrow_array::{Array, FixedSizeListArray, Float32Array};\nuse arrow_schema::{DataType, Field, Schema};\nuse lancedb::{\n connect,\n embeddings::{EmbeddingDefinition, EmbeddingFunction},\n Result,\n};\n\n#[derive(Debug, Clone)]\nstruct MyTextEmbedder {\n dim: usize,\n}\n\nimpl EmbeddingFunction for MyTextEmbedder {\n fn name(&self) -> &str {\n \"my-embedder\"\n }\n\n fn source_type(&self) -> Result> {\n Ok(Cow::Owned(DataType::Utf8))\n }\n\n fn dest_type(&self) -> Result> {\n Ok(Cow::Owned(DataType::new_fixed_size_list(\n DataType::Float32,\n self.dim as i32,\n true,\n )))\n }\n\n fn compute_source_embeddings(&self, source: Arc) -> Result> {\n let values = Arc::new(Float32Array::from(vec![1.0f32; source.len() * self.dim]));\n let field = Arc::new(Field::new(\"item\", DataType::Float32, true));\n Ok(Arc::new(FixedSizeListArray::new(\n field,\n self.dim as i32,\n values,\n None,\n )))\n }\n\n fn compute_query_embeddings(&self, _input: Arc) -> Result> {\n unimplemented!()\n }\n}\n\n#[tokio::main]\nasync fn main() -> Result<()> {\n let db = connect(\"./mydb\").execute().await?;\n db.embedding_registry()\n .register(\"my-embedder\", Arc::new(MyTextEmbedder { dim: 3 }))?;\n\n let schema = Arc::new(Schema::new(vec![Field::new(\"text\", DataType::Utf8, false)]));\n db.create_empty_table(\"mytable\", schema)\n .add_embedding(EmbeddingDefinition::new(\n \"text\",\n \"my-embedder\",\n Some(\"vector\"),\n ))?\n .execute()\n .await?;\n\n Ok(())\n}\n"; -export const RsManualQueryEmbeddings = "use std::{iter::once, sync::Arc};\n\nuse arrow_array::{record_batch, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::StreamExt;\nuse lancedb::{\n connect,\n embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},\n query::{ExecutableQuery, QueryBase},\n Result,\n};\n\n#[tokio::main]\nasync fn main() -> Result<()> {\n let db = connect(\"./mydb\").execute().await?;\n let api_key = std::env::var(\"OPENAI_API_KEY\").expect(\"OPENAI_API_KEY is not set\");\n let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(\n api_key,\n \"text-embedding-3-large\",\n )?);\n db.embedding_registry().register(\"openai\", embedding.clone())?;\n\n let schema = Arc::new(Schema::new(vec![Field::new(\"text\", DataType::Utf8, false)]));\n let table = db\n .create_empty_table(\"mytable\", schema)\n .add_embedding(EmbeddingDefinition::new(\"text\", \"openai\", Some(\"vector\")))?\n .execute()\n .await?;\n\n table\n .add(record_batch!((\"text\", Utf8, [\"This is a test.\", \"Another example.\"]))?)\n .execute()\n .await?;\n\n // Manually generate embeddings for the query (Enterprise path)\n let query = Arc::new(StringArray::from_iter_values(once(\"test example\")));\n let query_vector = embedding.compute_query_embeddings(query)?;\n // --8<-- [start:manual_query_search]\n // query_vector is assumed to already be generated by your embedding function\n let mut results = table.vector_search(query_vector)?.limit(5).execute().await?;\n\n while let Some(batch) = results.next().await {\n println!(\"{:?}\", batch?);\n }\n // --8<-- [end:manual_query_search]\n\n Ok(())\n}\n"; +export const RsManualQueryEmbeddings = "use std::{iter::once, sync::Arc};\n\nuse arrow_array::{record_batch, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::StreamExt;\nuse lancedb::{\n connect,\n embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},\n query::{ExecutableQuery, QueryBase},\n Result,\n};\n\n#[tokio::main]\nasync fn main() -> Result<()> {\n let db = connect(\"./mydb\").execute().await?;\n let api_key = \"your-openai-api-key\".to_string();\n let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(\n api_key,\n \"text-embedding-3-large\",\n )?);\n db.embedding_registry().register(\"openai\", embedding.clone())?;\n\n let schema = Arc::new(Schema::new(vec![Field::new(\"text\", DataType::Utf8, false)]));\n let table = db\n .create_empty_table(\"mytable\", schema)\n .add_embedding(EmbeddingDefinition::new(\"text\", \"openai\", Some(\"vector\")))?\n .execute()\n .await?;\n\n table\n .add(record_batch!((\"text\", Utf8, [\"This is a test.\", \"Another example.\"]))?)\n .execute()\n .await?;\n\n // Manually generate embeddings for the query (Enterprise path)\n let query = Arc::new(StringArray::from_iter_values(once(\"test example\")));\n let query_vector = embedding.compute_query_embeddings(query)?;\n // --8<-- [start:manual_query_search]\n // query_vector is assumed to already be generated by your embedding function\n let mut results = table.vector_search(query_vector)?.limit(5).execute().await?;\n\n while let Some(batch) = results.next().await {\n println!(\"{:?}\", batch?);\n }\n // --8<-- [end:manual_query_search]\n\n Ok(())\n}\n"; export const RsManualQuerySearch = "// query_vector is assumed to already be generated by your embedding function\nlet mut results = table.vector_search(query_vector)?.limit(5).execute().await?;\n\nwhile let Some(batch) = results.next().await {\n println!(\"{:?}\", batch?);\n}\n"; -export const RsOpenaiEmbeddings = "use std::{iter::once, sync::Arc};\n\nuse arrow_array::{record_batch, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::StreamExt;\nuse lancedb::{\n connect,\n embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},\n query::{ExecutableQuery, QueryBase},\n Result,\n};\n\n#[tokio::main]\nasync fn main() -> Result<()> {\n let db = connect(\"./mydb\").execute().await?;\n let api_key = std::env::var(\"OPENAI_API_KEY\").expect(\"OPENAI_API_KEY is not set\");\n let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(\n api_key,\n \"text-embedding-3-large\",\n )?);\n\n db.embedding_registry().register(\"openai\", embedding.clone())?;\n\n let schema = Arc::new(Schema::new(vec![Field::new(\"text\", DataType::Utf8, false)]));\n let table = db\n .create_empty_table(\"mytable\", schema)\n .add_embedding(EmbeddingDefinition::new(\"text\", \"openai\", Some(\"vector\")))?\n .execute()\n .await?;\n\n table\n .add(record_batch!((\"text\", Utf8, [\"This is a test.\", \"Another example.\"]))?)\n .execute()\n .await?;\n\n let query = Arc::new(StringArray::from_iter_values(once(\"test example\")));\n let query_vector = embedding.compute_query_embeddings(query)?;\n let mut results = table.vector_search(query_vector)?.limit(5).execute().await?;\n\n while let Some(batch) = results.next().await {\n println!(\"{:?}\", batch?);\n }\n\n Ok(())\n}\n"; +export const RsOpenaiEmbeddings = "use std::{iter::once, sync::Arc};\n\nuse arrow_array::{record_batch, StringArray};\nuse arrow_schema::{DataType, Field, Schema};\nuse futures::StreamExt;\nuse lancedb::{\n connect,\n embeddings::{openai::OpenAIEmbeddingFunction, EmbeddingDefinition, EmbeddingFunction},\n query::{ExecutableQuery, QueryBase},\n Result,\n};\n\n#[tokio::main]\nasync fn main() -> Result<()> {\n let db = connect(\"./mydb\").execute().await?;\n let api_key = \"your-openai-api-key\".to_string();\n let embedding = Arc::new(OpenAIEmbeddingFunction::new_with_model(\n api_key,\n \"text-embedding-3-large\",\n )?);\n\n db.embedding_registry().register(\"openai\", embedding.clone())?;\n\n let schema = Arc::new(Schema::new(vec![Field::new(\"text\", DataType::Utf8, false)]));\n let table = db\n .create_empty_table(\"mytable\", schema)\n .add_embedding(EmbeddingDefinition::new(\"text\", \"openai\", Some(\"vector\")))?\n .execute()\n .await?;\n\n table\n .add(record_batch!((\"text\", Utf8, [\"This is a test.\", \"Another example.\"]))?)\n .execute()\n .await?;\n\n let query = Arc::new(StringArray::from_iter_values(once(\"test example\")));\n let query_vector = embedding.compute_query_embeddings(query)?;\n let mut results = table.vector_search(query_vector)?.limit(5).execute().await?;\n\n while let Some(batch) = results.next().await {\n println!(\"{:?}\", batch?);\n }\n\n Ok(())\n}\n"; diff --git a/docs/web/snippets/materialized_view.mdx b/docs/web/snippets/materialized_view.mdx index 41f9def31..975f7635b 100644 --- a/docs/web/snippets/materialized_view.mdx +++ b/docs/web/snippets/materialized_view.mdx @@ -1,6 +1,6 @@ {/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} -export const PyMaterializedViewCreate = "import lancedb\n\ndb = lancedb.connect(\"./.lancedb\")\ntable = db.create_table(\n \"people\", [{\"name\": \"ada\", \"age\": 36}, {\"name\": \"kid\", \"age\": 7}]\n)\n\nview = db.create_materialized_view(\n \"adults\",\n \"people\",\n select=[\"name\", (\"shout\", \"upper(name)\")],\n where=\"age >= 18\",\n)\nprint(view.table.count_rows())\n# 1\n"; +export const PyMaterializedViewCreate = "import lancedb\n\ndb = lancedb.connect(\"./.lancedb\")\ntable = db.create_table(\n \"people\",\n [{\"name\": \"ada\", \"age\": 36}, {\"name\": \"kid\", \"age\": 7}],\n storage_options={\"new_table_enable_stable_row_ids\": \"true\"},\n)\n\nview = db.create_materialized_view(\n \"adults\",\n \"people\",\n select=[\"name\", (\"shout\", \"upper(name)\")],\n where=\"age >= 18\",\n)\nprint(view.table.count_rows())\n# 1\n"; -export const PyMaterializedViewRefresh = "table.add([{\"name\": \"bea\", \"age\": 41}])\n\nresult = view.refresh()\nprint(result.rows_written)\n# 2\n"; +export const PyMaterializedViewRefresh = "table.add([{\"name\": \"bea\", \"age\": 41}])\n\nview.refresh()\nprint(view.table.count_rows())\n# 2\n"; diff --git a/docs/web/snippets/multimodal.mdx b/docs/web/snippets/multimodal.mdx index 90e275db7..f41af6629 100644 --- a/docs/web/snippets/multimodal.mdx +++ b/docs/web/snippets/multimodal.mdx @@ -16,7 +16,7 @@ export const PyMultimodalImports = "import lancedb\nimport pyarrow as pa\nimport export const PyProcessResults = "# Convert back to PIL Image\nfor _, row in results.iterrows():\n image_bytes = row['image_blob']\n image = Image.open(io.BytesIO(image_bytes))\n print(f\"Retrieved image: {row['filename']}, Size: {image.size}\")\n # You can now use 'image' with other libraries or display it\n"; -export const PyQueryToPandasKwargs = "# Plain scan query: blob_mode is supported end to end\ndf_lazy = (\n tbl.search()\n .where(\"id = 1\")\n .select([\"id\", \"video\"])\n .to_pandas(blob_mode=\"lazy\")\n)\n\n# Same call shape works on async query builders\ndf_bytes = await (\n tbl_async.query()\n .where(\"id = 1\")\n .select([\"id\", \"video\"])\n .to_pandas(blob_mode=\"bytes\")\n)\n\n# Vector / FTS / hybrid queries return \"bytes\" or \"descriptions\".\n# Lazy BlobFile handles need a plain scan.\ndf_vec = (\n tbl.search(query_vector)\n .limit(10)\n .select([\"id\", \"video\"])\n .to_pandas(blob_mode=\"bytes\", split_blocks=True, self_destruct=True)\n)\n\n# to_arrow and to_list take the same blob_mode\nhits = tbl.search(query_vector).limit(2).select([\"id\", \"video\"])\narrow_bytes = hits.to_arrow(blob_mode=\"bytes\")\nrows = hits.to_list(blob_mode=\"bytes\")\n"; +export const PyQueryToPandasKwargs = "# Plain scan query: blob_mode is supported end to end\ndf_lazy = (\n tbl.search()\n .where(\"id = 1\")\n .select([\"id\", \"video\"])\n .to_pandas(blob_mode=\"lazy\")\n)\n\n# Same call shape works on async query builders\ndf_bytes = await (\n tbl_async.query()\n .where(\"id = 1\")\n .select([\"id\", \"video\"])\n .to_pandas(blob_mode=\"bytes\")\n)\n\n# Vector / FTS / hybrid queries can't materialize blob columns,\n# so omit them from the projection\ndf_vec = (\n tbl.search(query_vector)\n .limit(10)\n .select([\"id\", \"vector\"])\n .to_pandas(split_blocks=True, self_destruct=True)\n)\n"; export const PySearchData = "# Search for similar images\nquery_vector = np.random.rand(128).astype(np.float32)\nresults = tbl.search(query_vector).limit(1).to_pandas()\n"; diff --git a/docs/web/snippets/tables.mdx b/docs/web/snippets/tables.mdx index 6b9b98280..28b9ce172 100644 --- a/docs/web/snippets/tables.mdx +++ b/docs/web/snippets/tables.mdx @@ -90,7 +90,7 @@ export const PyMergeUpdateInsert = "import pyarrow as pa\n\ntable = db.create_ta export const PyOpenExistingTable = "db = tmp_db\n# Create a table first\ndata = [{\"vector\": [1.1, 1.2], \"lat\": 45.5, \"long\": -122.7}]\ndb.create_table(\"test_table\", data, mode=\"overwrite\")\n\n# List table names\nprint(db.list_tables().tables)\n\n# Open existing table\ntbl = db.open_table(\"test_table\")\n"; -export const PySchemaAddSetup = "table_name = \"schema_evolution_add_example\"\nif data is None:\n data = [\n {\n \"id\": 1,\n \"name\": \"Laptop\",\n \"price\": 1200.00,\n \"vector\": np.random.random(128).tolist(),\n },\n {\n \"id\": 2,\n \"name\": \"Smartphone\",\n \"price\": 800.00,\n \"vector\": np.random.random(128).tolist(),\n },\n {\n \"id\": 3,\n \"name\": \"Headphones\",\n \"price\": 150.00,\n \"vector\": np.random.random(128).tolist(),\n },\n ]\ntable = tmp_db.create_table(table_name, data, mode=\"overwrite\")\n"; +export const PySchemaAddSetup = "import numpy as np\n\ndata = [\n {\n \"id\": 1,\n \"name\": \"Laptop\",\n \"price\": 1200.00,\n \"vector\": np.random.random(128).tolist(),\n },\n {\n \"id\": 2,\n \"name\": \"Smartphone\",\n \"price\": 800.00,\n \"vector\": np.random.random(128).tolist(),\n },\n {\n \"id\": 3,\n \"name\": \"Headphones\",\n \"price\": 150.00,\n \"vector\": np.random.random(128).tolist(),\n },\n]\ntable = db.create_table(\"schema_evolution_add_example\", data, mode=\"overwrite\")\n"; export const PySchemaAlterSetup = "table_name = \"schema_evolution_alter_example\"\nif data is None:\n data = [\n {\n \"id\": 1,\n \"name\": \"Laptop\",\n \"price\": 1200,\n \"discount_price\": 1080.0,\n \"vector\": np.random.random(128).tolist(),\n },\n {\n \"id\": 2,\n \"name\": \"Smartphone\",\n \"price\": 800,\n \"discount_price\": 720.0,\n \"vector\": np.random.random(128).tolist(),\n },\n ]\nschema = pa.schema(\n {\n \"id\": pa.int64(),\n \"name\": pa.string(),\n \"price\": pa.int32(),\n \"discount_price\": pa.float64(),\n \"vector\": pa.list_(pa.float32(), 128),\n }\n)\ntable = tmp_db.create_table(table_name, data, schema=schema, mode=\"overwrite\")\n"; diff --git a/docs/web/sql/data-types.mdx b/docs/web/sql/data-types.mdx index 70bafdb96..8c7fa1781 100644 --- a/docs/web/sql/data-types.mdx +++ b/docs/web/sql/data-types.mdx @@ -6,7 +6,7 @@ icon: shapes keywords: ["sql types", "arrow", "cast", "pseudo-columns", "vector", "blob", "json"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) There is no SQL type layer over Lance storage. The table provider hands DataFusion the dataset's Arrow schema directly, so a column is the same object @@ -77,7 +77,7 @@ SELECT id FROM documents WHERE vector IS NOT NULL A list representation does not imply that ordinary arithmetic operators perform vector arithmetic. Use supported distance functions for similarity -calculations. SQL vector search depends on the deployment version and query +calculations. SQL vector search depends on the Enterprise server version and query plan; see [SQL](/basics/sql#no-vector-sql). diff --git a/docs/web/sql/ddl/function.mdx b/docs/web/sql/ddl/function.mdx index 39a661fc3..f0782008d 100644 --- a/docs/web/sql/ddl/function.mdx +++ b/docs/web/sql/ddl/function.mdx @@ -6,17 +6,22 @@ icon: function keywords: ["create function", "drop function", "show functions", "sql", "udf"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) ## CREATE FUNCTION {#create} ```sql -CREATE FUNCTION f(x BIGINT) RETURNS BIGINT +CREATE FUNCTION f(x int64) RETURNS int64 LANGUAGE python AS 'def f(x): return x * 2' WITH (pip = 'numpy==1.26.4', python_version = '3.11') ``` +Parameter and result types are Arrow type names, not SQL ones: `int64`, +`float64`, `utf8`, `bool`, `date32`, and lists such as `list` or +`fixed_size_list`. `BIGINT` is refused. A parameter accepts nulls +unless it is declared `NOT NULL`. + The body is Python source. Its dependencies are declared with it and are part of the **immutable function version** -- changing one produces a new version rather than altering the registered one, so a column filled by an earlier @@ -27,7 +32,7 @@ version is unaffected. | `pip` | pip requirements for the worker environment | | `conda` | conda packages instead of pip | | `conda_channels` | channels in priority order; requires `conda` | -| `python_version` | worker Python version, defaulting to yours | +| `python_version` | worker Python version; `3.12` when omitted, whatever Python you send the statement from | | `gpu` | `'true'` when every execution needs a GPU | **Errors and limits** diff --git a/docs/web/sql/ddl/index.mdx b/docs/web/sql/ddl/index.mdx index 2ac18dd4c..3181210c0 100644 --- a/docs/web/sql/ddl/index.mdx +++ b/docs/web/sql/ddl/index.mdx @@ -1,30 +1,38 @@ --- title: "Index" sidebarTitle: "Index" -description: "CREATE INDEX and DROP INDEX — one column, server-chosen name." +description: "CREATE INDEX and DROP INDEX — one column, with the algorithm chosen by USING." icon: bolt keywords: ["create index", "drop index", "sql"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) ## CREATE INDEX {#create} ```sql -CREATE INDEX ON documents (title) +CREATE INDEX ON documents USING btree (title) +CREATE INDEX text_search ON documents USING fts (text) ``` +`USING` is required: it names the algorithm. The scalar algorithms are `btree`, +`bitmap` and `label_list`; `fts` builds a full-text index; and the vector +algorithms are `ivf_pq`, `ivf_rq`, `ivf_flat`, `ivf_hnsw_sq` and `ivf_hnsw_pq`, +where the deployment supports them. Index options go in a `WITH (...)` clause. + -SQL `CREATE INDEX` covers **one column** and takes **no user-chosen name** -- -the server names it. For a multi-column or explicitly-named index, or to -choose the algorithm, use the SDK's -[`create_index`](/indexing). +In LanceDB Enterprise, SQL `CREATE INDEX` covers **one column**. A name between +`INDEX` and `ON` names the index, where the deployment's configuration allows +it; without one, the server names it after the column, as in `title_idx`. For a +multi-column index, use the SDK's [`create_index`](/indexing). -**Enterprise job type:** `create_ivf_pq`, `create_btree`, `create_fts`, and one -more per algorithm -- the type names which algorithm is building, not a -different feature. See [Build & Repair](/indexing/reindexing). +**Enterprise job type:** `reindex_` followed by the algorithm, such as +`reindex_btree` or `reindex_ivf_pq`; the name says which algorithm is building, +not a different feature. An SDK build that must not replace an existing index +runs as `create_` followed by the algorithm instead. See +[Build & Repair](/indexing/reindexing). ## DROP INDEX {#drop} @@ -35,7 +43,9 @@ DROP INDEX documents.title_idx Takes the qualified `table.index` name. Find it with the SDK's [`list_indices`](/indexing) if you created the index without -naming it yourself. +naming it yourself. A deployment builds the index in the background after +`CREATE INDEX` returns; until that build finishes, the index is not listed and +`DROP INDEX` reports it as not found. ## What to read next {#next} diff --git a/docs/web/sql/ddl/job.mdx b/docs/web/sql/ddl/job.mdx index ae99b014e..bc985d34d 100644 --- a/docs/web/sql/ddl/job.mdx +++ b/docs/web/sql/ddl/job.mdx @@ -6,7 +6,7 @@ icon: hourglass-half keywords: ["show jobs", "show errors", "pause job", "resume job", "cancel job", "sql"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) A [job](/basics/jobs) is not a catalog object -- it has no schema and is not namespaced -- but it has its own SQL statements, grouped here the same way the @@ -73,7 +73,7 @@ being ignored. Cancellation is a request, matching the SDK's [`cancel()`](/basics/jobs#managing): it does not guarantee the job stops before doing more work, and asking twice or cancelling a finished job does nothing. -`CANCEL JOB` itself can be unavailable on a deployment with no job registry +`CANCEL JOB` itself can be unavailable on an Enterprise cluster with no job registry configured, which is a deployment condition rather than something you can work around from SQL. diff --git a/docs/web/sql/ddl/materialized-view.mdx b/docs/web/sql/ddl/materialized-view.mdx index 51b5ef55b..19dba5aa4 100644 --- a/docs/web/sql/ddl/materialized-view.mdx +++ b/docs/web/sql/ddl/materialized-view.mdx @@ -6,13 +6,13 @@ icon: table keywords: ["materialized view", "create materialized view", "show materialized views", "sql"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) A materialized view stores a query's result as rows, kept current by a refresh -- the counterpart to a [view](/sql/ddl/view), which stores only the query. -Materialized views are off by default on a deployment and have to be enabled +Materialized views are off by default in LanceDB Enterprise and have to be enabled there. Which statements and query shapes a deployment accepts, whether it needs the source table to have stable row ids, and how it refreshes a view depend on its LanceDB Enterprise version and configuration. @@ -24,10 +24,9 @@ its LanceDB Enterprise version and configuration. CREATE MATERIALIZED VIEW AS SELECT ... ``` -On a `db://` connection, the SDK's -[`create_materialized_view`](/management/materialized-view#create) sends this -statement. Once created, a view is queried like a table, refreshed with SQL and -listed with SQL. +Once created, a materialized view can be queried like a table. Use the +statements below to list, refresh and drop it. For SDK methods, see +[Materialized views](/management/materialized-view#create). ## SHOW MATERIALIZED VIEWS {#show} @@ -36,8 +35,8 @@ SHOW MATERIALIZED VIEWS ``` -Cheaper than the SDK equivalent. `list_materialized_views` finds views by -reading every table's schema, costing one open per table; this asks the server. +This asks the server for its views, as the SDK's `list_materialized_views` does +on a `db://` connection. ## Refreshing one {#refresh} @@ -47,7 +46,7 @@ REFRESH MATERIALIZED VIEW chunks ``` See [REFRESH](/sql/dml/refresh#refresh-materialized-view) for the full syntax, -including `CASCADE` and `full = true`. +and compatibility notes for older server versions. ## Dropping one {#drop} @@ -57,8 +56,8 @@ stored rows, unlike dropping a [view](/sql/ddl/view#drop). **Enterprise job type:** `mat_view_refresh` for a refresh, or `sql` on a -deployment that rebuilds views through SQL; `drop_table` for dropping one, same -as any table. +deployment that rebuilds views through SQL. Dropping one runs the same job as +dropping any table, `drop_table` or `drop_object`. ## What to read next {#next} @@ -68,6 +67,6 @@ as any table. Creating one from the SDK. - How a refresh works, locally and on a deployment. + How a refresh works, locally and in LanceDB Enterprise. diff --git a/docs/web/sql/ddl/namespace.mdx b/docs/web/sql/ddl/namespace.mdx index daedf9720..efd209b02 100644 --- a/docs/web/sql/ddl/namespace.mdx +++ b/docs/web/sql/ddl/namespace.mdx @@ -6,7 +6,7 @@ icon: folder-tree keywords: ["namespace", "schema", "create schema", "drop schema", "sql"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) A [namespace](/management/namespace) is addressed in SQL as a `SCHEMA` -- the standard's name for the same concept. @@ -14,14 +14,16 @@ standard's name for the same concept. ## CREATE SCHEMA {#create} ```sql -CREATE SCHEMA public -CREATE SCHEMA db1.ns1.ns2 +CREATE SCHEMA analytics +CREATE SCHEMA db1.`ns1$ns2` CREATE SCHEMA IF NOT EXISTS `ns1$ns2` ``` -A dotted path creates the whole chain: `db1.ns1.ns2` creates `ns2` inside -`ns1` inside `db1`, provided the parents already exist. A name containing the -namespace path delimiter itself is backtick-quoted, as in `` `ns1$ns2` `` above. +A nested namespace is one backtick-quoted name with `$` between its +components: `` db1.`ns1$ns2` `` creates `ns2` inside `ns1` in database `db1`, +provided `ns1` already exists. The part before the dot is the database, so a +three-part path such as `db1.ns1.ns2` is rejected. `public` is the database's +root namespace, which is never created or dropped. `IF NOT EXISTS` makes the statement a no-op rather than an error when the namespace is already there. @@ -29,14 +31,14 @@ namespace is already there. ## DROP SCHEMA {#drop} ```sql -DROP SCHEMA db1.public +DROP SCHEMA db1.analytics DROP SCHEMA IF EXISTS `a$b` DROP SCHEMA `a$b` RESTRICT ``` `CASCADE` is rejected: **"DROP SCHEMA CASCADE is not supported; drop the -objects inside first."** A namespace's tables, views and functions must be +objects inside first."** A namespace's tables, views and other objects must be dropped explicitly before the namespace itself can go. `RESTRICT` -- the default -- states that requirement rather than changing it. diff --git a/docs/web/sql/ddl/table.mdx b/docs/web/sql/ddl/table.mdx index bd64458be..390c24b65 100644 --- a/docs/web/sql/ddl/table.mdx +++ b/docs/web/sql/ddl/table.mdx @@ -6,7 +6,7 @@ icon: table keywords: ["create table", "alter table", "drop table", "describe", "clone", "ddl"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) ## CREATE TABLE {#create} @@ -30,16 +30,15 @@ The table's columns and types, without a plan -- the SQL counterpart of ## ALTER TABLE {#alter} -`ALTER TABLE ... ADD COLUMN` covers three different things, distinguished by -what follows the column name. +What `ALTER TABLE ... ADD COLUMN` does depends on what follows the column +name. ### A plain column {#plain-column} -```sql -ALTER TABLE t ADD COLUMN c INT -``` - -Added empty and nullable. Nothing is computed. +SQL `ADD COLUMN` requires an `AS` binding, so a plain column such as +`ALTER TABLE t ADD COLUMN c INT` is rejected. Add an empty, nullable column +with the SDK instead -- see +[Add nullable columns](/management/table#schema-add-nullable-columns). ### A computed column {#computed-column} @@ -52,6 +51,9 @@ The column is committed empty and the expression is stored, not evaluated -- declaring one on a billion-row table costs what declaring one on an empty table costs. [`REFRESH COLUMN`](/sql/dml/refresh) fills it. +{/* +Draft for the Functions release. + ### A function column {#function-column} ```sql @@ -86,6 +88,7 @@ ALTER TABLE t ADD COLUMN c AS f(x) WITH (num_workers = '2') `WITH` on a plain computed column accepts only `version`. Resource options such as `num_workers` apply to function columns, where there are workers to size. +*/} ### Table properties {#properties} @@ -95,19 +98,23 @@ ALTER TABLE t SET UNENFORCED PRIMARY KEY (id) Records which column identifies a row. **Unenforced** is not a hedge in the naming: uniqueness is not checked on write, and declaring it does not make -duplicates impossible. It tells the planner what to assume. +duplicates impossible. It tells the planner what to assume. The column must be +declared `NOT NULL`. ```sql ALTER TABLE t SET WAL -ALTER TABLE t SET WAL WITH (sharding = 'bucket', buckets = 4) +ALTER TABLE t SET WAL WITH (sharding = 'bucket', column = 'id', buckets = 4) ALTER TABLE t SET WAL WITH (maintained_indexes = '') ALTER TABLE t UNSET WAL ``` A write-ahead log makes small frequent writes cheap by batching them out of the commit path. `sharding` and `buckets` spread the log so concurrent writers do -not contend on one place; `maintained_indexes` chooses which indexes are kept -current against WAL rows. See +not contend on one place; `bucket` sharding names its `column`, which must be +the table's unenforced primary key, and inserts into the table then need a +scalar index on that key. `maintained_indexes` chooses which indexes are kept +current against WAL rows. A table with a computed column cannot take a +write-ahead log. See [Background jobs](/maintenance/background-jobs#wal) for how the log gets folded back into the table. @@ -162,7 +169,8 @@ Removes the table and every version of it. There is no undrop. -**Enterprise job type:** `drop_table`. See +**Enterprise job type:** `drop_table` or `drop_object`, depending on how the +deployment stores its catalog. See [Background jobs](/maintenance/background-jobs#teardown). diff --git a/docs/web/sql/ddl/view.mdx b/docs/web/sql/ddl/view.mdx index fc0807058..8b4337fac 100644 --- a/docs/web/sql/ddl/view.mdx +++ b/docs/web/sql/ddl/view.mdx @@ -6,7 +6,7 @@ icon: eye keywords: ["create view", "drop view", "sql", "catalog"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) A view stores a **query**, not rows. Reading it runs the query against current data, so a view is never stale -- and never faster than the query it wraps. It diff --git a/docs/web/sql/dml/copy.mdx b/docs/web/sql/dml/copy.mdx index 837b92285..658b20b59 100644 --- a/docs/web/sql/dml/copy.mdx +++ b/docs/web/sql/dml/copy.mdx @@ -6,7 +6,11 @@ icon: file-export keywords: ["copy", "export", "parquet", "sql"] --- -Enterprise [SQL runs on a deployment.](/sql) + +Recent deployment versions refuse `COPY` over the SQL endpoint with "COPY is not +supported". On those, export a result by reading it through a Flight SQL client +or an SDK and writing the files from your application. + ```sql COPY (SELECT id, title FROM documents) TO 's3://bucket/export/' STORED AS PARQUET diff --git a/docs/web/sql/dml/delete.mdx b/docs/web/sql/dml/delete.mdx index da71dd9eb..90965062b 100644 --- a/docs/web/sql/dml/delete.mdx +++ b/docs/web/sql/dml/delete.mdx @@ -6,12 +6,16 @@ icon: trash keywords: ["delete", "sql"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) ```sql -DELETE FROM documents WHERE created_at < '2025-01-01' +DELETE FROM documents WHERE created_at < TIMESTAMP '2025-01-01 00:00:00' ``` +Compare a timestamp column with a typed literal, as above, or with +`CAST('2025-01-01' AS TIMESTAMP)`. A bare string such as `'2025-01-01'` is +rejected in a `DELETE` predicate, even though a `SELECT` accepts it. + Deletion marks rows rather than erasing them, so the data remains recoverable for as long as the version holding it survives [retention](/maintenance/cleanup). diff --git a/docs/web/sql/dml/insert.mdx b/docs/web/sql/dml/insert.mdx index 2be017f06..61fcb9994 100644 --- a/docs/web/sql/dml/insert.mdx +++ b/docs/web/sql/dml/insert.mdx @@ -6,7 +6,7 @@ icon: plus keywords: ["insert", "insert into", "sql"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) ```sql INSERT INTO documents (id, title) VALUES (1, 'Merlin'), (2, 'Arthur') diff --git a/docs/web/sql/dml/refresh.mdx b/docs/web/sql/dml/refresh.mdx index 2d00c1104..1a14adef8 100644 --- a/docs/web/sql/dml/refresh.mdx +++ b/docs/web/sql/dml/refresh.mdx @@ -6,7 +6,7 @@ icon: arrows-rotate keywords: ["refresh column", "refresh materialized view", "backfill", "cascade"] --- -Enterprise [SQL runs on a deployment.](/sql) +Enterprise [SQL runs in LanceDB Enterprise.](/sql) LanceDB extensions with no DataFusion equivalent. They are what makes [feature engineering](/feature-engineering) reachable from SQL: a column @@ -52,33 +52,35 @@ REFRESH COLUMN embedding ON documents `num_workers` sets a count; the budget options size each worker. They are alternatives rather than a pair. +{/* +Draft for the Functions release. + `error_policy` and `num_workers` are refused for a column filled by a registered [function](/management/function) with its own resource budget -- use the CPU, memory and GPU options there. +*/} ## REFRESH MATERIALIZED VIEW {#refresh-materialized-view} ```sql -REFRESH MATERIALIZED VIEW [CASCADE] [WITH (