mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-16 07:36:50 +00:00
feat(python): support exist_ok when creating views (#4984)
Add `exist_ok=False` to synchronous and asynchronous Python view
creation. Passing `True` sends `CREATE VIEW IF NOT EXISTS` and returns
the existing view's stored query and schema unchanged; the default
continues to fail on an existing name.
```python
view = db.create_view("recent_orders", query, exist_ok=True)
view = await async_db.create_view("recent_orders", query, exist_ok=True)
```
Requires a SQL server supporting `CREATE VIEW IF NOT EXISTS`. Rust gains
an awaitable builder with `.exist_ok(true)`. Existing
`.create_view(...).await` calls continue to work; callers requiring
`Future` must convert the builder, and custom `Database` implementations
must accept the new `exist_ok` argument. Includes Python round-trip
coverage and Rust coverage for direct/deferred results and metadata
preservation.
This commit is contained in:
@@ -23,6 +23,16 @@ catalog removal; physical cleanup can continue afterward.
|
||||
|
||||
## CREATE VIEW {#create}
|
||||
|
||||
{/* Draft for the view exist_ok release.
|
||||
|
||||
`CREATE VIEW IF NOT EXISTS` creates a missing view and returns an existing
|
||||
view's stored query and schema unchanged. Without `IF NOT EXISTS`, an existing
|
||||
name is an error. In Python, use `db.create_view(name, query, exist_ok=True)`;
|
||||
in Rust, use `db.create_view(name, query, &namespace).exist_ok(true).await?`.
|
||||
Returning an existing view requires `SELECT` on that view.
|
||||
|
||||
*/}
|
||||
|
||||
<CodeGroup>
|
||||
```sql SQL icon="database"
|
||||
CREATE VIEW "recent_matches" AS
|
||||
|
||||
@@ -253,7 +253,12 @@ class Connection(object):
|
||||
self, name: str, namespace_path: Optional[List[str]] = None
|
||||
) -> Tuple[str, int, int]: ...
|
||||
async def create_view(
|
||||
self, name: str, query: str, namespace_path: Optional[List[str]] = None
|
||||
self,
|
||||
name: str,
|
||||
query: str,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
*,
|
||||
exist_ok: bool = False,
|
||||
) -> Tuple[str, List[str], str, str, List[str], pa.Schema]: ...
|
||||
async def describe_view(
|
||||
self, name: str, namespace_path: Optional[List[str]] = None
|
||||
|
||||
@@ -1273,7 +1273,12 @@ class DBConnection(EnforceOverrides):
|
||||
)
|
||||
|
||||
def create_view(
|
||||
self, name: str, query: str, *, namespace_path: Optional[List[str]] = None
|
||||
self,
|
||||
name: str,
|
||||
query: str,
|
||||
*,
|
||||
exist_ok: bool = False,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
) -> ViewDescription:
|
||||
"""Create a view: a named query the database plans on every read.
|
||||
|
||||
@@ -1281,9 +1286,10 @@ class DBConnection(EnforceOverrides):
|
||||
is refused now rather than at the first read. A view holds no rows, and
|
||||
its readers see its sources as they are at read time.
|
||||
|
||||
There is no replace: a name already taken is an error, and changing a
|
||||
view is a drop followed by a create. LanceDB Embedded connections raise
|
||||
``NotImplementedError``.
|
||||
A name already taken is an error unless ``exist_ok=True``, which returns
|
||||
the existing view's stored query and schema without changing them.
|
||||
Changing a view is a drop followed by a create. LanceDB Embedded
|
||||
connections raise ``NotImplementedError``.
|
||||
|
||||
>>> import lancedb
|
||||
>>> db = lancedb.connect("db://my_database") # doctest: +SKIP
|
||||
@@ -2592,10 +2598,17 @@ class LanceDBConnection(DBConnection):
|
||||
|
||||
@override
|
||||
def create_view(
|
||||
self, name: str, query: str, *, namespace_path: Optional[List[str]] = None
|
||||
self,
|
||||
name: str,
|
||||
query: str,
|
||||
*,
|
||||
exist_ok: bool = False,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
) -> ViewDescription:
|
||||
return LOOP.run(
|
||||
self._conn.create_view(name, query, namespace_path=namespace_path)
|
||||
self._conn.create_view(
|
||||
name, query, exist_ok=exist_ok, namespace_path=namespace_path
|
||||
)
|
||||
)
|
||||
|
||||
@override
|
||||
@@ -4107,7 +4120,12 @@ class AsyncConnection(object):
|
||||
)
|
||||
|
||||
async def create_view(
|
||||
self, name: str, query: str, *, namespace_path: Optional[List[str]] = None
|
||||
self,
|
||||
name: str,
|
||||
query: str,
|
||||
*,
|
||||
exist_ok: bool = False,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
) -> ViewDescription:
|
||||
"""Create a view: a named query the database plans on every read.
|
||||
|
||||
@@ -4115,7 +4133,9 @@ class AsyncConnection(object):
|
||||
[DBConnection.create_view][lancedb.DBConnection.create_view].
|
||||
"""
|
||||
return _view_description(
|
||||
await self._inner.create_view(name, query, list(namespace_path or []))
|
||||
await self._inner.create_view(
|
||||
name, query, list(namespace_path or []), exist_ok=exist_ok
|
||||
)
|
||||
)
|
||||
|
||||
async def describe_view(
|
||||
|
||||
@@ -1197,10 +1197,17 @@ class RemoteDBConnection(DBConnection):
|
||||
|
||||
@override
|
||||
def create_view(
|
||||
self, name: str, query: str, *, namespace_path: Optional[List[str]] = None
|
||||
self,
|
||||
name: str,
|
||||
query: str,
|
||||
*,
|
||||
exist_ok: bool = False,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
) -> ViewDescription:
|
||||
return LOOP.run(
|
||||
self._conn.create_view(name, query, namespace_path=namespace_path)
|
||||
self._conn.create_view(
|
||||
name, query, exist_ok=exist_ok, namespace_path=namespace_path
|
||||
)
|
||||
)
|
||||
|
||||
@override
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import json
|
||||
|
||||
import pyarrow as pa
|
||||
import pytest
|
||||
|
||||
import lancedb
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
("exist_ok", "existing"),
|
||||
[(None, False), (False, False), (True, False), (True, True)],
|
||||
)
|
||||
async def test_create_view_exist_ok_preserves_server_metadata(
|
||||
asynchronous, exist_ok, existing
|
||||
):
|
||||
flight = pytest.importorskip("pyarrow.flight")
|
||||
original_query = "SELECT 'original' AS original"
|
||||
submitted_query = (
|
||||
"SELECT 'replacement' AS replacement" if existing else original_query
|
||||
)
|
||||
schema = {
|
||||
"fields": [{"name": "original", "nullable": True, "type": {"type": "utf8"}}],
|
||||
"metadata": {
|
||||
"lancedb::view::default_db": "analytics",
|
||||
"lancedb::view::default_namespace": "team",
|
||||
},
|
||||
}
|
||||
result = pa.table(
|
||||
{"name": ["saved"], "query": [original_query], "schema": [json.dumps(schema)]}
|
||||
)
|
||||
|
||||
class ViewServer(flight.FlightServerBase):
|
||||
def __init__(self):
|
||||
super().__init__("grpc://127.0.0.1:0")
|
||||
self.requests = []
|
||||
|
||||
def do_get(self, context, ticket):
|
||||
self.requests.append(ticket.ticket)
|
||||
return flight.RecordBatchStream(result)
|
||||
|
||||
with ViewServer() as server:
|
||||
connection_options = {
|
||||
"api_key": "test-key",
|
||||
"host_override": "http://localhost:10024",
|
||||
"sql_host_override": f"grpc://127.0.0.1:{server.port}",
|
||||
}
|
||||
kwargs = {} if exist_ok is None else {"exist_ok": exist_ok}
|
||||
if asynchronous:
|
||||
db = await lancedb.connect_async("db://analytics", **connection_options)
|
||||
view = await db.create_view(
|
||||
"saved", submitted_query, namespace_path=["team"], **kwargs
|
||||
)
|
||||
else:
|
||||
db = lancedb.connect("db://analytics", **connection_options)
|
||||
view = db.create_view(
|
||||
"saved", submitted_query, namespace_path=["team"], **kwargs
|
||||
)
|
||||
(request,) = server.requests
|
||||
|
||||
prefix = "CREATE VIEW IF NOT EXISTS" if exist_ok else "CREATE VIEW"
|
||||
assert f'{prefix} "saved" AS {submitted_query}'.encode() in request
|
||||
assert view.name == "saved"
|
||||
assert view.namespace_path == ["team"]
|
||||
assert view.default_namespace_path == ["team"]
|
||||
assert view.query == original_query
|
||||
assert view.schema.names == ["original"]
|
||||
@@ -1158,18 +1158,20 @@ impl Connection {
|
||||
})
|
||||
}
|
||||
|
||||
#[pyo3(signature = (name, query, namespace_path=None))]
|
||||
#[pyo3(signature = (name, query, namespace_path=None, *, exist_ok=false))]
|
||||
pub fn create_view(
|
||||
self_: PyRef<'_, Self>,
|
||||
name: String,
|
||||
query: String,
|
||||
namespace_path: Option<Vec<String>>,
|
||||
exist_ok: bool,
|
||||
) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.get_inner()?.clone();
|
||||
let namespace_path = namespace_path.unwrap_or_default();
|
||||
future_into_py(self_.py(), async move {
|
||||
let view = inner
|
||||
.create_view(name, query, &namespace_path)
|
||||
.exist_ok(exist_ok)
|
||||
.await
|
||||
.infer_error()?;
|
||||
view_description_to_py(view)
|
||||
|
||||
@@ -53,6 +53,9 @@ use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider};
|
||||
|
||||
mod create_secret;
|
||||
mod create_table;
|
||||
mod create_view;
|
||||
|
||||
pub use create_view::CreateViewBuilder;
|
||||
|
||||
pub use create_secret::CreateSecretBuilder;
|
||||
|
||||
@@ -976,9 +979,10 @@ impl Connection {
|
||||
/// refused now rather than at the first read. A view holds no rows, and
|
||||
/// its readers see its sources as they are at read time.
|
||||
///
|
||||
/// There is no replace: a name already taken is an error, and changing a
|
||||
/// view is a drop followed by a create, each authorized against what it
|
||||
/// actually touches. LanceDB Embedded databases return [`Error::NotSupported`].
|
||||
/// A name already taken is an error unless [`CreateViewBuilder::exist_ok`] is
|
||||
/// true, which returns the stored definition and schema without changing them.
|
||||
/// Changing a view is a drop followed by a create, each authorized against what
|
||||
/// it actually touches. LanceDB Embedded databases return [`Error::NotSupported`].
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
@@ -1011,16 +1015,18 @@ impl Connection {
|
||||
/// # Ok(())
|
||||
/// # }
|
||||
/// ```
|
||||
pub async fn create_view(
|
||||
pub fn create_view(
|
||||
&self,
|
||||
name: impl AsRef<str>,
|
||||
query: impl AsRef<str>,
|
||||
namespace_path: &[String],
|
||||
) -> Result<ViewDescription> {
|
||||
validate_view_reference(name.as_ref(), namespace_path)?;
|
||||
self.internal
|
||||
.create_view(name.as_ref(), query.as_ref(), namespace_path)
|
||||
.await
|
||||
) -> CreateViewBuilder {
|
||||
CreateViewBuilder::new(
|
||||
self.internal.clone(),
|
||||
name.as_ref().to_owned(),
|
||||
query.as_ref().to_owned(),
|
||||
namespace_path.to_vec(),
|
||||
)
|
||||
}
|
||||
|
||||
/// What this database records about one view: its defining query and the
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
use std::future::{Future, IntoFuture};
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::Result;
|
||||
use crate::database::Database;
|
||||
use crate::utils::validate_view_reference;
|
||||
use crate::view::ViewDescription;
|
||||
|
||||
/// Configure creation of a view with [`super::Connection::create_view`].
|
||||
#[must_use]
|
||||
pub struct CreateViewBuilder {
|
||||
database: Arc<dyn Database>,
|
||||
name: String,
|
||||
query: String,
|
||||
namespace_path: Vec<String>,
|
||||
exist_ok: bool,
|
||||
}
|
||||
|
||||
impl CreateViewBuilder {
|
||||
pub(super) fn new(
|
||||
database: Arc<dyn Database>,
|
||||
name: String,
|
||||
query: String,
|
||||
namespace_path: Vec<String>,
|
||||
) -> Self {
|
||||
Self {
|
||||
database,
|
||||
name,
|
||||
query,
|
||||
namespace_path,
|
||||
exist_ok: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the existing view without changing its definition if it exists.
|
||||
/// Defaults to false, which reports an error for an existing name.
|
||||
pub fn exist_ok(mut self, exist_ok: bool) -> Self {
|
||||
self.exist_ok = exist_ok;
|
||||
self
|
||||
}
|
||||
|
||||
/// Create the view or return the existing view when `exist_ok` is true.
|
||||
pub async fn execute(self) -> Result<ViewDescription> {
|
||||
validate_view_reference(&self.name, &self.namespace_path)?;
|
||||
self.database
|
||||
.create_view(&self.name, &self.query, &self.namespace_path, self.exist_ok)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
impl IntoFuture for CreateViewBuilder {
|
||||
type Output = Result<ViewDescription>;
|
||||
type IntoFuture = Pin<Box<dyn Future<Output = Self::Output> + Send>>;
|
||||
|
||||
fn into_future(self) -> Self::IntoFuture {
|
||||
Box::pin(self.execute())
|
||||
}
|
||||
}
|
||||
@@ -693,6 +693,7 @@ pub trait Database:
|
||||
_name: &str,
|
||||
_query: &str,
|
||||
_namespace_path: &[String],
|
||||
_exist_ok: bool,
|
||||
) -> Result<ViewDescription> {
|
||||
view_ops_not_supported()
|
||||
}
|
||||
|
||||
@@ -1099,11 +1099,12 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
|
||||
name: &str,
|
||||
query: &str,
|
||||
namespace_path: &[String],
|
||||
exist_ok: bool,
|
||||
) -> Result<ViewDescription> {
|
||||
validate_view_name(name)?;
|
||||
validate_namespace(namespace_path)?;
|
||||
self.sql_client()?
|
||||
.create_view(name, query, namespace_path)
|
||||
.create_view(name, query, namespace_path, exist_ok)
|
||||
.await
|
||||
}
|
||||
async fn describe_view(
|
||||
|
||||
@@ -99,13 +99,18 @@ impl SqlClient {
|
||||
name: &str,
|
||||
query: &str,
|
||||
namespace: &[String],
|
||||
exist_ok: bool,
|
||||
) -> Result<ViewDescription> {
|
||||
validate_view_name(name)?;
|
||||
validate_namespace(namespace)?;
|
||||
let namespace_path = namespace.to_vec();
|
||||
let if_not_exists = if exist_ok { " IF NOT EXISTS" } else { "" };
|
||||
self.object_job(
|
||||
ExecuteQueryRequest {
|
||||
query: format!("CREATE VIEW {} AS {query}", quote_identifier(name)),
|
||||
query: format!(
|
||||
"CREATE VIEW{if_not_exists} {} AS {query}",
|
||||
quote_identifier(name)
|
||||
),
|
||||
default_namespace_path: namespace.to_vec(),
|
||||
parameters: None,
|
||||
},
|
||||
|
||||
@@ -24,6 +24,7 @@ use super::stream::{PROTOCOL_HEADER, PROTOCOL_VERSION, QueryResponse};
|
||||
use super::*;
|
||||
use crate::Connection;
|
||||
use crate::database::Database;
|
||||
use crate::embeddings::MemoryRegistry;
|
||||
use crate::materialized_view::CreateMaterializedViewRequest;
|
||||
use crate::remote::RemoteCatalogOptions;
|
||||
use crate::remote::client::HeaderProvider;
|
||||
@@ -3710,26 +3711,49 @@ async fn view_create_and_describe_return_metadata_through_sql() {
|
||||
..Default::default()
|
||||
};
|
||||
let statements = service.statements.clone();
|
||||
let headers = service.headers.clone();
|
||||
let (client, shutdown) = start_sql_service(service, ClientConfig::default()).await;
|
||||
let db = RemoteDatabase::new_mock(|_| -> http::Response<String> {
|
||||
panic!("unexpected REST request");
|
||||
})
|
||||
.with_sql_client(client);
|
||||
let created = db
|
||||
let connection = Connection::new(Arc::new(db), Arc::new(MemoryRegistry::new()));
|
||||
let created = connection
|
||||
.create_view("adults", "SELECT name FROM people", &["team".into()])
|
||||
.await
|
||||
.unwrap();
|
||||
let described = db.describe_view("adults", &["team".into()]).await.unwrap();
|
||||
let existing = connection
|
||||
.create_view(
|
||||
"adults",
|
||||
"SELECT replacement FROM other_people",
|
||||
&["team".into()],
|
||||
)
|
||||
.exist_ok(true)
|
||||
.await
|
||||
.unwrap();
|
||||
let described = connection
|
||||
.describe_view("adults", &["team".into()])
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(created, described);
|
||||
assert_eq!(existing, described);
|
||||
assert_eq!(created.schema.fields()[0].name(), "name");
|
||||
assert_eq!(created.default_namespace_path, ["team"]);
|
||||
assert_eq!(
|
||||
statements.lock().unwrap().as_slice(),
|
||||
[
|
||||
"CREATE VIEW \"adults\" AS SELECT name FROM people",
|
||||
"CREATE VIEW IF NOT EXISTS \"adults\" AS SELECT replacement FROM other_people",
|
||||
"DESCRIBE VIEW \"adults\""
|
||||
]
|
||||
);
|
||||
assert!(
|
||||
headers
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.all(|header| header.namespace_path == "team")
|
||||
);
|
||||
shutdown.send(()).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user