mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-16 07:36:50 +00:00
feat(python): support exist_ok for materialized views (#4991)
This commit is contained in:
@@ -27,6 +27,16 @@ CREATE MATERIALIZED VIEW <name> AS SELECT ...
|
||||
```
|
||||
</CodeGroup>
|
||||
|
||||
{/* Draft for the materialized-view exist_ok release.
|
||||
|
||||
`IF NOT EXISTS` preserves an existing materialized view's definition, data and
|
||||
refresh settings without refreshing it. The Python equivalent is
|
||||
`db.create_materialized_view("saved", "source", exist_ok=True)`. A skipped
|
||||
asynchronous creation returns a completed job without submitting a refresh job.
|
||||
Existing names are errors by default.
|
||||
|
||||
*/}
|
||||
|
||||
Once created, a materialized view can be queried like a table. Use the
|
||||
statements below to list, refresh and drop it. For SDK methods, see
|
||||
[Materialized views](/management/materialized-view#create).
|
||||
|
||||
@@ -369,6 +369,7 @@ class Connection(object):
|
||||
auto_refresh_interval_seconds: Optional[float] = None,
|
||||
*,
|
||||
query: Optional[str] = None,
|
||||
exist_ok: bool = False,
|
||||
) -> Table: ...
|
||||
async def create_materialized_view_async(
|
||||
self,
|
||||
@@ -382,6 +383,7 @@ class Connection(object):
|
||||
auto_refresh_interval_seconds: Optional[float] = None,
|
||||
*,
|
||||
query: Optional[str] = None,
|
||||
exist_ok: bool = False,
|
||||
) -> Job: ...
|
||||
def list_materialized_views(
|
||||
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
|
||||
|
||||
@@ -710,11 +710,15 @@ class DBConnection(EnforceOverrides):
|
||||
where: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
with_no_data: bool = False,
|
||||
exist_ok: bool = False,
|
||||
auto_refresh_enabled: Optional[bool] = None,
|
||||
auto_refresh_interval_seconds: Optional[float] = None,
|
||||
) -> MaterializedView:
|
||||
"""Define a materialized view named ``name`` over the table ``source``.
|
||||
|
||||
``exist_ok=True`` preserves an existing view's definition, data and
|
||||
refresh settings without refreshing it. Existing names error by default.
|
||||
|
||||
LanceDB Embedded creation and refresh require a source build with the
|
||||
``experimental-functions`` Cargo feature. Published packages leave it
|
||||
disabled; LanceDB Enterprise operations remain available.
|
||||
@@ -780,11 +784,15 @@ class DBConnection(EnforceOverrides):
|
||||
where: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
with_no_data: bool = False,
|
||||
exist_ok: bool = False,
|
||||
auto_refresh_enabled: Optional[bool] = None,
|
||||
auto_refresh_interval_seconds: Optional[float] = None,
|
||||
) -> Job[None]:
|
||||
"""Submit materialized-view creation and return its job.
|
||||
|
||||
``exist_ok=True`` preserves an existing view without refreshing it.
|
||||
A skipped creation returns a completed job without a registry job ID.
|
||||
|
||||
The job may already be complete in LanceDB Embedded. In LanceDB
|
||||
Distributed, creation submits CREATE MATERIALIZED VIEW through the SQL
|
||||
server and the returned ``id`` is the registry job ID. That ID can be
|
||||
@@ -2150,11 +2158,15 @@ class LanceDBConnection(DBConnection):
|
||||
where: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
with_no_data: bool = False,
|
||||
exist_ok: bool = False,
|
||||
auto_refresh_enabled: Optional[bool] = None,
|
||||
auto_refresh_interval_seconds: Optional[float] = None,
|
||||
) -> MaterializedView:
|
||||
"""Define a materialized view named ``name`` over the table ``source``.
|
||||
|
||||
``exist_ok=True`` preserves an existing view's definition, data and
|
||||
refresh settings without refreshing it. Existing names error by default.
|
||||
|
||||
LanceDB Embedded creation and refresh require a source build with the
|
||||
``experimental-functions`` Cargo feature. Published packages leave it
|
||||
disabled; LanceDB Enterprise operations remain available.
|
||||
@@ -2185,6 +2197,7 @@ class LanceDBConnection(DBConnection):
|
||||
where=where,
|
||||
limit=limit,
|
||||
with_no_data=with_no_data,
|
||||
exist_ok=exist_ok,
|
||||
auto_refresh_enabled=auto_refresh_enabled,
|
||||
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
|
||||
)
|
||||
@@ -2202,6 +2215,7 @@ class LanceDBConnection(DBConnection):
|
||||
where: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
with_no_data: bool = False,
|
||||
exist_ok: bool = False,
|
||||
auto_refresh_enabled: Optional[bool] = None,
|
||||
auto_refresh_interval_seconds: Optional[float] = None,
|
||||
) -> Job[None]:
|
||||
@@ -2214,6 +2228,7 @@ class LanceDBConnection(DBConnection):
|
||||
where=where,
|
||||
limit=limit,
|
||||
with_no_data=with_no_data,
|
||||
exist_ok=exist_ok,
|
||||
auto_refresh_enabled=auto_refresh_enabled,
|
||||
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
|
||||
)
|
||||
@@ -3453,11 +3468,15 @@ class AsyncConnection(object):
|
||||
where: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
with_no_data: bool = False,
|
||||
exist_ok: bool = False,
|
||||
auto_refresh_enabled: Optional[bool] = None,
|
||||
auto_refresh_interval_seconds: Optional[float] = None,
|
||||
) -> AsyncMaterializedView:
|
||||
"""Define a materialized view named ``name`` over the table ``source``.
|
||||
|
||||
``exist_ok=True`` preserves an existing view's definition, data and
|
||||
refresh settings without refreshing it. Existing names error by default.
|
||||
|
||||
LanceDB Embedded creation and refresh require a source build with the
|
||||
``experimental-functions`` Cargo feature. Published packages leave it
|
||||
disabled; LanceDB Enterprise operations remain available.
|
||||
@@ -3480,6 +3499,7 @@ class AsyncConnection(object):
|
||||
filter=where,
|
||||
limit=limit,
|
||||
with_no_data=with_no_data,
|
||||
exist_ok=exist_ok,
|
||||
auto_refresh_enabled=auto_refresh_enabled,
|
||||
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
|
||||
**native_options,
|
||||
@@ -3496,11 +3516,15 @@ class AsyncConnection(object):
|
||||
where: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
with_no_data: bool = False,
|
||||
exist_ok: bool = False,
|
||||
auto_refresh_enabled: Optional[bool] = None,
|
||||
auto_refresh_interval_seconds: Optional[float] = None,
|
||||
) -> AsyncJob[None]:
|
||||
"""Submit materialized-view creation and return its job.
|
||||
|
||||
``exist_ok=True`` preserves an existing view without refreshing it.
|
||||
A skipped creation returns a completed job without a registry job ID.
|
||||
|
||||
Wait for the returned job before opening or querying the view. In LanceDB
|
||||
Enterprise, creation uses SQL; ``job_type`` is ``"query"`` and ``id``
|
||||
identifies that query on this connection.
|
||||
@@ -3521,6 +3545,7 @@ class AsyncConnection(object):
|
||||
filter=where,
|
||||
limit=limit,
|
||||
with_no_data=with_no_data,
|
||||
exist_ok=exist_ok,
|
||||
auto_refresh_enabled=auto_refresh_enabled,
|
||||
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
|
||||
**native_options,
|
||||
|
||||
@@ -705,6 +705,7 @@ class LanceNamespaceDBConnection(DBConnection):
|
||||
with_no_data: bool = False,
|
||||
auto_refresh_enabled: Optional[bool] = None,
|
||||
auto_refresh_interval_seconds: Optional[float] = None,
|
||||
exist_ok: bool = False,
|
||||
) -> "MaterializedView":
|
||||
"""Define a materialized view over a table in the root namespace.
|
||||
See
|
||||
@@ -723,6 +724,7 @@ class LanceNamespaceDBConnection(DBConnection):
|
||||
with_no_data=with_no_data,
|
||||
auto_refresh_enabled=auto_refresh_enabled,
|
||||
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
|
||||
exist_ok=exist_ok,
|
||||
)
|
||||
).name
|
||||
)
|
||||
@@ -741,6 +743,7 @@ class LanceNamespaceDBConnection(DBConnection):
|
||||
with_no_data: bool = False,
|
||||
auto_refresh_enabled: Optional[bool] = None,
|
||||
auto_refresh_interval_seconds: Optional[float] = None,
|
||||
exist_ok: bool = False,
|
||||
) -> Job[None]:
|
||||
job = LOOP.run(
|
||||
self._inner.create_materialized_view_async(
|
||||
@@ -753,6 +756,7 @@ class LanceNamespaceDBConnection(DBConnection):
|
||||
with_no_data=with_no_data,
|
||||
auto_refresh_enabled=auto_refresh_enabled,
|
||||
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
|
||||
exist_ok=exist_ok,
|
||||
)
|
||||
)
|
||||
return Job(job)
|
||||
@@ -1352,6 +1356,7 @@ class AsyncLanceNamespaceDBConnection:
|
||||
with_no_data: bool = False,
|
||||
auto_refresh_enabled: Optional[bool] = None,
|
||||
auto_refresh_interval_seconds: Optional[float] = None,
|
||||
exist_ok: bool = False,
|
||||
) -> "AsyncMaterializedView":
|
||||
"""Define a materialized view over a table in the root namespace."""
|
||||
view = await self._inner.create_materialized_view(
|
||||
@@ -1364,6 +1369,7 @@ class AsyncLanceNamespaceDBConnection:
|
||||
with_no_data=with_no_data,
|
||||
auto_refresh_enabled=auto_refresh_enabled,
|
||||
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
|
||||
exist_ok=exist_ok,
|
||||
)
|
||||
# Reopen through the namespace so the view's table carries the
|
||||
# namespace client and pushdown configuration a bare inner table lacks.
|
||||
@@ -1381,6 +1387,7 @@ class AsyncLanceNamespaceDBConnection:
|
||||
with_no_data: bool = False,
|
||||
auto_refresh_enabled: Optional[bool] = None,
|
||||
auto_refresh_interval_seconds: Optional[float] = None,
|
||||
exist_ok: bool = False,
|
||||
) -> AsyncJob[None]:
|
||||
"""Submit materialized-view creation and return its job."""
|
||||
return await self._inner.create_materialized_view_async(
|
||||
@@ -1393,6 +1400,7 @@ class AsyncLanceNamespaceDBConnection:
|
||||
with_no_data=with_no_data,
|
||||
auto_refresh_enabled=auto_refresh_enabled,
|
||||
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
|
||||
exist_ok=exist_ok,
|
||||
)
|
||||
|
||||
async def open_materialized_view(self, name: str) -> "AsyncMaterializedView":
|
||||
|
||||
@@ -803,6 +803,7 @@ class RemoteDBConnection(DBConnection):
|
||||
where: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
with_no_data: bool = False,
|
||||
exist_ok: bool = False,
|
||||
auto_refresh_enabled: Optional[bool] = None,
|
||||
auto_refresh_interval_seconds: Optional[float] = None,
|
||||
) -> MaterializedView:
|
||||
@@ -817,6 +818,7 @@ class RemoteDBConnection(DBConnection):
|
||||
where=where,
|
||||
limit=limit,
|
||||
with_no_data=with_no_data,
|
||||
exist_ok=exist_ok,
|
||||
auto_refresh_enabled=auto_refresh_enabled,
|
||||
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
|
||||
)
|
||||
@@ -841,6 +843,7 @@ class RemoteDBConnection(DBConnection):
|
||||
where: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
with_no_data: bool = False,
|
||||
exist_ok: bool = False,
|
||||
auto_refresh_enabled: Optional[bool] = None,
|
||||
auto_refresh_interval_seconds: Optional[float] = None,
|
||||
) -> Job[None]:
|
||||
@@ -853,6 +856,7 @@ class RemoteDBConnection(DBConnection):
|
||||
where=where,
|
||||
limit=limit,
|
||||
with_no_data=with_no_data,
|
||||
exist_ok=exist_ok,
|
||||
auto_refresh_enabled=auto_refresh_enabled,
|
||||
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import pytest
|
||||
|
||||
import lancedb
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.parametrize("background", [False, True])
|
||||
@pytest.mark.parametrize("namespace", [False, True])
|
||||
@pytest.mark.parametrize("defining_query", [False, True])
|
||||
async def test_exist_ok_preserves_materialized_data(
|
||||
tmp_path, asynchronous, background, namespace, defining_query
|
||||
):
|
||||
source_args = (
|
||||
{"query": "SELECT * FROM source"} if defining_query else {"source": "source"}
|
||||
)
|
||||
replacement_args = {
|
||||
**(
|
||||
{"query": "SELECT * FROM missing"}
|
||||
if defining_query
|
||||
else {"source": "missing"}
|
||||
),
|
||||
"auto_refresh_enabled": True,
|
||||
"auto_refresh_interval_seconds": 1,
|
||||
}
|
||||
options = {"storage_options": {"new_table_enable_stable_row_ids": "true"}}
|
||||
if asynchronous:
|
||||
db = (
|
||||
lancedb.connect_namespace_async("dir", {"root": str(tmp_path)}, **options)
|
||||
if namespace
|
||||
else await lancedb.connect_async(tmp_path, **options)
|
||||
)
|
||||
source = await db.create_table("source", [{"id": 1}])
|
||||
original = await db.create_materialized_view(
|
||||
"saved", **source_args, exist_ok=True
|
||||
)
|
||||
definition = await original.definition()
|
||||
version = await original.table.version()
|
||||
await source.add([{"id": 2}])
|
||||
if background:
|
||||
job = await db.create_materialized_view_async(
|
||||
"saved", **replacement_args, exist_ok=True
|
||||
)
|
||||
await job.wait()
|
||||
kept = await db.open_materialized_view("saved")
|
||||
else:
|
||||
kept = await db.create_materialized_view(
|
||||
"saved", **replacement_args, exist_ok=True
|
||||
)
|
||||
assert (await kept.definition()).query == definition.query
|
||||
assert await kept.table.version() == version
|
||||
assert (await kept.table.to_arrow()).to_pylist() == [{"id": 1}]
|
||||
with pytest.raises(ValueError, match="already exists"):
|
||||
await db.create_materialized_view("saved", **source_args)
|
||||
else:
|
||||
db = (
|
||||
lancedb.connect_namespace("dir", {"root": str(tmp_path)}, **options)
|
||||
if namespace
|
||||
else lancedb.connect(tmp_path, **options)
|
||||
)
|
||||
source = db.create_table("source", [{"id": 1}])
|
||||
original = db.create_materialized_view("saved", **source_args, exist_ok=True)
|
||||
definition = original.definition
|
||||
version = original.table.version
|
||||
source.add([{"id": 2}])
|
||||
if background:
|
||||
job = db.create_materialized_view_async(
|
||||
"saved", **replacement_args, exist_ok=True
|
||||
)
|
||||
job.wait()
|
||||
kept = db.open_materialized_view("saved")
|
||||
else:
|
||||
kept = db.create_materialized_view(
|
||||
"saved", **replacement_args, exist_ok=True
|
||||
)
|
||||
assert kept.definition.query == definition.query
|
||||
assert kept.table.version == version
|
||||
assert kept.table.to_arrow().to_pylist() == [{"id": 1}]
|
||||
with pytest.raises(ValueError, match="already exists"):
|
||||
db.create_materialized_view("saved", **source_args)
|
||||
@@ -197,6 +197,7 @@ def test_sync_remote_create_uses_public_async_connection(query):
|
||||
where=None,
|
||||
limit=None,
|
||||
with_no_data=False,
|
||||
exist_ok=False,
|
||||
auto_refresh_enabled=None,
|
||||
auto_refresh_interval_seconds=None,
|
||||
):
|
||||
@@ -729,6 +730,7 @@ def test_remote_create_forwards_refresh_options_without_interval_default(options
|
||||
"filter": None,
|
||||
"limit": None,
|
||||
"with_no_data": False,
|
||||
"exist_ok": False,
|
||||
"auto_refresh_enabled": options.get("auto_refresh_enabled"),
|
||||
"auto_refresh_interval_seconds": options.get(
|
||||
"auto_refresh_interval_seconds"
|
||||
|
||||
@@ -507,7 +507,7 @@ impl Connection {
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[pyo3(signature = (name, source=None, projections=None, filter=None, limit=None, with_no_data=false, auto_refresh_enabled=None, auto_refresh_interval_seconds=None, *, query=None))]
|
||||
#[pyo3(signature = (name, source=None, projections=None, filter=None, limit=None, with_no_data=false, auto_refresh_enabled=None, auto_refresh_interval_seconds=None, *, query=None, exist_ok=false))]
|
||||
pub fn create_materialized_view(
|
||||
self_: PyRef<'_, Self>,
|
||||
name: String,
|
||||
@@ -519,6 +519,7 @@ impl Connection {
|
||||
auto_refresh_enabled: Option<bool>,
|
||||
auto_refresh_interval_seconds: Option<f64>,
|
||||
query: Option<String>,
|
||||
exist_ok: bool,
|
||||
) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.get_inner()?.clone();
|
||||
let builder = materialized_view_builder(
|
||||
@@ -529,7 +530,7 @@ impl Connection {
|
||||
projections.is_some() || filter.is_some() || limit.is_some(),
|
||||
)?;
|
||||
future_into_py(self_.py(), async move {
|
||||
let mut builder = builder;
|
||||
let mut builder = builder.exist_ok(exist_ok);
|
||||
if let Some(enabled) = auto_refresh_enabled {
|
||||
builder = builder.auto_refresh_enabled(enabled);
|
||||
}
|
||||
@@ -552,7 +553,7 @@ impl Connection {
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[pyo3(signature = (name, source=None, projections=None, filter=None, limit=None, with_no_data=false, auto_refresh_enabled=None, auto_refresh_interval_seconds=None, *, query=None))]
|
||||
#[pyo3(signature = (name, source=None, projections=None, filter=None, limit=None, with_no_data=false, auto_refresh_enabled=None, auto_refresh_interval_seconds=None, *, query=None, exist_ok=false))]
|
||||
pub fn create_materialized_view_async(
|
||||
self_: PyRef<'_, Self>,
|
||||
name: String,
|
||||
@@ -564,6 +565,7 @@ impl Connection {
|
||||
auto_refresh_enabled: Option<bool>,
|
||||
auto_refresh_interval_seconds: Option<f64>,
|
||||
query: Option<String>,
|
||||
exist_ok: bool,
|
||||
) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.get_inner()?.clone();
|
||||
let builder = materialized_view_builder(
|
||||
@@ -574,7 +576,7 @@ impl Connection {
|
||||
projections.is_some() || filter.is_some() || limit.is_some(),
|
||||
)?;
|
||||
future_into_py(self_.py(), async move {
|
||||
let mut builder = builder;
|
||||
let mut builder = builder.exist_ok(exist_ok);
|
||||
if let Some(enabled) = auto_refresh_enabled {
|
||||
builder = builder.auto_refresh_enabled(enabled);
|
||||
}
|
||||
|
||||
@@ -287,6 +287,8 @@ pub struct MaterializedViewInfo {
|
||||
#[doc(hidden)]
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct CreateMaterializedViewRequest {
|
||||
/// Preserve an existing materialized view without refreshing it.
|
||||
pub exist_ok: bool,
|
||||
/// Name of the new view.
|
||||
pub name: String,
|
||||
/// Namespace in which to create the view.
|
||||
@@ -1586,6 +1588,7 @@ pub struct CreateMaterializedViewBuilder {
|
||||
filter: Option<String>,
|
||||
limit: Option<u64>,
|
||||
with_no_data: bool,
|
||||
exist_ok: bool,
|
||||
auto_refresh_enabled: Option<bool>,
|
||||
auto_refresh_interval_seconds: Option<f64>,
|
||||
}
|
||||
@@ -1626,6 +1629,7 @@ impl CreateMaterializedViewBuilder {
|
||||
filter: None,
|
||||
limit: None,
|
||||
with_no_data: false,
|
||||
exist_ok: false,
|
||||
auto_refresh_enabled: None,
|
||||
auto_refresh_interval_seconds: None,
|
||||
}
|
||||
@@ -1678,6 +1682,35 @@ impl CreateMaterializedViewBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
/// Preserve an existing view's definition, data and refresh settings.
|
||||
/// Defaults to false. A skipped creation does not refresh the view.
|
||||
///
|
||||
/// ```no_run
|
||||
/// # #![recursion_limit = "256"]
|
||||
/// # async fn example(db: &lancedb::Connection) -> lancedb::Result<()> {
|
||||
/// db.create_materialized_view("saved", "source").exist_ok(true).execute().await?;
|
||||
/// # Ok(())
|
||||
/// # }
|
||||
/// ```
|
||||
pub fn exist_ok(mut self, exist_ok: bool) -> Self {
|
||||
self.exist_ok = exist_ok;
|
||||
self
|
||||
}
|
||||
|
||||
async fn existing_view(&self) -> Result<Option<MaterializedView>> {
|
||||
match self
|
||||
.connection
|
||||
.open_table(&self.name)
|
||||
.namespace(self.namespace.clone())
|
||||
.execute()
|
||||
.await
|
||||
{
|
||||
Ok(table) => MaterializedView::from_table(table).await.map(Some),
|
||||
Err(Error::TableNotFound { .. }) => Ok(None),
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
|
||||
/// Enable or disable automatic refresh in LanceDB Enterprise.
|
||||
/// When omitted, the server determines whether automatic refresh is enabled.
|
||||
/// LanceDB Embedded does not support automatic refresh.
|
||||
@@ -1759,6 +1792,7 @@ impl CreateMaterializedViewBuilder {
|
||||
.connection
|
||||
.database()
|
||||
.create_materialized_view_async(CreateMaterializedViewRequest {
|
||||
exist_ok: self.exist_ok,
|
||||
name: self.name.clone(),
|
||||
namespace_path: self.namespace.clone(),
|
||||
query: self.query()?,
|
||||
@@ -1793,6 +1827,11 @@ impl CreateMaterializedViewBuilder {
|
||||
|
||||
async fn execute_native(self) -> Result<MaterializedView> {
|
||||
crate::require_experimental_functions()?;
|
||||
if self.exist_ok
|
||||
&& let Some(view) = self.existing_view().await?
|
||||
{
|
||||
return Ok(view);
|
||||
}
|
||||
if self.auto_refresh_enabled == Some(true) || self.auto_refresh_interval_seconds.is_some() {
|
||||
return Err(Error::NotSupported {
|
||||
message: "automatic materialized-view refresh requires LanceDB Enterprise"
|
||||
@@ -1826,7 +1865,15 @@ impl CreateMaterializedViewBuilder {
|
||||
prepare_definition(&source, definition).await?
|
||||
}
|
||||
};
|
||||
let view = prepared.create_in(&self.namespace, &self.name).await?;
|
||||
let view = match prepared.create_in(&self.namespace, &self.name).await {
|
||||
Ok(view) => view,
|
||||
Err(Error::TableAlreadyExists { .. }) if self.exist_ok => {
|
||||
return self.existing_view().await?.ok_or_else(|| Error::Runtime {
|
||||
message: "materialized view disappeared after concurrent creation".into(),
|
||||
});
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
if !self.with_no_data {
|
||||
view.refresh().execute().await?;
|
||||
}
|
||||
|
||||
@@ -803,16 +803,24 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
|
||||
} else {
|
||||
format!(" WITH ({})", options.join(", "))
|
||||
};
|
||||
let if_not_exists = if request.exist_ok {
|
||||
" IF NOT EXISTS"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
let submission = client
|
||||
.submit_materialized_view_job(
|
||||
format!(
|
||||
"CREATE MATERIALIZED VIEW \"{name}\"{options} AS {}",
|
||||
"CREATE MATERIALIZED VIEW{if_not_exists} \"{name}\"{options} AS {}",
|
||||
request.query
|
||||
),
|
||||
request.namespace_path,
|
||||
)
|
||||
.await?;
|
||||
match submission {
|
||||
crate::remote::sql::materialized_view::MaterializedViewJobSubmission::AlreadyExists => {
|
||||
Ok(Job::new_done())
|
||||
}
|
||||
crate::remote::sql::materialized_view::MaterializedViewJobSubmission::Registry {
|
||||
job_id,
|
||||
} => Ok(Job::new(Box::new(RemoteJob::new(
|
||||
@@ -4865,6 +4873,7 @@ mod tests {
|
||||
for seconds in [0.0, -1.0, f64::NAN, f64::INFINITY] {
|
||||
let result = db
|
||||
.create_materialized_view_async(CreateMaterializedViewRequest {
|
||||
exist_ok: false,
|
||||
name: "adults".into(),
|
||||
namespace_path: vec![],
|
||||
query: "SELECT * FROM people".into(),
|
||||
|
||||
@@ -39,6 +39,7 @@ fn strings(batch: &RecordBatch, name: &str) -> Result<Vec<String>> {
|
||||
}
|
||||
|
||||
pub(in crate::remote) enum MaterializedViewJobSubmission {
|
||||
AlreadyExists,
|
||||
Registry { job_id: String },
|
||||
Query(Arc<Query>),
|
||||
}
|
||||
@@ -66,6 +67,7 @@ impl SqlClient {
|
||||
let mut reader: SendableRecordBatchStream = query.reader().await?;
|
||||
let mut rows = 0;
|
||||
let mut registry_job_id = None;
|
||||
let mut already_exists = false;
|
||||
while let Some(batch) = reader.try_next().await? {
|
||||
rows += batch.num_rows();
|
||||
if rows > 1 {
|
||||
@@ -74,6 +76,16 @@ impl SqlClient {
|
||||
.into(),
|
||||
});
|
||||
}
|
||||
if let Some(column) = batch.column_by_name("status") {
|
||||
let values = column
|
||||
.as_any()
|
||||
.downcast_ref::<StringArray>()
|
||||
.ok_or_else(|| Error::Runtime {
|
||||
message: "materialized-view submission status is not UTF-8".into(),
|
||||
})?;
|
||||
already_exists =
|
||||
values.len() == 1 && !values.is_null(0) && values.value(0) == "already_exists";
|
||||
}
|
||||
if let Some(column) = batch.column_by_name("job_id") {
|
||||
let values = column
|
||||
.as_any()
|
||||
@@ -100,6 +112,9 @@ impl SqlClient {
|
||||
message: "materialized-view submission must return exactly one status row".into(),
|
||||
});
|
||||
}
|
||||
if already_exists && registry_job_id.is_none() {
|
||||
return Ok(MaterializedViewJobSubmission::AlreadyExists);
|
||||
}
|
||||
let job_id = registry_job_id.ok_or_else(|| Error::Runtime {
|
||||
message: "server acknowledged an early materialized-view handle but returned no job_id"
|
||||
.into(),
|
||||
|
||||
@@ -1220,6 +1220,7 @@ async fn remote_create_uses_sql_and_preserves_options_and_namespace() {
|
||||
] {
|
||||
let job = db
|
||||
.create_materialized_view_async(CreateMaterializedViewRequest {
|
||||
exist_ok: false,
|
||||
name: "adults".into(),
|
||||
namespace_path: vec!["team".into(), "reports".into()],
|
||||
query: "SELECT * FROM people".into(),
|
||||
@@ -1302,6 +1303,7 @@ async fn remote_create_returns_the_registry_job_from_the_early_status_row() {
|
||||
|
||||
let job = db
|
||||
.create_materialized_view_async(CreateMaterializedViewRequest {
|
||||
exist_ok: false,
|
||||
name: "adults".into(),
|
||||
namespace_path: vec!["team".into(), "reports".into()],
|
||||
query: "SELECT * FROM people".into(),
|
||||
@@ -1329,6 +1331,63 @@ async fn remote_create_returns_the_registry_job_from_the_early_status_row() {
|
||||
shutdown.send(()).unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_create_if_not_exists_accepts_a_completed_noop_without_a_registry_job() {
|
||||
for defer in [false, true] {
|
||||
let service = TestSqlService {
|
||||
adaptive: true,
|
||||
defer,
|
||||
materialized_view_job_handle_supported: true,
|
||||
..Default::default()
|
||||
};
|
||||
let statement = "CREATE MATERIALIZED VIEW IF NOT EXISTS \"adults\" AS SELECT * FROM people";
|
||||
service.result_overrides.lock().unwrap().insert(
|
||||
statement.into(),
|
||||
RecordBatch::try_from_iter([
|
||||
(
|
||||
"status",
|
||||
Arc::new(StringArray::from(vec!["already_exists"])) as arrow_array::ArrayRef,
|
||||
),
|
||||
(
|
||||
"job_id",
|
||||
Arc::new(StringArray::from(vec![""])) as arrow_array::ArrayRef,
|
||||
),
|
||||
])
|
||||
.unwrap(),
|
||||
);
|
||||
let statements = service.statements.clone();
|
||||
let (client, shutdown) = start_sql_service(service, ClientConfig::default()).await;
|
||||
let db = RemoteDatabase::new_mock(|request| -> http::Response<String> {
|
||||
panic!(
|
||||
"a skipped creation must not poll a registry job: {}",
|
||||
request.url()
|
||||
);
|
||||
})
|
||||
.with_sql_client(client);
|
||||
let job = db
|
||||
.create_materialized_view_async(CreateMaterializedViewRequest {
|
||||
exist_ok: true,
|
||||
name: "adults".into(),
|
||||
namespace_path: vec!["team".into()],
|
||||
query: "SELECT * FROM people".into(),
|
||||
with_no_data: false,
|
||||
auto_refresh_enabled: None,
|
||||
auto_refresh_interval_seconds: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
if defer {
|
||||
assert!(job.id().is_none());
|
||||
} else {
|
||||
assert_eq!(job.job_type().as_deref(), Some("query"));
|
||||
}
|
||||
job.wait().await.unwrap();
|
||||
assert_eq!(job.status().await.unwrap(), "finished");
|
||||
assert_eq!(*statements.lock().unwrap(), [statement]);
|
||||
shutdown.send(()).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_create_registry_handle_reopens_describes_and_cancels() {
|
||||
let service = TestSqlService {
|
||||
@@ -1394,6 +1453,7 @@ async fn remote_create_registry_handle_reopens_describes_and_cancels() {
|
||||
|
||||
let job = db
|
||||
.create_materialized_view_async(CreateMaterializedViewRequest {
|
||||
exist_ok: false,
|
||||
name: "adults".into(),
|
||||
namespace_path: vec!["team".into(), "reports".into()],
|
||||
query: "SELECT * FROM people".into(),
|
||||
|
||||
@@ -2658,6 +2658,13 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
|
||||
.submit_materialized_view_job(statement, self.namespace.clone())
|
||||
.await?;
|
||||
match submission {
|
||||
crate::remote::sql::materialized_view::MaterializedViewJobSubmission::AlreadyExists => {
|
||||
Err(Error::Runtime {
|
||||
message:
|
||||
"REFRESH MATERIALIZED VIEW returned a create-only already_exists status"
|
||||
.into(),
|
||||
})
|
||||
}
|
||||
crate::remote::sql::materialized_view::MaterializedViewJobSubmission::Registry {
|
||||
job_id,
|
||||
} => Ok(Job::new_typed(Box::new(MaterializedViewRefreshJob {
|
||||
|
||||
Reference in New Issue
Block a user