diff --git a/docs/web/ingestion/create-table.mdx b/docs/web/ingestion/create-table.mdx index 7d4363be7..0921de22e 100644 --- a/docs/web/ingestion/create-table.mdx +++ b/docs/web/ingestion/create-table.mdx @@ -148,9 +148,11 @@ non-nullable column, or provide actual nulls for it, ingestion fails; nullable c omitted or written with null values. Without an explicit schema, Python infers list-like vector values as fixed-size `float32` vector fields from the observed dimension. -For Python ingest, malformed vector values fail by default. If you expect occasional wrong-length, -null, or NaN vectors, choose an `on_bad_vectors` policy: `"drop"` removes those rows, `"fill"` writes -`fill_value`, and `"null"` writes nulls. +For Python ingestion, malformed vectors fail by default (`on_bad_vectors="error"`). This includes +vectors with incorrect sizes, NaNs, or null components, including pandas NaNs converted to Arrow +nulls. Use `"drop"` to remove affected rows, or `"null"` to replace affected vectors with nulls. +Use `"fill"` to replace NaN or null components with `fill_value` while preserving valid components. +This also pads short vectors with `fill_value` and truncates long vectors to the expected dimension. ### From an Arrow Table {#from-an-arrow-table} You can also create LanceDB tables directly from Arrow tables. diff --git a/python/python/lancedb/db.py b/python/python/lancedb/db.py index a74763afb..252051c09 100644 --- a/python/python/lancedb/db.py +++ b/python/python/lancedb/db.py @@ -422,10 +422,13 @@ class DBConnection(EnforceOverrides): it will not add the provided data but will validate against any schema that's specified. on_bad_vectors: str, default "error" - What to do if any of the vectors are not the same size or contains NaNs. - One of "error", "drop", "fill". + How to handle vectors with incorrect sizes, NaNs, or null elements. + One of "error", "drop", "fill", "null". fill_value: float The value to use when filling vectors. Only used if on_bad_vectors="fill". + NaN or null elements and missing trailing elements are replaced with this + value. Valid elements are preserved, and vectors longer than the expected + dimension are truncated. storage_options: dict, optional Additional options for the storage backend. Options already set on the connection will be inherited by the table, but can be overridden here. @@ -2521,10 +2524,13 @@ class AsyncConnection(object): it will not add the provided data but will validate against any schema that's specified. on_bad_vectors: str, default "error" - What to do if any of the vectors are not the same size or contains NaNs. - One of "error", "drop", "fill". + How to handle vectors with incorrect sizes, NaNs, or null elements. + One of "error", "drop", "fill", "null". fill_value: float The value to use when filling vectors. Only used if on_bad_vectors="fill". + NaN or null elements and missing trailing elements are replaced with this + value. Valid elements are preserved, and vectors longer than the expected + dimension are truncated. storage_options: dict, optional Additional options for the storage backend. Options already set on the connection will be inherited by the table, but can be overridden here.