From a075aa62f8cdd87ef666eea2f3d7555507e8d773 Mon Sep 17 00:00:00 2001 From: Igor Ganapolsky Date: Mon, 17 Aug 2026 10:48:02 -0700 Subject: [PATCH] fix(python): treat naive lit(datetime) as UTC wall clock (#3262) (#3775) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Fixes naive `lit(datetime)` equality filters against table timestamp columns on non-UTC hosts, and adds the integration matrix from #3262. ## Failure (before) On a machine in US Eastern (UTC−4 / EDT), with PyPI `lancedb==0.36.0`: ```python from datetime import datetime import lancedb from lancedb.expr import col, lit db = lancedb.connect("memory://") ts = datetime(2024, 7, 1, 10, 0, 0) # naive table = db.create_table("t", [{"id": 1, "ts": ts}]) rows = table.search().where(col("ts") == lit(ts)).to_list() # actual: [] (0 rows) # expected: 1 row ``` ### Root cause In `python/src/expr.rs`, `expr_lit` converted every `datetime` via Python's `.timestamp()`: - **naive** `.timestamp()` = local wall → UTC epoch (shifted by host offset) - **PyArrow naive** storage = UTC wall-clock microseconds (no local shift) So `lit(naive)` became `CAST('2024-07-01 14:00:00' AS TIMESTAMP)` on EDT while the table held `10:00:00`. ## After Naive datetimes are interpreted as UTC wall clock (`replace(tzinfo=timezone.utc).timestamp()`), matching Arrow storage. Aware datetimes still use `.timestamp()` (correct epoch). Same repro on this branch: **1 matching row**. ## Tests Added `TestExprDatetimeTimezoneIntegration` covering: | Case | Result | |------|--------| | both naive | match | | both same TZ (UTC) | match | | different TZs, same instant | match | | table TZ + naive lit | match (wall clock) | | table naive + aware lit | match | | naive lit SQL is wall clock, not local-shifted | asserts `10:00:00` in SQL | ### Verification ```bash cd python maturin develop pytest python/tests/test_expr.py -v ``` **102 passed** (full `test_expr.py`, including the 6 new cases). Closes #3262 --------- Co-authored-by: Will Jones Co-authored-by: Claude Opus 5 (1M context) --- python/python/tests/test_expr.py | 98 ++++++++++++++++++++++++++++++++ python/src/expr.rs | 21 ++++++- 2 files changed, 118 insertions(+), 1 deletion(-) diff --git a/python/python/tests/test_expr.py b/python/python/tests/test_expr.py index 6aa78943e..0eb6f8929 100644 --- a/python/python/tests/test_expr.py +++ b/python/python/tests/test_expr.py @@ -632,3 +632,101 @@ class TestExprBytesIntegration: .to_arrow() ) assert result.num_rows == 2 + + +# ── datetime / timezone integration for lit() (issue #3262) ────────────────── + + +class TestExprDatetimeTimezoneIntegration: + """Integration coverage for lit(datetime) against table timestamp columns. + + PyArrow stores naive timestamps as UTC wall-clock microseconds. Python's + datetime.timestamp() treats naive values as *local* time, which used to + shift lit(naive) by the host UTC offset and break equality filters on + non-UTC machines. These cases lock the expected semantics. + """ + + def test_both_naive_match(self, tmp_path): + """Table naive + lit naive with the same wall clock must match.""" + db = lancedb.connect(str(tmp_path / "naive")) + ts = datetime(2024, 7, 1, 10, 0, 0) + table = db.create_table( + "t", [{"id": 1, "ts": ts}, {"id": 2, "ts": datetime(2024, 7, 2, 10, 0, 0)}] + ) + result = table.search().where(col("ts") == lit(ts)).to_list() + assert len(result) == 1 + assert result[0]["id"] == 1 + + def test_both_same_timezone_match(self, tmp_path): + """Table UTC + lit UTC for the same instant must match.""" + db = lancedb.connect(str(tmp_path / "utc")) + ts = datetime(2024, 7, 1, 10, 0, 0, tzinfo=timezone.utc) + table = db.create_table( + "t", + pa.table( + { + "id": [1, 2], + "ts": pa.array( + [ts, datetime(2024, 7, 2, 10, 0, 0, tzinfo=timezone.utc)], + type=pa.timestamp("us", tz="UTC"), + ), + } + ), + ) + result = table.search().where(col("ts") == lit(ts)).to_list() + assert len(result) == 1 + assert result[0]["id"] == 1 + + def test_different_timezones_same_instant(self, tmp_path): + """UTC table row equals lit of the same instant in a different zone.""" + db = lancedb.connect(str(tmp_path / "diff_tz")) + ts_utc = datetime(2024, 7, 1, 10, 0, 0, tzinfo=timezone.utc) + # Same instant as 06:00 in UTC-4 + ts_est = datetime(2024, 7, 1, 6, 0, 0, tzinfo=timezone(timedelta(hours=-4))) + table = db.create_table( + "t", + pa.table( + { + "id": [1], + "ts": pa.array([ts_utc], type=pa.timestamp("us", tz="UTC")), + } + ), + ) + result = table.search().where(col("ts") == lit(ts_est)).to_list() + assert len(result) == 1 + assert result[0]["id"] == 1 + + def test_table_tz_literal_naive(self, tmp_path): + """UTC table + naive lit uses wall-clock equality (10:00 == 10:00 UTC).""" + db = lancedb.connect(str(tmp_path / "tz_naive")) + ts_utc = datetime(2024, 7, 1, 10, 0, 0, tzinfo=timezone.utc) + ts_naive = datetime(2024, 7, 1, 10, 0, 0) + table = db.create_table( + "t", + pa.table( + { + "id": [1], + "ts": pa.array([ts_utc], type=pa.timestamp("us", tz="UTC")), + } + ), + ) + result = table.search().where(col("ts") == lit(ts_naive)).to_list() + assert len(result) == 1 + assert result[0]["id"] == 1 + + def test_table_naive_literal_aware(self, tmp_path): + """Naive table + UTC lit with the same wall clock must match.""" + db = lancedb.connect(str(tmp_path / "naive_aware")) + ts_naive = datetime(2024, 7, 1, 10, 0, 0) + ts_utc = datetime(2024, 7, 1, 10, 0, 0, tzinfo=timezone.utc) + table = db.create_table("t", [{"id": 1, "ts": ts_naive}]) + result = table.search().where(col("ts") == lit(ts_utc)).to_list() + assert len(result) == 1 + assert result[0]["id"] == 1 + + def test_naive_lit_sql_is_wall_clock_not_local_shifted(self): + """Regression: naive lit must not apply the host local UTC offset.""" + ts = datetime(2024, 7, 1, 10, 0, 0) + sql = lit(ts).to_sql() + # Must encode 10:00 wall clock, not 10:00+local_offset. + assert "2024-07-01 10:00:00" in sql diff --git a/python/src/expr.rs b/python/src/expr.rs index 242e88b05..eae1d96ec 100644 --- a/python/src/expr.rs +++ b/python/src/expr.rs @@ -191,8 +191,27 @@ pub fn expr_lit(value: Bound<'_, PyAny>) -> PyResult { } // datetime.datetime is a subclass of datetime.date, so it must be checked first. + // + // Python's datetime.timestamp() treats *naive* datetimes as local wall time. + // PyArrow (and therefore Lance table storage) encodes naive timestamps as + // UTC wall-clock microseconds. Using .timestamp() for naive values therefore + // shifts the literal by the local UTC offset on non-UTC machines, so + // `col("ts") == lit(naive_dt)` fails against a table that holds the same + // naive value. Fix: treat naive datetimes as UTC wall clock (match Arrow); + // keep aware datetimes on the real .timestamp() path (correct epoch). if let Ok(dt) = value.cast::() { - let ts: f64 = dt.call_method0("timestamp")?.extract()?; + let ts: f64 = if dt.getattr("tzinfo")?.is_none() { + // Force UTC interpretation of the naive wall clock. + let utc = pyo3::types::PyModule::import(value.py(), "datetime")? + .getattr("timezone")? + .getattr("utc")?; + let kwargs = pyo3::types::PyDict::new(value.py()); + kwargs.set_item("tzinfo", utc)?; + let aware = dt.call_method("replace", (), Some(&kwargs))?; + aware.call_method0("timestamp")?.extract()? + } else { + dt.call_method0("timestamp")?.extract()? + }; let micros = (ts * 1_000_000.0).round() as i64; return Ok(PyExpr(df_lit(ScalarValue::TimestampMicrosecond( Some(micros),