diff --git a/python/python/tests/test_expr.py b/python/python/tests/test_expr.py index 6aa78943e..0eb6f8929 100644 --- a/python/python/tests/test_expr.py +++ b/python/python/tests/test_expr.py @@ -632,3 +632,101 @@ class TestExprBytesIntegration: .to_arrow() ) assert result.num_rows == 2 + + +# ── datetime / timezone integration for lit() (issue #3262) ────────────────── + + +class TestExprDatetimeTimezoneIntegration: + """Integration coverage for lit(datetime) against table timestamp columns. + + PyArrow stores naive timestamps as UTC wall-clock microseconds. Python's + datetime.timestamp() treats naive values as *local* time, which used to + shift lit(naive) by the host UTC offset and break equality filters on + non-UTC machines. These cases lock the expected semantics. + """ + + def test_both_naive_match(self, tmp_path): + """Table naive + lit naive with the same wall clock must match.""" + db = lancedb.connect(str(tmp_path / "naive")) + ts = datetime(2024, 7, 1, 10, 0, 0) + table = db.create_table( + "t", [{"id": 1, "ts": ts}, {"id": 2, "ts": datetime(2024, 7, 2, 10, 0, 0)}] + ) + result = table.search().where(col("ts") == lit(ts)).to_list() + assert len(result) == 1 + assert result[0]["id"] == 1 + + def test_both_same_timezone_match(self, tmp_path): + """Table UTC + lit UTC for the same instant must match.""" + db = lancedb.connect(str(tmp_path / "utc")) + ts = datetime(2024, 7, 1, 10, 0, 0, tzinfo=timezone.utc) + table = db.create_table( + "t", + pa.table( + { + "id": [1, 2], + "ts": pa.array( + [ts, datetime(2024, 7, 2, 10, 0, 0, tzinfo=timezone.utc)], + type=pa.timestamp("us", tz="UTC"), + ), + } + ), + ) + result = table.search().where(col("ts") == lit(ts)).to_list() + assert len(result) == 1 + assert result[0]["id"] == 1 + + def test_different_timezones_same_instant(self, tmp_path): + """UTC table row equals lit of the same instant in a different zone.""" + db = lancedb.connect(str(tmp_path / "diff_tz")) + ts_utc = datetime(2024, 7, 1, 10, 0, 0, tzinfo=timezone.utc) + # Same instant as 06:00 in UTC-4 + ts_est = datetime(2024, 7, 1, 6, 0, 0, tzinfo=timezone(timedelta(hours=-4))) + table = db.create_table( + "t", + pa.table( + { + "id": [1], + "ts": pa.array([ts_utc], type=pa.timestamp("us", tz="UTC")), + } + ), + ) + result = table.search().where(col("ts") == lit(ts_est)).to_list() + assert len(result) == 1 + assert result[0]["id"] == 1 + + def test_table_tz_literal_naive(self, tmp_path): + """UTC table + naive lit uses wall-clock equality (10:00 == 10:00 UTC).""" + db = lancedb.connect(str(tmp_path / "tz_naive")) + ts_utc = datetime(2024, 7, 1, 10, 0, 0, tzinfo=timezone.utc) + ts_naive = datetime(2024, 7, 1, 10, 0, 0) + table = db.create_table( + "t", + pa.table( + { + "id": [1], + "ts": pa.array([ts_utc], type=pa.timestamp("us", tz="UTC")), + } + ), + ) + result = table.search().where(col("ts") == lit(ts_naive)).to_list() + assert len(result) == 1 + assert result[0]["id"] == 1 + + def test_table_naive_literal_aware(self, tmp_path): + """Naive table + UTC lit with the same wall clock must match.""" + db = lancedb.connect(str(tmp_path / "naive_aware")) + ts_naive = datetime(2024, 7, 1, 10, 0, 0) + ts_utc = datetime(2024, 7, 1, 10, 0, 0, tzinfo=timezone.utc) + table = db.create_table("t", [{"id": 1, "ts": ts_naive}]) + result = table.search().where(col("ts") == lit(ts_utc)).to_list() + assert len(result) == 1 + assert result[0]["id"] == 1 + + def test_naive_lit_sql_is_wall_clock_not_local_shifted(self): + """Regression: naive lit must not apply the host local UTC offset.""" + ts = datetime(2024, 7, 1, 10, 0, 0) + sql = lit(ts).to_sql() + # Must encode 10:00 wall clock, not 10:00+local_offset. + assert "2024-07-01 10:00:00" in sql diff --git a/python/src/expr.rs b/python/src/expr.rs index 242e88b05..eae1d96ec 100644 --- a/python/src/expr.rs +++ b/python/src/expr.rs @@ -191,8 +191,27 @@ pub fn expr_lit(value: Bound<'_, PyAny>) -> PyResult { } // datetime.datetime is a subclass of datetime.date, so it must be checked first. + // + // Python's datetime.timestamp() treats *naive* datetimes as local wall time. + // PyArrow (and therefore Lance table storage) encodes naive timestamps as + // UTC wall-clock microseconds. Using .timestamp() for naive values therefore + // shifts the literal by the local UTC offset on non-UTC machines, so + // `col("ts") == lit(naive_dt)` fails against a table that holds the same + // naive value. Fix: treat naive datetimes as UTC wall clock (match Arrow); + // keep aware datetimes on the real .timestamp() path (correct epoch). if let Ok(dt) = value.cast::() { - let ts: f64 = dt.call_method0("timestamp")?.extract()?; + let ts: f64 = if dt.getattr("tzinfo")?.is_none() { + // Force UTC interpretation of the naive wall clock. + let utc = pyo3::types::PyModule::import(value.py(), "datetime")? + .getattr("timezone")? + .getattr("utc")?; + let kwargs = pyo3::types::PyDict::new(value.py()); + kwargs.set_item("tzinfo", utc)?; + let aware = dt.call_method("replace", (), Some(&kwargs))?; + aware.call_method0("timestamp")?.extract()? + } else { + dt.call_method0("timestamp")?.extract()? + }; let micros = (ts * 1_000_000.0).round() as i64; return Ok(PyExpr(df_lit(ScalarValue::TimestampMicrosecond( Some(micros),