fix(ToDatetime): accept pandas columns of datetime.date objects

CHANGES.rst

@@ -53,6 +53,11 @@ Bugfixes
 - The parallel coordinate plot created by :meth:`ParamSearch.show_results` could
   have incorrect tick labels in some cases. This has been fixed in :pr:`2215` by
   :user:`Jérôme Dockès <jeromedockes>`.
+- :class:`ToDatetime` (and therefore :class:`TableVectorizer`) now accepts pandas
+  columns containing ``datetime.date`` objects. Pandas stores those in an
+  ``object`` column, so they used to be rejected, whereas the equivalent polars
+  ``Date`` column was accepted.
+  :pr:`2231` by :user:`Sanjay Santhanam <Sanjays2402>`.
 
 Deprecations
 ------------

skrub/_to_datetime.py

@@ -1,3 +1,4 @@
+import datetime
 import warnings
 
 import pandas as pd
@@ -82,6 +83,28 @@ def _convert_time_zone_polars(col, time_zone):
             return col.dt.replace_time_zone("UTC").dt.convert_time_zone(time_zone)
 
 
+def _cast_date_objects(column):
+    """Convert a pandas object column holding ``datetime.date`` to Datetime.
+
+    Pandas has no dedicated dtype for ``datetime.date``, so such a column has
+    the ``object`` dtype and is neither a date nor a string column. Polars, on
+    the other hand, stores them as ``pl.Date``. Casting them here makes the
+    handling of the two dataframe libraries consistent. Columns that do not
+    contain only dates are returned unchanged.
+    """
+    if not (sbd.is_pandas(column) and sbd.is_object(column)):
+        return column
+    not_null = sbd.drop_nulls(column)
+    if sbd.shape(not_null)[0] == 0:
+        return column
+    if not all(type(value) is datetime.date for value in not_null.head(_SAMPLE_SIZE)):
+        return column
+    try:
+        return pd.to_datetime(column)
+    except Exception:
+        return column
+
+
 class ToDatetime(SingleColumnTransformer):
     """
     Parse datetimes represented as strings and return ``Datetime`` columns.
@@ -390,6 +413,7 @@ def fit_transform(self, column, y=None):
 
         self.all_outputs_ = [sbd.name(column)]
 
+        column = _cast_date_objects(column)
         if sbd.is_any_date(column):
             self.format_ = None
             self.output_dtype_ = sbd.dtype(column)

skrub/tests/test_to_datetime.py

@@ -1,4 +1,4 @@
-from datetime import timezone
+from datetime import date, timezone
 from functools import partial
 
 import numpy as np
@@ -215,11 +215,7 @@ def test_to_datetime_func(df_module, datetime_col):
     df_module.assert_column_equal(
         to_datetime(datetime_col), ToDatetime().fit_transform(datetime_col)
     )
-    cols = (
-        ("datetime-col",)
-        if df_module.name == "pandas"
-        else ("datetime-col", "date-col")
-    )
+    cols = ("datetime-col", "date-col")
     df_module.assert_frame_equal(
         to_datetime(df_module.example_dataframe),
         ApplyToCols(ToDatetime(), cols=cols).fit_transform(df_module.example_dataframe),
@@ -249,3 +245,30 @@ def test_specific_time_encoding():
         pd.Timestamp(1584226801, unit="s", tz=ZoneInfo("Europe/Paris")),
     ]
     assert _get_time_zone(pd.Series(name="dt", data=col)) == "Europe/Paris"
+
+
+def test_pandas_date_objects():
+    """A pandas object column of datetime.date is parsed, not rejected.
+
+    Non-regression test for
+    https://github.com/skrub-data/skrub/issues/2084
+    """
+    col = pd.Series([date(2002, 1, 1), None, date(2003, 2, 2)], name="when")
+    assert sbd.is_object(col)
+
+    to_dt = ToDatetime()
+    out = to_dt.fit_transform(col)
+    assert sbd.is_any_date(out)
+    assert out[0] == pd.Timestamp("2002-01-01")
+    assert pd.isna(out[1])
+
+    transformed = to_dt.transform(pd.Series([date(2005, 5, 5)], name="when"))
+    assert transformed[0] == pd.Timestamp("2005-05-05")
+
+
+def test_object_column_that_is_not_dates_is_still_rejected():
+    with pytest.raises(RejectColumn, match="Could not find a datetime format"):
+        ToDatetime().fit_transform(pd.Series(["hello", "world"], name="when"))
+
+    with pytest.raises(RejectColumn):
+        ToDatetime().fit_transform(pd.Series([date(2002, 1, 1), "hello"], name="when"))