diff --git a/doc/source/whatsnew/v3.1.0.rst b/doc/source/whatsnew/v3.1.0.rst index 86d7507553714..68899cd12543b 100644 --- a/doc/source/whatsnew/v3.1.0.rst +++ b/doc/source/whatsnew/v3.1.0.rst @@ -482,6 +482,7 @@ Groupby/resample/rolling Reshaping ^^^^^^^^^ - Bug in :func:`concat` raising ``InvalidIndexError`` when ``keys`` or the concatenated objects' index was an overlapping :class:`IntervalIndex` (:issue:`64825`) +- Bug in :func:`concat` where the frequency of the combined :class:`DatetimeIndex` depended on the order of the concatenated objects when they shared a non-inferable frequency such as :class:`~pandas.tseries.offsets.CustomBusinessDay` (:issue:`64253`) - Bug in :func:`merge` where merging on a :class:`MultiIndex` containing ``NaN`` values mapped ``NaN`` keys to the last level value instead of ``NaN`` (:issue:`64492`) - Bug in :meth:`DataFrame.melt` where ``var_name`` colliding with an ``id_vars`` column or ``value_name`` silently overwrote the affected column data instead of raising (:issue:`65654`) - Bug in :meth:`DataFrame.pivot_table` with ``margins=True`` raising ``TypeError`` when ``values`` has an :class:`ExtensionDtype` that cannot hold ``NA`` (e.g. :class:`IntervalDtype` with an integer subtype) and no ``columns`` were specified (:issue:`55484`) diff --git a/pandas/core/indexes/api.py b/pandas/core/indexes/api.py index 89666f763d42b..55ea3c639f866 100644 --- a/pandas/core/indexes/api.py +++ b/pandas/core/indexes/api.py @@ -23,7 +23,10 @@ maybe_sequence_to_range, ) from pandas.core.indexes.category import CategoricalIndex -from pandas.core.indexes.datetimes import DatetimeIndex +from pandas.core.indexes.datetimes import ( + DatetimeIndex, + date_range, +) from pandas.core.indexes.interval import IntervalIndex from pandas.core.indexes.multi import MultiIndex from pandas.core.indexes.period import PeriodIndex @@ -145,6 +148,10 @@ def _get_combined_index( if sort and sort is not lib.no_default: index = safe_sort_index(index) + if not intersect: + # GH#64253: keep the combined-index freq order-independent even when an + # explicit sort rebuilds the index (which would otherwise drop freq). + index = _restore_shared_datetime_freq(index, indexes) return index @@ -182,6 +189,35 @@ def safe_sort_index(index: Index) -> Index: return index +def _restore_shared_datetime_freq(result: Index, indexes: list[Index]) -> Index: + """ + Reattach a shared, non-inferable frequency to a combined ``DatetimeIndex``. + + A pairwise union of ``DatetimeIndex`` objects can drop a shared frequency in + an intermediate step (e.g. a ``CustomBusinessDay`` freq, which cannot be + recovered by ``inferred_freq``), leaving ``result.freq`` dependent on the + order of the inputs (GH#64253). If every input shares a single frequency and + ``result`` is exactly regular with respect to it, reattach that frequency so + the combined index does not depend on the input order. ``result`` is + returned unchanged (values and dtype untouched) in every other case. + """ + if ( + not isinstance(result, DatetimeIndex) + or result.freq is not None + or len(result) < 3 + ): + return result + freqs = {getattr(idx, "freq", None) for idx in indexes} + if len(freqs) != 1: + return result + freq = next(iter(freqs)) + if freq is None: + return result + if result.equals(date_range(result[0], result[-1], freq=freq, unit=result.unit)): + return result._with_freq(freq) + return result + + def union_indexes(indexes, sort: bool | lib.NoDefault = True) -> Index: """ Return the union of indexes. @@ -246,6 +282,11 @@ def union_indexes(indexes, sort: bool | lib.NoDefault = True) -> Index: for other in indexes[1:]: result = result.union(other, sort=None if sort else False) + + if num_dtis == len(indexes): + # GH#64253: a pairwise union can drop a shared, non-inferable freq + # in an intermediate step, making result.freq order-dependent. + result = _restore_shared_datetime_freq(result, indexes) return result elif kind == "array": diff --git a/pandas/tests/reshape/concat/test_datetimes.py b/pandas/tests/reshape/concat/test_datetimes.py index dee0dda326129..a4e298900effb 100644 --- a/pandas/tests/reshape/concat/test_datetimes.py +++ b/pandas/tests/reshape/concat/test_datetimes.py @@ -136,6 +136,25 @@ def test_concat_datetimeindex_freq_mixed_unit(self): # Not checked by assert_index_equal assert result.freq == "s" + def test_concat_datetimeindex_freq_order_independent(self): + # GH#64253 - the freq of the combined index must not depend on the + # order of the inputs when they share a (non-inferable) freq and the + # union is exactly regular with respect to it. + freq = pd.offsets.CustomBusinessDay(holidays=["2020-01-10"]) + idx1 = date_range("2020-01-01", periods=5, freq=freq) + idx2 = date_range(start=idx1[3], periods=5, freq=freq) + idx3 = date_range(start=idx2[3], periods=5, freq=freq) + s1 = Series(1, index=idx1, name="S1") + s2 = Series(1, index=idx2, name="S2") + s3 = Series(1, index=idx3, name="S3") + + result_123 = concat([s1, s2, s3], axis=1, sort=True) + result_132 = concat([s1, s3, s2], axis=1, sort=True) + + assert result_123.index.freq == freq + assert result_132.index.freq == freq + tm.assert_index_equal(result_123.index, result_132.index) + def test_concat_datetimeindex_tz_convert_freq(self): # GH#41585 - concat after tz_convert should not raise when # the converted timestamps no longer conform to the original freq