Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions doc/source/whatsnew/v3.1.0.rst
Original file line number Diff line number Diff line change
Expand Up @@ -482,6 +482,7 @@ Groupby/resample/rolling
Reshaping
^^^^^^^^^
- Bug in :func:`concat` raising ``InvalidIndexError`` when ``keys`` or the concatenated objects' index was an overlapping :class:`IntervalIndex` (:issue:`64825`)
- Bug in :func:`concat` where the frequency of the combined :class:`DatetimeIndex` depended on the order of the concatenated objects when they shared a non-inferable frequency such as :class:`~pandas.tseries.offsets.CustomBusinessDay` (:issue:`64253`)
- Bug in :func:`merge` where merging on a :class:`MultiIndex` containing ``NaN`` values mapped ``NaN`` keys to the last level value instead of ``NaN`` (:issue:`64492`)
- Bug in :meth:`DataFrame.melt` where ``var_name`` colliding with an ``id_vars`` column or ``value_name`` silently overwrote the affected column data instead of raising (:issue:`65654`)
- Bug in :meth:`DataFrame.pivot_table` with ``margins=True`` raising ``TypeError`` when ``values`` has an :class:`ExtensionDtype` that cannot hold ``NA`` (e.g. :class:`IntervalDtype` with an integer subtype) and no ``columns`` were specified (:issue:`55484`)
Expand Down
43 changes: 42 additions & 1 deletion pandas/core/indexes/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,10 @@
maybe_sequence_to_range,
)
from pandas.core.indexes.category import CategoricalIndex
from pandas.core.indexes.datetimes import DatetimeIndex
from pandas.core.indexes.datetimes import (
DatetimeIndex,
date_range,
)
from pandas.core.indexes.interval import IntervalIndex
from pandas.core.indexes.multi import MultiIndex
from pandas.core.indexes.period import PeriodIndex
Expand Down Expand Up @@ -145,6 +148,10 @@ def _get_combined_index(

if sort and sort is not lib.no_default:
index = safe_sort_index(index)
if not intersect:
# GH#64253: keep the combined-index freq order-independent even when an
# explicit sort rebuilds the index (which would otherwise drop freq).
index = _restore_shared_datetime_freq(index, indexes)
return index


Expand Down Expand Up @@ -182,6 +189,35 @@ def safe_sort_index(index: Index) -> Index:
return index


def _restore_shared_datetime_freq(result: Index, indexes: list[Index]) -> Index:
"""
Reattach a shared, non-inferable frequency to a combined ``DatetimeIndex``.

A pairwise union of ``DatetimeIndex`` objects can drop a shared frequency in
an intermediate step (e.g. a ``CustomBusinessDay`` freq, which cannot be
recovered by ``inferred_freq``), leaving ``result.freq`` dependent on the
order of the inputs (GH#64253). If every input shares a single frequency and
``result`` is exactly regular with respect to it, reattach that frequency so
the combined index does not depend on the input order. ``result`` is
returned unchanged (values and dtype untouched) in every other case.
"""
if (
not isinstance(result, DatetimeIndex)
or result.freq is not None
or len(result) < 3
):
return result
freqs = {getattr(idx, "freq", None) for idx in indexes}
if len(freqs) != 1:
return result
freq = next(iter(freqs))
if freq is None:
return result
if result.equals(date_range(result[0], result[-1], freq=freq, unit=result.unit)):
return result._with_freq(freq)
return result


def union_indexes(indexes, sort: bool | lib.NoDefault = True) -> Index:
"""
Return the union of indexes.
Expand Down Expand Up @@ -246,6 +282,11 @@ def union_indexes(indexes, sort: bool | lib.NoDefault = True) -> Index:

for other in indexes[1:]:
result = result.union(other, sort=None if sort else False)

if num_dtis == len(indexes):
# GH#64253: a pairwise union can drop a shared, non-inferable freq
# in an intermediate step, making result.freq order-dependent.
result = _restore_shared_datetime_freq(result, indexes)
return result

elif kind == "array":
Expand Down
19 changes: 19 additions & 0 deletions pandas/tests/reshape/concat/test_datetimes.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,25 @@ def test_concat_datetimeindex_freq_mixed_unit(self):
# Not checked by assert_index_equal
assert result.freq == "s"

def test_concat_datetimeindex_freq_order_independent(self):
# GH#64253 - the freq of the combined index must not depend on the
# order of the inputs when they share a (non-inferable) freq and the
# union is exactly regular with respect to it.
freq = pd.offsets.CustomBusinessDay(holidays=["2020-01-10"])
idx1 = date_range("2020-01-01", periods=5, freq=freq)
idx2 = date_range(start=idx1[3], periods=5, freq=freq)
idx3 = date_range(start=idx2[3], periods=5, freq=freq)
s1 = Series(1, index=idx1, name="S1")
s2 = Series(1, index=idx2, name="S2")
s3 = Series(1, index=idx3, name="S3")

result_123 = concat([s1, s2, s3], axis=1, sort=True)
result_132 = concat([s1, s3, s2], axis=1, sort=True)

assert result_123.index.freq == freq
assert result_132.index.freq == freq
tm.assert_index_equal(result_123.index, result_132.index)

def test_concat_datetimeindex_tz_convert_freq(self):
# GH#41585 - concat after tz_convert should not raise when
# the converted timestamps no longer conform to the original freq
Expand Down