Skip to content

chore: Simplify PandasLikeDataFrame|DaskLazyFrame.join method #2511

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 22 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
cf7c7fe
chore: Simplify PandasLikeDataFrame and DaskLazyFrame join
FBruzzesi May 5, 2025
180a35d
format
FBruzzesi May 5, 2025
d9e4f56
Merge branch 'main' into chore/simplify-join
FBruzzesi May 7, 2025
047208a
cleanup signature, overload still fails
FBruzzesi May 7, 2025
b17b216
minors
FBruzzesi May 8, 2025
b21403a
move pragma: no cover
FBruzzesi May 8, 2025
5e8db52
rename from _<strategy>_join to _join_<strategy>
FBruzzesi May 9, 2025
0fa2578
Merge branch 'main' into chore/simplify-join
FBruzzesi May 9, 2025
1c1daae
refactor: Rearrange `BaseFrame.join`
dangotbanned May 10, 2025
cf830d3
Merge remote-tracking branch 'upstream/main' into chore/simplify-join
dangotbanned May 10, 2025
ae64d74
low hanging feedback
FBruzzesi May 10, 2025
3cd567f
use left join in anti before filtering
FBruzzesi May 10, 2025
6068da1
fix Modin anti join
FBruzzesi May 10, 2025
a0fca3d
factor out with _join_filter_rename helper method
FBruzzesi May 10, 2025
01cb3f6
Merge branch 'main' into chore/simplify-join
FBruzzesi May 16, 2025
3ade37f
solve conflicts
FBruzzesi May 24, 2025
622a578
Merge branch 'main' into chore/simplify-join
dangotbanned May 24, 2025
23adb84
Merge branch 'main' into chore/simplify-join
FBruzzesi Jun 1, 2025
de09b93
Merge branch 'main' into chore/simplify-join
FBruzzesi Jun 4, 2025
84e32bc
Merge branch 'main' into chore/simplify-join
FBruzzesi Jun 13, 2025
b43a72e
Merge branch 'main' into chore/simplify-join
FBruzzesi Jun 19, 2025
6839448
Merge branch 'main' into chore/simplify-join
FBruzzesi Jun 20, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion narwhals/_compliant/dataframe.py
Original file line number Diff line number Diff line change
Expand Up @@ -331,7 +331,7 @@ def join(
self,
other: Self,
*,
how: Literal["left", "inner", "cross", "anti", "semi"],
how: JoinStrategy,
left_on: Sequence[str] | None,
right_on: Sequence[str] | None,
suffix: str,
Expand Down
247 changes: 139 additions & 108 deletions narwhals/_dask/dataframe.py
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,130 @@ def sort(self, *by: str, descending: bool | Sequence[bool], nulls_last: bool) ->
self.native.sort_values(list(by), ascending=ascending, na_position=position)
)

def join( # noqa: C901
def _join_inner(
self, other: Self, *, left_on: Sequence[str], right_on: Sequence[str], suffix: str
) -> Self:
return self._with_native(
self.native.merge(
other.native,
left_on=left_on,
right_on=right_on,
how="inner",
suffixes=("", suffix),
)
)

def _join_left(
self, other: Self, *, left_on: Sequence[str], right_on: Sequence[str], suffix: str
) -> Self:
result_native = self.native.merge(
other.native,
how="left",
left_on=left_on,
right_on=right_on,
suffixes=("", suffix),
)
extra = [
right_key if right_key not in self.columns else f"{right_key}{suffix}"
for left_key, right_key in zip(left_on, right_on)
if right_key != left_key
]
return self._with_native(result_native.drop(columns=extra))

def _join_full(
self, other: Self, *, left_on: Sequence[str], right_on: Sequence[str], suffix: str
) -> Self:
# dask does not retain keys post-join
# we must append the suffix to each key before-hand

right_on_mapper = _remap_full_join_keys(left_on, right_on, suffix)
other_native = other.native.rename(columns=right_on_mapper)
check_column_names_are_unique(other_native.columns)
right_suffixed = list(right_on_mapper.values())
return self._with_native(
self.native.merge(
other_native,
left_on=left_on,
right_on=right_suffixed,
how="outer",
suffixes=("", suffix),
)
)

def _join_cross(self, other: Self, *, suffix: str) -> Self:
key_token = generate_temporary_column_name(
n_bytes=8, columns=(*self.columns, *other.columns)
)
return self._with_native(
self.native.assign(**{key_token: 0})
.merge(
other.native.assign(**{key_token: 0}),
how="inner",
left_on=key_token,
right_on=key_token,
suffixes=("", suffix),
)
.drop(columns=key_token)
)

def _join_semi(
self, other: Self, *, left_on: Sequence[str], right_on: Sequence[str]
) -> Self:
other_native = self._join_filter_rename(
other=other,
columns_to_select=list(right_on),
columns_mapping=dict(zip(right_on, left_on)),
)
return self._with_native(
self.native.merge(
other_native, how="inner", left_on=left_on, right_on=left_on
)
)

def _join_anti(
self, other: Self, *, left_on: Sequence[str], right_on: Sequence[str]
) -> Self:
indicator_token = generate_temporary_column_name(
n_bytes=8, columns=(*self.columns, *other.columns)
)
other_native = self._join_filter_rename(
other=other,
columns_to_select=list(right_on),
columns_mapping=dict(zip(right_on, left_on)),
)
df = self.native.merge(
other_native,
how="left",
indicator=indicator_token, # pyright: ignore[reportArgumentType]
left_on=left_on,
right_on=left_on,
)
return self._with_native(
df[df[indicator_token] == "left_only"].drop(columns=[indicator_token])
)

def _join_filter_rename(
self, other: Self, columns_to_select: list[str], columns_mapping: dict[str, str]
) -> dd.DataFrame:
"""Helper function to avoid creating extra columns and row duplication.

Used in `"anti"` and `"semi`" join's.

Notice that a native object is returned.
"""
return (
select_columns_by_name(
other.native,
column_names=columns_to_select,
backend_version=self._backend_version,
implementation=self._implementation,
)
# rename to avoid creating extra columns in join
.rename(columns=columns_mapping)
.drop_duplicates()
)

def join(
self,
other: Self,
*,
Expand All @@ -254,122 +377,30 @@ def join( # noqa: C901
suffix: str,
) -> Self:
if how == "cross":
key_token = generate_temporary_column_name(
n_bytes=8, columns=[*self.columns, *other.columns]
)

return self._with_native(
self.native.assign(**{key_token: 0})
.merge(
other.native.assign(**{key_token: 0}),
how="inner",
left_on=key_token,
right_on=key_token,
suffixes=("", suffix),
)
.drop(columns=key_token)
)
return self._join_cross(other=other, suffix=suffix)

if how == "anti":
indicator_token = generate_temporary_column_name(
n_bytes=8, columns=[*self.columns, *other.columns]
)
if left_on is None or right_on is None: # pragma: no cover
raise ValueError(left_on, right_on)

if right_on is None: # pragma: no cover
msg = "`right_on` cannot be `None` in anti-join"
raise TypeError(msg)
other_native = (
select_columns_by_name(
other.native,
list(right_on),
self._backend_version,
self._implementation,
)
.rename( # rename to avoid creating extra columns in join
columns=dict(zip(right_on, left_on)) # type: ignore[arg-type]
)
.drop_duplicates()
if how == "inner":
return self._join_inner(
other=other, left_on=left_on, right_on=right_on, suffix=suffix
)
df = self.native.merge(
other_native,
how="outer",
indicator=indicator_token, # pyright: ignore[reportArgumentType]
left_on=left_on,
right_on=left_on,
)
return self._with_native(
df[df[indicator_token] == "left_only"].drop(columns=[indicator_token])
)

if how == "anti":
return self._join_anti(other=other, left_on=left_on, right_on=right_on)
if how == "semi":
if right_on is None: # pragma: no cover
msg = "`right_on` cannot be `None` in semi-join"
raise TypeError(msg)
other_native = (
select_columns_by_name(
other.native,
list(right_on),
self._backend_version,
self._implementation,
)
.rename( # rename to avoid creating extra columns in join
columns=dict(zip(right_on, left_on)) # type: ignore[arg-type]
)
.drop_duplicates() # avoids potential rows duplication from inner join
)
return self._with_native(
self.native.merge(
other_native, how="inner", left_on=left_on, right_on=left_on
)
)

return self._join_semi(other=other, left_on=left_on, right_on=right_on)
if how == "left":
result_native = self.native.merge(
other.native,
how="left",
left_on=left_on,
right_on=right_on,
suffixes=("", suffix),
return self._join_left(
other=other, left_on=left_on, right_on=right_on, suffix=suffix
)
extra = []
for left_key, right_key in zip(left_on, right_on): # type: ignore[arg-type]
if right_key != left_key and right_key not in self.columns:
extra.append(right_key)
elif right_key != left_key:
extra.append(f"{right_key}_right")
return self._with_native(result_native.drop(columns=extra))

if how == "full":
# dask does not retain keys post-join
# we must append the suffix to each key before-hand

# help mypy
assert left_on is not None # noqa: S101
assert right_on is not None # noqa: S101

right_on_mapper = _remap_full_join_keys(left_on, right_on, suffix)
other_native = other.native.rename(columns=right_on_mapper)
check_column_names_are_unique(other_native.columns)
right_on = list(right_on_mapper.values()) # we now have the suffixed keys
return self._with_native(
self.native.merge(
other_native,
left_on=left_on,
right_on=right_on,
how="outer",
suffixes=("", suffix),
)
return self._join_full(
other=other, left_on=left_on, right_on=right_on, suffix=suffix
)

return self._with_native(
self.native.merge(
other.native,
left_on=left_on,
right_on=right_on,
how=how,
suffixes=("", suffix),
)
)
msg = f"Unreachable code, got unexpected join method: {how}" # pragma: no cover
raise AssertionError(msg)

def join_asof(
self,
Expand Down
Loading
Loading