| 1 | From cebc07fed0707457487656836e627a154416d252 Mon Sep 17 00:00:00 2001
|
| 2 | From: Palash Lalwani <[email protected]>
|
| 3 | Date: Wed, 22 Jul 2026 21:24:52 +0530
|
| 4 | Subject: [PATCH] fix: don't pass grouping columns to
|
| 5 | groupby().parallel_apply() on pandas >= 3.0
|
| 6 |
|
| 7 | Since pandas 3.0, DataFrameGroupBy.apply operates on _obj_with_exclusions, so the
|
| 8 | grouping columns are no longer passed to the applied function. Iterating a
|
| 9 | DataFrameGroupBy still yields them, which is how pandarallel builds its chunks, so
|
| 10 | parallel_apply kept passing them.
|
| 11 |
|
| 12 | This produced a silent divergence rather than an error: user functions that iterate
|
| 13 | columns or aggregate across the frame got different results from parallel_apply than
|
| 14 | from apply.
|
| 15 |
|
| 16 | Restrict each group to the columns apply would have used, gated on the pandas version
|
| 17 | so behaviour on pandas < 3.0 is unchanged.
|
| 18 |
|
| 19 | The groupby test fixture read df.b, but the test also covers groupby(["a", "b"]) where
|
| 20 | b is now a grouping column, making the reference apply() call raise on pandas 3.0.
|
| 21 | Switched the fixture to column c, which is never a grouping column in these tests.
|
| 22 | ---
|
| 23 | pandarallel/data_types/dataframe_groupby.py | 18 +++++++++++++++++-
|
| 24 | tests/test_pandarallel.py | 8 ++++++--
|
| 25 | 2 files changed, 23 insertions(+), 3 deletions(-)
|
| 26 |
|
| 27 | diff --git a/pandarallel/data_types/dataframe_groupby.py b/pandarallel/data_types/dataframe_groupby.py
|
| 28 | index f2dbb91..9b448d7 100644
|
| 29 | |
| 30 | |
| 31 | @@ -17,8 +17,24 @@ def get_chunks(
|
| 32 | chunks = chunk(dataframe_groupby.ngroups, nb_workers)
|
| 33 | iterator = iter(dataframe_groupby)
|
| 34 |
|
| 35 | + # Since pandas 3.0, `DataFrameGroupBy.apply` operates on
|
| 36 | + # `_obj_with_exclusions`, i.e. the grouping columns are no longer passed to
|
| 37 | + # the applied function. Iterating a `DataFrameGroupBy` still yields them
|
| 38 | + # though, so restrict each group to the columns `apply` would have used.
|
| 39 | + # On pandas < 3.0 the grouping columns are still passed, so keep them.
|
| 40 | + columns = (
|
| 41 | + dataframe_groupby._obj_with_exclusions.columns
|
| 42 | + if get_pandas_version() >= (3, 0)
|
| 43 | + else None
|
| 44 | + )
|
| 45 | +
|
| 46 | for chunk_ in chunks:
|
| 47 | - yield [next(iterator) for _ in range(chunk_.stop - chunk_.start)]
|
| 48 | + groups = [next(iterator) for _ in range(chunk_.stop - chunk_.start)]
|
| 49 | +
|
| 50 | + if columns is not None:
|
| 51 | + groups = [(key, df[columns]) for key, df in groups]
|
| 52 | +
|
| 53 | + yield groups
|
| 54 |
|
| 55 | @staticmethod
|
| 56 | def work(
|
| 57 | diff --git a/tests/test_pandarallel.py b/tests/test_pandarallel.py
|
| 58 | index 0f91c32..2f43458 100644
|
| 59 | |
| 60 | |
| 61 | @@ -93,12 +93,16 @@ def func(x):
|
| 62 |
|
| 63 | @pytest.fixture()
|
| 64 | def func_dataframe_groupby_apply():
|
| 65 | + # Uses column `c`, which is never a grouping column in the tests below. Since pandas
|
| 66 | + # 3.0 the grouping columns are not passed to the applied function, so a function
|
| 67 | + # reading `df.b` would fail on the `groupby(["a", "b"])` case before `parallel_apply`
|
| 68 | + # is even reached.
|
| 69 | def func(df):
|
| 70 | dum = 0
|
| 71 | - for item in df.b:
|
| 72 | + for item in df.c:
|
| 73 | dum += math.log10(math.sqrt(math.exp(item**2)))
|
| 74 |
|
| 75 | - return dum / len(df.b)
|
| 76 | + return dum / len(df.c)
|
| 77 |
|
| 78 | return func
|
| 79 |
|