Skip to content

Commit 0bb6a46

Browse files
committed
Accept numpy integers as train_size / test_size in train_test_split
`Dataset.train_test_split()` decides between "number of rows" and "fraction" with `isinstance(test_size, int)` / `isinstance(test_size, float)`, and rejects anything else: ds.train_test_split(test_size=np.int64(2)) # ValueError: Invalid value for test_size: 2 of type <class 'numpy.int64'> numpy floats are subclasses of `float` so they already work, but numpy integers are not subclasses of `int`, which makes the behaviour inconsistent for values that come out of the same numpy computation. Convert numpy integers to `int` before the checks.
1 parent b7cb10b commit 0bb6a46

2 files changed

Lines changed: 18 additions & 0 deletions

File tree

src/datasets/arrow_dataset.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5086,6 +5086,13 @@ def train_test_split(
50865086
if test_size is None and train_size is None:
50875087
test_size = 0.25
50885088

5089+
# numpy integers are not instances of `int` (contrary to numpy floats, which are instances of `float`),
5090+
# so they wouldn't pass the checks below
5091+
if isinstance(test_size, np.integer):
5092+
test_size = int(test_size)
5093+
if isinstance(train_size, np.integer):
5094+
train_size = int(train_size)
5095+
50895096
# Safety checks similar to scikit-learn's ones.
50905097
# (adapted from https://github.com/scikit-learn/scikit-learn/blob/fd237278e895b42abe8d8d09105cbb82dc2cbba7/sklearn/model_selection/_split.py#L1750)
50915098
n_samples = len(self)

tests/test_arrow_dataset.py

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3694,6 +3694,17 @@ def test_concatenate_datasets(dataset_type, axis, expected_shape, dataset_dict,
36943694
assert_arrow_metadata_are_synced_with_dataset_features(dataset)
36953695

36963696

3697+
@pytest.mark.parametrize("dtype", ["int32", "int64", "uint8"])
3698+
def test_train_test_split_with_numpy_integer_sizes(dtype):
3699+
dataset = Dataset.from_dict({"col_1": list(range(10))})
3700+
split = dataset.train_test_split(test_size=getattr(np, dtype)(2), seed=42)
3701+
assert len(split["test"]) == 2
3702+
assert len(split["train"]) == 8
3703+
split = dataset.train_test_split(train_size=getattr(np, dtype)(4), seed=42)
3704+
assert len(split["train"]) == 4
3705+
assert len(split["test"]) == 6
3706+
3707+
36973708
def test_concatenate_datasets_new_columns():
36983709
dataset1 = Dataset.from_dict({"col_1": ["a", "b", "c"]})
36993710
dataset2 = Dataset.from_dict({"col_1": ["d", "e", "f"], "col_2": [True, False, True]})

0 commit comments

Comments
 (0)