From a004b9c78a059cb4f3e56a353898cee3a44ed49b Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Fri, 21 Nov 2025 13:54:36 -0600 Subject: [PATCH 01/28] Remove lingering cupy allocator setting --- .../cuml/internals/api_context_managers.py | 4 -- python/cuml/tests/test_allocator.py | 60 ------------------- 2 files changed, 64 deletions(-) delete mode 100644 python/cuml/tests/test_allocator.py diff --git a/python/cuml/cuml/internals/api_context_managers.py b/python/cuml/cuml/internals/api_context_managers.py index de06babbe6..bb19a20d9c 100644 --- a/python/cuml/cuml/internals/api_context_managers.py +++ b/python/cuml/cuml/internals/api_context_managers.py @@ -8,9 +8,6 @@ from collections import deque from typing import TYPE_CHECKING -from cupy.cuda import using_allocator as cupy_using_allocator -from rmm.allocators.cupy import rmm_cupy_allocator - import cuml.internals.input_utils import cuml.internals.memory_utils from cuml.internals.array_sparse import SparseCumlArray @@ -70,7 +67,6 @@ def cleanup(): self.callback(cleanup) - self.enter_context(cupy_using_allocator(rmm_cupy_allocator)) self.prev_output_type = self.enter_context(_using_mirror_output_type()) # Set the output type to the prev_output_type. If "input", set to None diff --git a/python/cuml/tests/test_allocator.py b/python/cuml/tests/test_allocator.py deleted file mode 100644 index 192d59b55b..0000000000 --- a/python/cuml/tests/test_allocator.py +++ /dev/null @@ -1,60 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 -# - -import cupy as cp -import numpy as np -import pytest - -from cuml import LogisticRegression -from cuml.internals.input_utils import sparse_scipy_to_cp -from cuml.naive_bayes import MultinomialNB -from cuml.testing.datasets import small_classification_dataset - -try: - from cupy.cuda import using_allocator as cupy_using_allocator -except ImportError: - from cupy.cuda.memory import using_allocator as cupy_using_allocator - - -def dummy_allocator(nbytes): - raise AssertionError("Dummy allocator should not be called") - - -def test_dummy_allocator(): - with pytest.raises(AssertionError): - with cupy_using_allocator(dummy_allocator): - a = cp.arange(10) - del a - - -def test_logistic_regression(): - with cupy_using_allocator(dummy_allocator): - X_train, X_test, y_train, y_test = small_classification_dataset( - np.float32 - ) - y_train = y_train.astype(np.float32) - y_test = y_test.astype(np.float32) - culog = LogisticRegression() - culog.fit(X_train, y_train) - culog.predict(X_train) - - -def test_naive_bayes(nlp_20news): - X, y = nlp_20news - - X = sparse_scipy_to_cp(X, cp.float32).astype(cp.float32) - y = y.astype(cp.int32) - - with cupy_using_allocator(dummy_allocator): - model = MultinomialNB() - model.fit(X, y) - - y_hat = model.predict(X) - y_hat = model.predict(X) - y_hat = model.predict_proba(X) - y_hat = model.predict_log_proba(X) - y_hat = model.score(X, y) - - del y_hat From d87b398a7bc13b5080c3d1e6bab20027008cfca9 Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Mon, 24 Nov 2025 08:56:05 -0600 Subject: [PATCH 02/28] Remove `in_internal_api` (move forward) --- python/cuml/cuml/internals/__init__.py | 5 +---- python/cuml/cuml/internals/api_context_managers.py | 4 ---- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/python/cuml/cuml/internals/__init__.py b/python/cuml/cuml/internals/__init__.py index d9f9131772..47bd294a9a 100644 --- a/python/cuml/cuml/internals/__init__.py +++ b/python/cuml/cuml/internals/__init__.py @@ -3,10 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # -from cuml.internals.api_context_managers import ( - in_internal_api, - set_api_output_type, -) +from cuml.internals.api_context_managers import set_api_output_type from cuml.internals.api_decorators import ( api_base_fit_transform, api_base_return_any, diff --git a/python/cuml/cuml/internals/api_context_managers.py b/python/cuml/cuml/internals/api_context_managers.py index bb19a20d9c..c65a39262c 100644 --- a/python/cuml/cuml/internals/api_context_managers.py +++ b/python/cuml/cuml/internals/api_context_managers.py @@ -37,10 +37,6 @@ def _using_mirror_output_type(): GlobalSettings().output_type = prev_output_type -def in_internal_api(): - return GlobalSettings().root_cm is not None - - def set_api_output_type(output_type: str): assert GlobalSettings().root_cm is not None From 1c8521b7080c9a7338b2a94fbb439d15c1404487 Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Fri, 21 Nov 2025 14:03:33 -0600 Subject: [PATCH 03/28] Remove `api_return_sparse_array`/`api_base_return_sparse_array` These had no functional difference with `api_return_array`/`api_base_return_array`. 2 fewer decorators to understand. --- python/cuml/cuml/internals/__init__.py | 2 -- .../cuml/internals/api_context_managers.py | 35 ------------------- python/cuml/cuml/internals/api_decorators.py | 12 ------- python/cuml/cuml/internals/base_helpers.py | 5 +-- .../cuml/cuml/neighbors/nearest_neighbors.pyx | 2 +- python/cuml/cuml/preprocessing/label.py | 2 +- 6 files changed, 3 insertions(+), 55 deletions(-) diff --git a/python/cuml/cuml/internals/__init__.py b/python/cuml/cuml/internals/__init__.py index 47bd294a9a..2825a246cb 100644 --- a/python/cuml/cuml/internals/__init__.py +++ b/python/cuml/cuml/internals/__init__.py @@ -12,11 +12,9 @@ api_base_return_array_skipall, api_base_return_generic, api_base_return_generic_skipall, - api_base_return_sparse_array, api_return_any, api_return_array, api_return_generic, - api_return_sparse_array, exit_internal_api, ) from cuml.internals.base_helpers import BaseMetaClass, _tags_class_and_instance diff --git a/python/cuml/cuml/internals/api_context_managers.py b/python/cuml/cuml/internals/api_context_managers.py index c65a39262c..83c855cb21 100644 --- a/python/cuml/cuml/internals/api_context_managers.py +++ b/python/cuml/cuml/internals/api_context_managers.py @@ -315,27 +315,6 @@ def convert_to_outputtype(self, ret_val): return ret_val.to_output(output_type=output_type) -class ProcessReturnSparseArray(ProcessReturnArray): - def convert_to_cumlarray(self, ret_val): - # Get the output type - ( - ret_val_type_str, - is_sparse, - ) = cuml.internals.input_utils.determine_array_type_full(ret_val) - - # If we are a supported array and not already cuml, convert to cuml - if ret_val_type_str is not None and ret_val_type_str != "cuml": - if is_sparse: - ret_val = SparseCumlArray(ret_val, convert_index=False) - else: - ret_val = cuml.internals.input_utils.input_to_cuml_array( - ret_val, - order="K", - ).array - - return ret_val - - class ProcessReturnGeneric(ProcessReturnArray): def __init__(self, context: "InternalAPIContextBase"): super().__init__(context) @@ -403,12 +382,6 @@ class ReturnArrayCM( pass -class ReturnSparseArrayCM( - InternalAPIContextBase[ProcessEnterReturnArray, ProcessReturnSparseArray] -): - pass - - class ReturnGenericCM( InternalAPIContextBase[ProcessEnterReturnArray, ProcessReturnGeneric] ): @@ -427,14 +400,6 @@ class BaseReturnArrayCM( pass -class BaseReturnSparseArrayCM( - InternalAPIContextBase[ - ProcessEnterBaseReturnArray, ProcessReturnSparseArray - ] -): - pass - - class BaseReturnGenericCM( InternalAPIContextBase[ProcessEnterBaseReturnArray, ProcessReturnGeneric] ): diff --git a/python/cuml/cuml/internals/api_decorators.py b/python/cuml/cuml/internals/api_decorators.py index 3ffe280d2f..94e94043d9 100644 --- a/python/cuml/cuml/internals/api_decorators.py +++ b/python/cuml/cuml/internals/api_decorators.py @@ -18,12 +18,10 @@ BaseReturnAnyCM, BaseReturnArrayCM, BaseReturnGenericCM, - BaseReturnSparseArrayCM, InternalAPIContextBase, ReturnAnyCM, ReturnArrayCM, ReturnGenericCM, - ReturnSparseArrayCM, set_api_output_type, ) from cuml.internals.constants import CUML_WRAPPED_FLAG @@ -209,16 +207,6 @@ def wrapper(*args, **kwargs): set_n_features_in=True, ) -api_return_sparse_array = _make_decorator_function( - ReturnSparseArrayCM, process_return=True -) -api_base_return_sparse_array = _make_decorator_function( - BaseReturnSparseArrayCM, - needs_self=True, - process_return=True, - get_output_type=True, -) - api_base_return_any_skipall = api_base_return_any( set_output_type=False, set_n_features_in=False ) diff --git a/python/cuml/cuml/internals/base_helpers.py b/python/cuml/cuml/internals/base_helpers.py index b4c7c34431..e02049d9ce 100644 --- a/python/cuml/cuml/internals/base_helpers.py +++ b/python/cuml/cuml/internals/base_helpers.py @@ -7,7 +7,6 @@ api_base_return_any, api_base_return_array, api_base_return_generic, - api_base_return_sparse_array, api_return_any, ) from cuml.internals.base_return_types import _get_base_return_type @@ -23,10 +22,8 @@ def _wrap_attribute(class_name: str, attribute_name: str, attribute, **kwargs): if return_type == "generic": attribute = api_base_return_generic(**kwargs)(attribute) - elif return_type == "array": + elif return_type in ("array", "sparsearray"): attribute = api_base_return_array(**kwargs)(attribute) - elif return_type == "sparsearray": - attribute = api_base_return_sparse_array(**kwargs)(attribute) elif return_type == "base": attribute = api_base_return_any(**kwargs)(attribute) elif not attribute_name.startswith("_"): diff --git a/python/cuml/cuml/neighbors/nearest_neighbors.pyx b/python/cuml/cuml/neighbors/nearest_neighbors.pyx index 01520957b7..c183503384 100644 --- a/python/cuml/cuml/neighbors/nearest_neighbors.pyx +++ b/python/cuml/cuml/neighbors/nearest_neighbors.pyx @@ -1095,7 +1095,7 @@ class NearestNeighbors(Base, return self.metric_params or {} -@cuml.internals.api_return_sparse_array() +@cuml.internals.api_return_array() def kneighbors_graph(X=None, n_neighbors=5, mode='connectivity', verbose=False, handle=None, algorithm="brute", metric="euclidean", p=2, include_self=False, metric_params=None): diff --git a/python/cuml/cuml/preprocessing/label.py b/python/cuml/cuml/preprocessing/label.py index 37888d92b3..2da54f1e68 100644 --- a/python/cuml/cuml/preprocessing/label.py +++ b/python/cuml/cuml/preprocessing/label.py @@ -14,7 +14,7 @@ from cuml.prims.label import check_labels, invert_labels, make_monotonic -@cuml.internals.api_return_sparse_array() +@cuml.internals.api_return_array() def label_binarize( y, classes, neg_label=0, pos_label=1, sparse_output=False ) -> SparseCumlArray: From db07390abb7e71134a8c1c31d207a61ffe80db11 Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Fri, 21 Nov 2025 14:07:55 -0600 Subject: [PATCH 04/28] Drop some trivial contextmanager subclasses --- .../cuml/cuml/internals/api_context_managers.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/python/cuml/cuml/internals/api_context_managers.py b/python/cuml/cuml/internals/api_context_managers.py index 83c855cb21..6ab2fe974b 100644 --- a/python/cuml/cuml/internals/api_context_managers.py +++ b/python/cuml/cuml/internals/api_context_managers.py @@ -213,10 +213,6 @@ def __init__(self, context: "InternalAPIContextBase"): self.base_obj: Base = self._context._args[0] -class ProcessEnterReturnAny(ProcessEnter): - pass - - class ProcessEnterReturnArray(ProcessEnter): def __init__(self, context: "InternalAPIContextBase"): super().__init__(context) @@ -265,10 +261,6 @@ def set_output_type(): self._context.callback(set_output_type) -class ProcessReturnAny(ProcessReturn): - pass - - class ProcessReturnArray(ProcessReturn): def __init__(self, context: "InternalAPIContextBase"): super().__init__(context) @@ -370,9 +362,7 @@ def process_generic(self, ret_val): return ret_val -class ReturnAnyCM( - InternalAPIContextBase[ProcessEnterReturnAny, ProcessReturnAny] -): +class ReturnAnyCM(InternalAPIContextBase[ProcessEnter, ProcessReturn]): pass @@ -388,9 +378,7 @@ class ReturnGenericCM( pass -class BaseReturnAnyCM( - InternalAPIContextBase[ProcessEnterReturnAny, ProcessReturnAny] -): +class BaseReturnAnyCM(InternalAPIContextBase[ProcessEnter, ProcessReturn]): pass From 56a3640a38e7fa42577a730176dbbcf71ad9b66a Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Fri, 21 Nov 2025 14:56:05 -0600 Subject: [PATCH 05/28] Consolidate `ProcessReturn` - Simplify `ProcessReturn` from 3 classes into one - Remove parametrizing the context manager by return processing - Remove `__class_getitem__` usage entirely in favor of defining `ProcessEnter_Type` explicitly on subclasses At this point, return handling for reflection is the same everywhere, and the only switch is whether to enable it or not. --- .../cuml/internals/api_context_managers.py | 230 +++++------------- python/cuml/cuml/internals/api_decorators.py | 18 +- 2 files changed, 61 insertions(+), 187 deletions(-) diff --git a/python/cuml/cuml/internals/api_context_managers.py b/python/cuml/cuml/internals/api_context_managers.py index 6ab2fe974b..9cea2e7b3a 100644 --- a/python/cuml/cuml/internals/api_context_managers.py +++ b/python/cuml/cuml/internals/api_context_managers.py @@ -2,20 +2,13 @@ # SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - import contextlib -import typing from collections import deque -from typing import TYPE_CHECKING -import cuml.internals.input_utils -import cuml.internals.memory_utils +import cuml.internals.input_utils as iu from cuml.internals.array_sparse import SparseCumlArray from cuml.internals.global_settings import GlobalSettings -if TYPE_CHECKING: - from cuml.internals.base import Base - @contextlib.contextmanager def _using_mirror_output_type(): @@ -46,7 +39,7 @@ def set_api_output_type(output_type: str): return # Try to convert any array objects to their type - array_type = cuml.internals.input_utils.determine_array_type(output_type) + array_type = iu.determine_array_type(output_type) # Ensure that this is an array-like object assert output_type is None or array_type is not None @@ -119,46 +112,69 @@ def get_internal_context() -> InternalAPIContext: return GlobalSettings().root_cm -class ProcessEnter(object): +class ProcessEnter: def __init__(self, context: "InternalAPIContextBase"): - super().__init__() - self._context = context - self._process_enter_cbs: typing.Deque[typing.Callable] = deque() + self._process_enter_cbs = deque() def process_enter(self): for cb in self._process_enter_cbs: cb() -class ProcessReturn(object): +class ProcessReturn: def __init__(self, context: "InternalAPIContextBase"): - super().__init__() - self._context = context + # Only convert output: + # - when returning results from a root api call + # - when the output type is explicitly set + self._convert_output = ( + self._context.is_root or GlobalSettings().output_type != "mirror" + ) - self._process_return_cbs: typing.Deque[ - typing.Callable[[typing.Any], typing.Any] - ] = deque() + def process_return(self, res): + """Traverse a result, converting it to the proper output type""" + if isinstance(res, tuple): + return tuple(self.process_return(i) for i in res) + elif isinstance(res, list): + return [self.process_return(i) for i in res] + elif isinstance(res, dict): + return {k: self.process_return(v) for k, v in res.items()} - def process_return(self, ret_val): - for cb in self._process_return_cbs: - ret_val = cb(ret_val) + # Get the output type + arr_type, is_sparse = iu.determine_array_type_full(res) - return ret_val + if arr_type is None: + # Not an array, just return + return res + # If we are a supported array and not already cuml, convert to cuml + if arr_type != "cuml": + if is_sparse: + res = SparseCumlArray(res, convert_index=False) + else: + res = iu.input_to_cuml_array(res, order="K").array -EnterT = typing.TypeVar("EnterT", bound=ProcessEnter) -ProcessT = typing.TypeVar("ProcessT", bound=ProcessReturn) + if not self._convert_output: + # Return CumlArray/SparseCumlArray directly + return res + output_type = GlobalSettings().output_type -class InternalAPIContextBase( - contextlib.ExitStack, typing.Generic[EnterT, ProcessT] -): - ProcessEnter_Type: typing.Type[EnterT] = None - ProcessReturn_Type: typing.Type[ProcessT] = None + if output_type in (None, "mirror", "input"): + output_type = self._context.root_cm.output_type + + assert ( + output_type is not None + and output_type != "mirror" + and output_type != "input" + ), ("Invalid root_cm.output_type: '{}'.").format(output_type) + return res.to_output(output_type=output_type) + + +class InternalAPIContextBase(contextlib.ExitStack): def __init__(self, func=None, args=None): super().__init__() @@ -170,7 +186,7 @@ def __init__(self, func=None, args=None): self.is_root = False self._enter_obj: ProcessEnter = self.ProcessEnter_Type(self) - self._process_obj: ProcessReturn = None + self._process_obj = None def __enter__(self): # Enter the root context to know if we are the root cm @@ -183,34 +199,19 @@ def __enter__(self): self._enter_obj.process_enter() # Now create the process functions since we know if we are root or not - self._process_obj = self.ProcessReturn_Type(self) + self._process_obj = ProcessReturn(self) return super().__enter__() def process_return(self, ret_val): return self._process_obj.process_return(ret_val) - def __class_getitem__(cls: typing.Type["InternalAPIContextBase"], params): - param_names = [ - param.__name__ if hasattr(param, "__name__") else str(param) - for param in params - ] - - type_name = f"{cls.__name__}[{', '.join(param_names)}]" - - ns = { - "ProcessEnter_Type": params[0], - "ProcessReturn_Type": params[1], - } - - return type(type_name, (cls,), ns) - class ProcessEnterBaseMixin(ProcessEnter): def __init__(self, context: "InternalAPIContextBase"): super().__init__(context) - self.base_obj: Base = self._context._args[0] + self.base_obj = self._context._args[0] class ProcessEnterReturnArray(ProcessEnter): @@ -261,134 +262,17 @@ def set_output_type(): self._context.callback(set_output_type) -class ProcessReturnArray(ProcessReturn): - def __init__(self, context: "InternalAPIContextBase"): - super().__init__(context) - - self._process_return_cbs.append(self.convert_to_cumlarray) - - if self._context.is_root or GlobalSettings().output_type != "mirror": - self._process_return_cbs.append(self.convert_to_outputtype) - - def convert_to_cumlarray(self, ret_val): - # Get the output type - ( - ret_val_type_str, - is_sparse, - ) = cuml.internals.input_utils.determine_array_type_full(ret_val) - - # If we are a supported array and not already cuml, convert to cuml - if ret_val_type_str is not None and ret_val_type_str != "cuml": - if is_sparse: - ret_val = SparseCumlArray(ret_val, convert_index=False) - else: - ret_val = cuml.internals.input_utils.input_to_cuml_array( - ret_val, order="K" - ).array - - return ret_val - - def convert_to_outputtype(self, ret_val): - output_type = GlobalSettings().output_type - - if ( - output_type is None - or output_type == "mirror" - or output_type == "input" - ): - output_type = self._context.root_cm.output_type - - assert ( - output_type is not None - and output_type != "mirror" - and output_type != "input" - ), ("Invalid root_cm.output_type: '{}'.").format(output_type) - - return ret_val.to_output(output_type=output_type) - - -class ProcessReturnGeneric(ProcessReturnArray): - def __init__(self, context: "InternalAPIContextBase"): - super().__init__(context) - - # Clear the existing callbacks to allow processing one at a time - self._single_array_cbs = self._process_return_cbs - - # Make a new queue - self._process_return_cbs = deque() - - self._process_return_cbs.append(self.process_generic) - - def process_single(self, ret_val): - for cb in self._single_array_cbs: - ret_val = cb(ret_val) - - return ret_val - - def process_tuple(self, ret_val: tuple): - # Convert to a list - out_val = list(ret_val) - - for idx, item in enumerate(out_val): - out_val[idx] = self.process_generic(item) +class ReturnAnyCM(InternalAPIContextBase): + ProcessEnter_Type = ProcessEnter - return tuple(out_val) - def process_dict(self, ret_val): - for name, item in ret_val.items(): - ret_val[name] = self.process_generic(item) +class ReturnArrayCM(InternalAPIContextBase): + ProcessEnter_Type = ProcessEnterReturnArray - return ret_val - def process_list(self, ret_val): - for idx, item in enumerate(ret_val): - ret_val[idx] = self.process_generic(item) +class BaseReturnAnyCM(InternalAPIContextBase): + ProcessEnter_Type = ProcessEnter - return ret_val - def process_generic(self, ret_val): - if cuml.internals.input_utils.is_array_like(ret_val): - return self.process_single(ret_val) - - if isinstance(ret_val, tuple): - return self.process_tuple(ret_val) - - if isinstance(ret_val, dict): - return self.process_dict(ret_val) - - if isinstance(ret_val, list): - return self.process_list(ret_val) - - return ret_val - - -class ReturnAnyCM(InternalAPIContextBase[ProcessEnter, ProcessReturn]): - pass - - -class ReturnArrayCM( - InternalAPIContextBase[ProcessEnterReturnArray, ProcessReturnArray] -): - pass - - -class ReturnGenericCM( - InternalAPIContextBase[ProcessEnterReturnArray, ProcessReturnGeneric] -): - pass - - -class BaseReturnAnyCM(InternalAPIContextBase[ProcessEnter, ProcessReturn]): - pass - - -class BaseReturnArrayCM( - InternalAPIContextBase[ProcessEnterBaseReturnArray, ProcessReturnArray] -): - pass - - -class BaseReturnGenericCM( - InternalAPIContextBase[ProcessEnterBaseReturnArray, ProcessReturnGeneric] -): - pass +class BaseReturnArrayCM(InternalAPIContextBase): + ProcessEnter_Type = ProcessEnterBaseReturnArray diff --git a/python/cuml/cuml/internals/api_decorators.py b/python/cuml/cuml/internals/api_decorators.py index 94e94043d9..62dd607098 100644 --- a/python/cuml/cuml/internals/api_decorators.py +++ b/python/cuml/cuml/internals/api_decorators.py @@ -17,11 +17,9 @@ from cuml.internals.api_context_managers import ( BaseReturnAnyCM, BaseReturnArrayCM, - BaseReturnGenericCM, InternalAPIContextBase, ReturnAnyCM, ReturnArrayCM, - ReturnGenericCM, set_api_output_type, ) from cuml.internals.constants import CUML_WRAPPED_FLAG @@ -180,6 +178,7 @@ def wrapper(*args, **kwargs): needs_self=True, set_output_type=True, set_n_features_in=True, + process_return=False, ) api_return_array = _make_decorator_function(ReturnArrayCM, process_return=True) api_base_return_array = _make_decorator_function( @@ -188,15 +187,8 @@ def wrapper(*args, **kwargs): process_return=True, get_output_type=True, ) -api_return_generic = _make_decorator_function( - ReturnGenericCM, process_return=True -) -api_base_return_generic = _make_decorator_function( - BaseReturnGenericCM, - needs_self=True, - process_return=True, - get_output_type=True, -) +api_return_generic = api_return_array +api_base_return_generic = api_base_return_array api_base_fit_transform = _make_decorator_function( # TODO: add tests for this decorator( BaseReturnArrayCM, @@ -211,9 +203,7 @@ def wrapper(*args, **kwargs): set_output_type=False, set_n_features_in=False ) api_base_return_array_skipall = api_base_return_array(get_output_type=False) -api_base_return_generic_skipall = api_base_return_generic( - get_output_type=False -) +api_base_return_generic_skipall = api_base_return_array_skipall @contextlib.contextmanager From 3940964fc3028d986f01f326ec1454f32b1745cb Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Fri, 21 Nov 2025 15:01:33 -0600 Subject: [PATCH 06/28] Merge `ProcessReturn` into `InternalAPIContextBase` No need for second class at all at this point. --- .../cuml/internals/api_context_managers.py | 65 ++++++++----------- 1 file changed, 27 insertions(+), 38 deletions(-) diff --git a/python/cuml/cuml/internals/api_context_managers.py b/python/cuml/cuml/internals/api_context_managers.py index 9cea2e7b3a..4b18afc3e3 100644 --- a/python/cuml/cuml/internals/api_context_managers.py +++ b/python/cuml/cuml/internals/api_context_managers.py @@ -123,16 +123,38 @@ def process_enter(self): cb() -class ProcessReturn: - def __init__(self, context: "InternalAPIContextBase"): - self._context = context +class InternalAPIContextBase(contextlib.ExitStack): + def __init__(self, func=None, args=None): + super().__init__() + + self._func = func + self._args = args + + self.root_cm = get_internal_context() + + self.is_root = False + + self._enter_obj: ProcessEnter = self.ProcessEnter_Type(self) + + def __enter__(self): + # Enter the root context to know if we are the root cm + self.is_root = self.enter_context(self.root_cm) == 1 + + # If we are the first, push any callbacks from the root into this CM + # If we are not the first, this will have no effect + self.push(self.root_cm.pop_all()) + + self._enter_obj.process_enter() + # Only convert output: # - when returning results from a root api call # - when the output type is explicitly set self._convert_output = ( - self._context.is_root or GlobalSettings().output_type != "mirror" + self.is_root or GlobalSettings().output_type != "mirror" ) + return super().__enter__() + def process_return(self, res): """Traverse a result, converting it to the proper output type""" if isinstance(res, tuple): @@ -163,7 +185,7 @@ def process_return(self, res): output_type = GlobalSettings().output_type if output_type in (None, "mirror", "input"): - output_type = self._context.root_cm.output_type + output_type = self.root_cm.output_type assert ( output_type is not None @@ -174,39 +196,6 @@ def process_return(self, res): return res.to_output(output_type=output_type) -class InternalAPIContextBase(contextlib.ExitStack): - def __init__(self, func=None, args=None): - super().__init__() - - self._func = func - self._args = args - - self.root_cm = get_internal_context() - - self.is_root = False - - self._enter_obj: ProcessEnter = self.ProcessEnter_Type(self) - self._process_obj = None - - def __enter__(self): - # Enter the root context to know if we are the root cm - self.is_root = self.enter_context(self.root_cm) == 1 - - # If we are the first, push any callbacks from the root into this CM - # If we are not the first, this will have no effect - self.push(self.root_cm.pop_all()) - - self._enter_obj.process_enter() - - # Now create the process functions since we know if we are root or not - self._process_obj = ProcessReturn(self) - - return super().__enter__() - - def process_return(self, ret_val): - return self._process_obj.process_return(ret_val) - - class ProcessEnterBaseMixin(ProcessEnter): def __init__(self, context: "InternalAPIContextBase"): super().__init__(context) From f14f61630144eea1b0a1ac2dd9fa0cf7e88d89e3 Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Fri, 21 Nov 2025 15:05:31 -0600 Subject: [PATCH 07/28] Remove duplicate *ReturnAny, remove mixin class Some further trivial simplifications --- .../cuml/cuml/internals/api_context_managers.py | 17 +++-------------- python/cuml/cuml/internals/api_decorators.py | 3 +-- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/python/cuml/cuml/internals/api_context_managers.py b/python/cuml/cuml/internals/api_context_managers.py index 4b18afc3e3..92a2f31df9 100644 --- a/python/cuml/cuml/internals/api_context_managers.py +++ b/python/cuml/cuml/internals/api_context_managers.py @@ -196,13 +196,6 @@ def process_return(self, res): return res.to_output(output_type=output_type) -class ProcessEnterBaseMixin(ProcessEnter): - def __init__(self, context: "InternalAPIContextBase"): - super().__init__(context) - - self.base_obj = self._context._args[0] - - class ProcessEnterReturnArray(ProcessEnter): def __init__(self, context: "InternalAPIContextBase"): super().__init__(context) @@ -213,12 +206,12 @@ def push_output_types(self): self._context.enter_context(self._context.root_cm.push_output_types()) -class ProcessEnterBaseReturnArray( - ProcessEnterReturnArray, ProcessEnterBaseMixin -): +class ProcessEnterBaseReturnArray(ProcessEnterReturnArray): def __init__(self, context: "InternalAPIContextBase"): super().__init__(context) + self.base_obj = self._context._args[0] + # IMPORTANT: Only perform output type processing if # `root_cm.output_type` is None. Since we default to using the incoming # value if its set, there is no need to do any processing if the user @@ -259,9 +252,5 @@ class ReturnArrayCM(InternalAPIContextBase): ProcessEnter_Type = ProcessEnterReturnArray -class BaseReturnAnyCM(InternalAPIContextBase): - ProcessEnter_Type = ProcessEnter - - class BaseReturnArrayCM(InternalAPIContextBase): ProcessEnter_Type = ProcessEnterBaseReturnArray diff --git a/python/cuml/cuml/internals/api_decorators.py b/python/cuml/cuml/internals/api_decorators.py index 62dd607098..d9d969a5a1 100644 --- a/python/cuml/cuml/internals/api_decorators.py +++ b/python/cuml/cuml/internals/api_decorators.py @@ -15,7 +15,6 @@ # TODO: Try to resolve circular import that makes this necessary: from cuml.internals import input_utils as iu from cuml.internals.api_context_managers import ( - BaseReturnAnyCM, BaseReturnArrayCM, InternalAPIContextBase, ReturnAnyCM, @@ -174,7 +173,7 @@ def wrapper(*args, **kwargs): api_return_any = _make_decorator_function(ReturnAnyCM, process_return=False) api_base_return_any = _make_decorator_function( - BaseReturnAnyCM, + ReturnAnyCM, needs_self=True, set_output_type=True, set_n_features_in=True, From 1697a47dc1221c7369a35be05e9079ae81ecca3a Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Fri, 21 Nov 2025 15:16:26 -0600 Subject: [PATCH 08/28] Remove *generic decorators These are now redundant and were aliases to their array counterparts. Removing lets us simplify further. --- .../_thirdparty/sklearn/preprocessing/_data.py | 16 ++++++++-------- python/cuml/cuml/cluster/hdbscan/hdbscan.pyx | 2 +- python/cuml/cuml/datasets/blobs.py | 2 +- python/cuml/cuml/datasets/classification.py | 2 +- python/cuml/cuml/datasets/regression.pyx | 2 +- python/cuml/cuml/explainer/sampling.py | 2 +- python/cuml/cuml/internals/__init__.py | 3 --- python/cuml/cuml/internals/api_decorators.py | 3 --- python/cuml/cuml/internals/base_helpers.py | 5 +---- python/cuml/cuml/metrics/_ranking.py | 2 +- python/cuml/cuml/metrics/cluster/entropy.pyx | 2 +- python/cuml/cuml/metrics/cluster/utils.py | 2 +- .../cuml/neighbors/kneighbors_classifier.pyx | 2 +- .../cuml/neighbors/kneighbors_classifier_mg.pyx | 6 +++--- .../cuml/neighbors/kneighbors_regressor_mg.pyx | 4 ++-- .../cuml/cuml/neighbors/nearest_neighbors_mg.pyx | 4 ++-- python/cuml/cuml/prims/label/classlabels.py | 2 +- python/cuml/cuml/tsa/arima.pyx | 6 +++--- python/cuml/cuml/tsa/auto_arima.pyx | 4 ++-- 19 files changed, 31 insertions(+), 40 deletions(-) diff --git a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_data.py b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_data.py index d4d0bef0c7..dc8ba64147 100644 --- a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_data.py +++ b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_data.py @@ -42,7 +42,7 @@ ) from ....common.array_descriptor import CumlArrayDescriptor -from ....internals import api_return_generic +from ....internals import api_return_array from ....internals.array import CumlArray from ....internals.array_sparse import SparseCumlArray from ....internals.memory_utils import using_output_type @@ -102,7 +102,7 @@ def _handle_zeros_in_scale(scale, copy=True): return scale -@api_return_generic(get_output_type=True) +@api_return_array(get_output_type=True) def scale(X, *, axis=0, with_mean=True, with_std=True, copy=True): """Standardize a dataset along any axis @@ -448,7 +448,7 @@ def inverse_transform(self, X) -> CumlArray: return X -@api_return_generic(get_output_type=True) +@api_return_array(get_output_type=True) def minmax_scale(X, feature_range=(0, 1), *, axis=0, copy=True): """Transform features by scaling each feature to a given range. @@ -1057,7 +1057,7 @@ def inverse_transform(self, X) -> SparseCumlArray: return X -@api_return_generic(get_output_type=True) +@api_return_array(get_output_type=True) def maxabs_scale(X, *, axis=0, copy=True): """Scale each feature to the [-1, 1] range without breaking the sparsity. @@ -1319,7 +1319,7 @@ def inverse_transform(self, X) -> SparseCumlArray: return X -@api_return_generic(get_output_type=True) +@api_return_array(get_output_type=True) def robust_scale(X, *, axis=0, with_centering=True, with_scaling=True, quantile_range=(25.0, 75.0), copy=True): """ @@ -1679,7 +1679,7 @@ def transform(self, X) -> SparseCumlArray: return XP # TODO keep order -@api_return_generic(get_output_type=True) +@api_return_array(get_output_type=True) def normalize(X, norm='l2', *, axis=1, copy=True, return_norm=False): """Scale input vectors individually to unit norm (vector length). @@ -1859,7 +1859,7 @@ def transform(self, X, copy=None) -> SparseCumlArray: return normalize(X, norm=self.norm, axis=1, copy=copy) -@api_return_generic(get_output_type=True) +@api_return_array(get_output_type=True) def binarize(X, *, threshold=0.0, copy=True): """Boolean thresholding of array-like or sparse matrix @@ -1988,7 +1988,7 @@ def transform(self, X, copy=None) -> SparseCumlArray: return binarize(X, threshold=self.threshold, copy=copy) -@api_return_generic(get_output_type=True) +@api_return_array(get_output_type=True) def add_dummy_feature(X, value=1.0): """Augment dataset with an additional dummy feature. diff --git a/python/cuml/cuml/cluster/hdbscan/hdbscan.pyx b/python/cuml/cuml/cluster/hdbscan/hdbscan.pyx index 1830577aa5..f788246770 100644 --- a/python/cuml/cuml/cluster/hdbscan/hdbscan.pyx +++ b/python/cuml/cuml/cluster/hdbscan/hdbscan.pyx @@ -1275,7 +1275,7 @@ def membership_vector(clusterer, points_to_predict, int batch_size=4096, convert return membership_vec -@cuml.internals.api_return_generic() +@cuml.internals.api_return_array() def approximate_predict(clusterer, points_to_predict, convert_dtype=True): """Predict the cluster label of new points. The returned labels will be those of the original clustering found by ``clusterer``, diff --git a/python/cuml/cuml/datasets/blobs.py b/python/cuml/cuml/datasets/blobs.py index 5792ac194a..8afe878b68 100644 --- a/python/cuml/cuml/datasets/blobs.py +++ b/python/cuml/cuml/datasets/blobs.py @@ -71,7 +71,7 @@ def _get_centers(rs, centers, center_box, n_samples, n_features, dtype): @nvtx.annotate(message="datasets.make_blobs", domain="cuml_python") -@cuml.internals.api_return_generic() +@cuml.internals.api_return_array() def make_blobs( n_samples=100, n_features=2, diff --git a/python/cuml/cuml/datasets/classification.py b/python/cuml/cuml/datasets/classification.py index 67de42dea0..af282523cc 100644 --- a/python/cuml/cuml/datasets/classification.py +++ b/python/cuml/cuml/datasets/classification.py @@ -33,7 +33,7 @@ def _generate_hypercube(samples, dimensions, random_state): @nvtx.annotate(message="datasets.make_classification", domain="cuml_python") -@cuml.internals.api_return_generic() +@cuml.internals.api_return_array() def make_classification( n_samples=100, n_features=20, diff --git a/python/cuml/cuml/datasets/regression.pyx b/python/cuml/cuml/datasets/regression.pyx index 42470012cd..5d1986f4e6 100644 --- a/python/cuml/cuml/datasets/regression.pyx +++ b/python/cuml/cuml/datasets/regression.pyx @@ -63,7 +63,7 @@ inp_to_dtype = { @nvtx.annotate(message="datasets.make_regression", domain="cuml_python") -@cuml.internals.api_return_generic() +@cuml.internals.api_return_array() def make_regression( n_samples=100, n_features=2, diff --git a/python/cuml/cuml/explainer/sampling.py b/python/cuml/cuml/explainer/sampling.py index 2b6253718f..2f93f6dcd4 100644 --- a/python/cuml/cuml/explainer/sampling.py +++ b/python/cuml/cuml/explainer/sampling.py @@ -13,7 +13,7 @@ from cuml.preprocessing import SimpleImputer -@cuml.internals.api_return_generic() +@cuml.internals.api_return_array() def kmeans_sampling(X, k, round_values=True, detailed=False, random_state=0): """ Adapted from : diff --git a/python/cuml/cuml/internals/__init__.py b/python/cuml/cuml/internals/__init__.py index 2825a246cb..c2bc395eed 100644 --- a/python/cuml/cuml/internals/__init__.py +++ b/python/cuml/cuml/internals/__init__.py @@ -10,11 +10,8 @@ api_base_return_any_skipall, api_base_return_array, api_base_return_array_skipall, - api_base_return_generic, - api_base_return_generic_skipall, api_return_any, api_return_array, - api_return_generic, exit_internal_api, ) from cuml.internals.base_helpers import BaseMetaClass, _tags_class_and_instance diff --git a/python/cuml/cuml/internals/api_decorators.py b/python/cuml/cuml/internals/api_decorators.py index d9d969a5a1..1c311df095 100644 --- a/python/cuml/cuml/internals/api_decorators.py +++ b/python/cuml/cuml/internals/api_decorators.py @@ -186,8 +186,6 @@ def wrapper(*args, **kwargs): process_return=True, get_output_type=True, ) -api_return_generic = api_return_array -api_base_return_generic = api_base_return_array api_base_fit_transform = _make_decorator_function( # TODO: add tests for this decorator( BaseReturnArrayCM, @@ -202,7 +200,6 @@ def wrapper(*args, **kwargs): set_output_type=False, set_n_features_in=False ) api_base_return_array_skipall = api_base_return_array(get_output_type=False) -api_base_return_generic_skipall = api_base_return_array_skipall @contextlib.contextmanager diff --git a/python/cuml/cuml/internals/base_helpers.py b/python/cuml/cuml/internals/base_helpers.py index e02049d9ce..03ae5290f0 100644 --- a/python/cuml/cuml/internals/base_helpers.py +++ b/python/cuml/cuml/internals/base_helpers.py @@ -6,7 +6,6 @@ from cuml.internals.api_decorators import ( api_base_return_any, api_base_return_array, - api_base_return_generic, api_return_any, ) from cuml.internals.base_return_types import _get_base_return_type @@ -20,9 +19,7 @@ def _wrap_attribute(class_name: str, attribute_name: str, attribute, **kwargs): return_type = _get_base_return_type(class_name, attribute) - if return_type == "generic": - attribute = api_base_return_generic(**kwargs)(attribute) - elif return_type in ("array", "sparsearray"): + if return_type in ("generic", "array", "sparsearray"): attribute = api_base_return_array(**kwargs)(attribute) elif return_type == "base": attribute = api_base_return_any(**kwargs)(attribute) diff --git a/python/cuml/cuml/metrics/_ranking.py b/python/cuml/cuml/metrics/_ranking.py index 596ac35788..ff0d9cebce 100644 --- a/python/cuml/cuml/metrics/_ranking.py +++ b/python/cuml/cuml/metrics/_ranking.py @@ -14,7 +14,7 @@ from cuml.internals.input_utils import input_to_cupy_array -@cuml.internals.api_return_generic(get_output_type=True) +@cuml.internals.api_return_array(get_output_type=True) def precision_recall_curve( y_true, probs_pred ) -> typing.Tuple[CumlArray, CumlArray, CumlArray]: diff --git a/python/cuml/cuml/metrics/cluster/entropy.pyx b/python/cuml/cuml/metrics/cluster/entropy.pyx index 396a8b3665..e336f515af 100644 --- a/python/cuml/cuml/metrics/cluster/entropy.pyx +++ b/python/cuml/cuml/metrics/cluster/entropy.pyx @@ -27,7 +27,7 @@ cdef extern from "cuml/metrics/metrics.hpp" namespace "ML::Metrics" nogil: const int upper_class_range) except + -@cuml.internals.api_return_generic() +@cuml.internals.api_return_array() def _prepare_cluster_input(cluster) -> typing.Tuple[CumlArray, int, int, int]: """Helper function to avoid code duplication for clustering metrics.""" cluster_m, n_rows, _, _ = input_to_cupy_array( diff --git a/python/cuml/cuml/metrics/cluster/utils.py b/python/cuml/cuml/metrics/cluster/utils.py index 18854601a1..a709cc58a6 100644 --- a/python/cuml/cuml/metrics/cluster/utils.py +++ b/python/cuml/cuml/metrics/cluster/utils.py @@ -11,7 +11,7 @@ from cuml.prims.label import make_monotonic -@cuml.internals.api_return_generic(get_output_type=True) +@cuml.internals.api_return_array(get_output_type=True) def prepare_cluster_metric_inputs(labels_true, labels_pred): """Helper function to avoid code duplication for homogeneity score, mutual info score and completeness score. diff --git a/python/cuml/cuml/neighbors/kneighbors_classifier.pyx b/python/cuml/cuml/neighbors/kneighbors_classifier.pyx index 6760b91fe3..aedc5e1f6f 100644 --- a/python/cuml/cuml/neighbors/kneighbors_classifier.pyx +++ b/python/cuml/cuml/neighbors/kneighbors_classifier.pyx @@ -277,7 +277,7 @@ class KNeighborsClassifier(ClassifierMixin, 'type': 'dense', 'description': 'Labels probabilities', 'shape': '(n_samples, 1)'}) - @cuml.internals.api_base_return_generic() + @cuml.internals.api_base_return_array() def predict_proba(self, X, *, convert_dtype=True) -> CumlArray | list[CumlArray]: """ Use the trained k-nearest neighbors classifier to diff --git a/python/cuml/cuml/neighbors/kneighbors_classifier_mg.pyx b/python/cuml/cuml/neighbors/kneighbors_classifier_mg.pyx index 2a7e4b1eb0..271469a577 100644 --- a/python/cuml/cuml/neighbors/kneighbors_classifier_mg.pyx +++ b/python/cuml/cuml/neighbors/kneighbors_classifier_mg.pyx @@ -8,7 +8,7 @@ import typing import cuml.internals.logger as logger -from cuml.internals import api_base_return_generic_skipall +from cuml.internals import api_base_return_array_skipall from cuml.internals.array import CumlArray from cuml.neighbors.nearest_neighbors_mg import NearestNeighborsMG @@ -58,7 +58,7 @@ class KNeighborsClassifierMG(NearestNeighborsMG): def __init__(self, **kwargs): super(KNeighborsClassifierMG, self).__init__(**kwargs) - @api_base_return_generic_skipall + @api_base_return_array_skipall def predict( self, index, @@ -178,7 +178,7 @@ class KNeighborsClassifierMG(NearestNeighborsMG): return output_cais - @api_base_return_generic_skipall + @api_base_return_array_skipall def predict_proba(self, index, index_parts_to_ranks, index_nrows, query, query_parts_to_ranks, query_nrows, uniq_labels, n_unique, ncols, rank, diff --git a/python/cuml/cuml/neighbors/kneighbors_regressor_mg.pyx b/python/cuml/cuml/neighbors/kneighbors_regressor_mg.pyx index e4943bc5d7..e539315560 100644 --- a/python/cuml/cuml/neighbors/kneighbors_regressor_mg.pyx +++ b/python/cuml/cuml/neighbors/kneighbors_regressor_mg.pyx @@ -8,7 +8,7 @@ import typing import cuml.internals.logger as logger -from cuml.internals import api_base_return_generic_skipall +from cuml.internals import api_base_return_array_skipall from cuml.internals.array import CumlArray from cuml.neighbors.nearest_neighbors_mg import NearestNeighborsMG @@ -52,7 +52,7 @@ class KNeighborsRegressorMG(NearestNeighborsMG): def __init__(self, **kwargs): super(KNeighborsRegressorMG, self).__init__(**kwargs) - @api_base_return_generic_skipall + @api_base_return_array_skipall def predict( self, index, diff --git a/python/cuml/cuml/neighbors/nearest_neighbors_mg.pyx b/python/cuml/cuml/neighbors/nearest_neighbors_mg.pyx index a3f93d56c8..555d9ac16c 100644 --- a/python/cuml/cuml/neighbors/nearest_neighbors_mg.pyx +++ b/python/cuml/cuml/neighbors/nearest_neighbors_mg.pyx @@ -9,7 +9,7 @@ import typing import cuml.internals.logger as logger from cuml.common import input_to_cuml_array -from cuml.internals import api_base_return_generic_skipall +from cuml.internals import api_base_return_array_skipall from cuml.internals.array import CumlArray from cuml.neighbors import NearestNeighbors @@ -60,7 +60,7 @@ class NearestNeighborsMG(NearestNeighbors): super().__init__(**kwargs) self.batch_size = batch_size - @api_base_return_generic_skipall + @api_base_return_array_skipall def kneighbors( self, index, diff --git a/python/cuml/cuml/prims/label/classlabels.py b/python/cuml/cuml/prims/label/classlabels.py index 85509cfa32..273cc8f251 100644 --- a/python/cuml/cuml/prims/label/classlabels.py +++ b/python/cuml/cuml/prims/label/classlabels.py @@ -99,7 +99,7 @@ def _validate_kernel(dtype): ) -@cuml.internals.api_return_generic(input_arg="labels", get_output_type=True) +@cuml.internals.api_return_array(input_arg="labels", get_output_type=True) def make_monotonic( labels, classes=None, copy=False ) -> typing.Tuple[CumlArray, CumlArray]: diff --git a/python/cuml/cuml/tsa/arima.pyx b/python/cuml/cuml/tsa/arima.pyx index dbf6d95da9..16f6ee1718 100644 --- a/python/cuml/cuml/tsa/arima.pyx +++ b/python/cuml/cuml/tsa/arima.pyx @@ -509,7 +509,7 @@ class ARIMA(Base): return (order.p + order.P + order.q + order.Q + order.k + order.n_exog + 1) - @cuml.internals.api_base_return_generic(input_arg=None) + @cuml.internals.api_base_return_array(input_arg=None) def get_fit_params(self) -> Dict[str, CumlArray]: """Get all the fit parameters. Not to be confused with get_params Note: pack() can be used to get a compact vector of the parameters @@ -585,7 +585,7 @@ class ARIMA(Base): raise NotImplementedError("ARIMA is unable to be cloned via " "`get_params` and `set_params`.") - @cuml.internals.api_base_return_generic(input_arg=None) + @cuml.internals.api_base_return_array(input_arg=None) def predict( self, start=0, @@ -734,7 +734,7 @@ class ARIMA(Base): d_upper) @nvtx.annotate(message="tsa.arima.ARIMA.forecast", domain="cuml_python") - @cuml.internals.api_base_return_generic_skipall + @cuml.internals.api_base_return_array_skipall def forecast( self, nsteps: int, diff --git a/python/cuml/cuml/tsa/auto_arima.pyx b/python/cuml/cuml/tsa/auto_arima.pyx index 0d52ba5173..9427a9bc8f 100644 --- a/python/cuml/cuml/tsa/auto_arima.pyx +++ b/python/cuml/cuml/tsa/auto_arima.pyx @@ -452,7 +452,7 @@ class AutoARIMA(Base): logger.debug("Fitting {} ({})".format(model, method)) model.fit(h=h, maxiter=maxiter, method=method, truncate=truncate) - @cuml.internals.api_base_return_generic_skipall + @cuml.internals.api_base_return_array_skipall def predict( self, start=0, @@ -512,7 +512,7 @@ class AutoARIMA(Base): else: return y_p, lower, upper - @cuml.internals.api_base_return_generic_skipall + @cuml.internals.api_base_return_array_skipall def forecast(self, nsteps: int, level=None) -> typing.Union[CumlArray, From d59849a80d5e3de031f5dc58437147bbae48e00f Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Fri, 21 Nov 2025 16:05:50 -0600 Subject: [PATCH 09/28] Rip out ProcessEnter classes Simplify, just move all logic to InternalContextAPIBase. Terrible name, but we now have fewer moving pieces. All call logic is still the same as before. Some of the branches don't make sense to me, but we're not changing the logic, just the plumbing. --- .../cuml/internals/api_context_managers.py | 105 ++++++------------ python/cuml/cuml/internals/api_decorators.py | 19 ++-- 2 files changed, 39 insertions(+), 85 deletions(-) diff --git a/python/cuml/cuml/internals/api_context_managers.py b/python/cuml/cuml/internals/api_context_managers.py index 92a2f31df9..d98b4fc2bb 100644 --- a/python/cuml/cuml/internals/api_context_managers.py +++ b/python/cuml/cuml/internals/api_context_managers.py @@ -112,29 +112,43 @@ def get_internal_context() -> InternalAPIContext: return GlobalSettings().root_cm -class ProcessEnter: - def __init__(self, context: "InternalAPIContextBase"): - self._context = context - - self._process_enter_cbs = deque() - - def process_enter(self): - for cb in self._process_enter_cbs: - cb() - - class InternalAPIContextBase(contextlib.ExitStack): - def __init__(self, func=None, args=None): + def __init__( + self, func=None, args=None, is_base_method=False, process_return=True + ): super().__init__() self._func = func self._args = args + self._is_base_method = is_base_method + self._process_return = process_return self.root_cm = get_internal_context() self.is_root = False - self._enter_obj: ProcessEnter = self.ProcessEnter_Type(self) + self._should_set_output_type_from_base = ( + self._is_base_method + and self.root_cm.prev_output_type in (None, "input") + ) + + def set_output_type_from_base(self, root_cm): + output_type = root_cm.output_type + + base = self._args[0] + + # Check if output_type is None, can happen if no output type has + # been set by estimator + if output_type is None: + output_type = base.output_type + + if output_type == "input": + output_type = base._input_type + + if output_type != root_cm.output_type: + set_api_output_type(output_type) + + assert output_type != "mirror" def __enter__(self): # Enter the root context to know if we are the root cm @@ -144,7 +158,10 @@ def __enter__(self): # If we are not the first, this will have no effect self.push(self.root_cm.pop_all()) - self._enter_obj.process_enter() + if self._process_return: + self.enter_context(self.root_cm.push_output_types()) + if self._should_set_output_type_from_base: + self.callback(self.set_output_type_from_base, self.root_cm) # Only convert output: # - when returning results from a root api call @@ -194,63 +211,3 @@ def process_return(self, res): ), ("Invalid root_cm.output_type: '{}'.").format(output_type) return res.to_output(output_type=output_type) - - -class ProcessEnterReturnArray(ProcessEnter): - def __init__(self, context: "InternalAPIContextBase"): - super().__init__(context) - - self._process_enter_cbs.append(self.push_output_types) - - def push_output_types(self): - self._context.enter_context(self._context.root_cm.push_output_types()) - - -class ProcessEnterBaseReturnArray(ProcessEnterReturnArray): - def __init__(self, context: "InternalAPIContextBase"): - super().__init__(context) - - self.base_obj = self._context._args[0] - - # IMPORTANT: Only perform output type processing if - # `root_cm.output_type` is None. Since we default to using the incoming - # value if its set, there is no need to do any processing if the user - # has specified the output type - if ( - self._context.root_cm.prev_output_type is None - or self._context.root_cm.prev_output_type == "input" - ): - self._process_enter_cbs.append(self.base_output_type_callback) - - def base_output_type_callback(self): - root_cm = self._context.root_cm - - def set_output_type(): - output_type = root_cm.output_type - - # Check if output_type is None, can happen if no output type has - # been set by estimator - if output_type is None: - output_type = self.base_obj.output_type - - if output_type == "input": - output_type = self.base_obj._input_type - - if output_type != root_cm.output_type: - set_api_output_type(output_type) - - assert output_type != "mirror" - - self._context.callback(set_output_type) - - -class ReturnAnyCM(InternalAPIContextBase): - ProcessEnter_Type = ProcessEnter - - -class ReturnArrayCM(InternalAPIContextBase): - ProcessEnter_Type = ProcessEnterReturnArray - - -class BaseReturnArrayCM(InternalAPIContextBase): - ProcessEnter_Type = ProcessEnterBaseReturnArray diff --git a/python/cuml/cuml/internals/api_decorators.py b/python/cuml/cuml/internals/api_decorators.py index 1c311df095..ba26791413 100644 --- a/python/cuml/cuml/internals/api_decorators.py +++ b/python/cuml/cuml/internals/api_decorators.py @@ -15,10 +15,7 @@ # TODO: Try to resolve circular import that makes this necessary: from cuml.internals import input_utils as iu from cuml.internals.api_context_managers import ( - BaseReturnArrayCM, InternalAPIContextBase, - ReturnAnyCM, - ReturnArrayCM, set_api_output_type, ) from cuml.internals.constants import CUML_WRAPPED_FLAG @@ -76,7 +73,6 @@ def _get_value(args, kwargs, name, index, default_value, accept_lists=False): def _make_decorator_function( - context_manager_cls: InternalAPIContextBase, process_return=True, needs_self: bool = False, **defaults, @@ -128,7 +124,12 @@ def wrapper(*args, **kwargs): # Accept list/tuple inputs when accelerator is active accept_lists = cuml.accel.enabled() - with context_manager_cls(func, args) as cm: + with InternalAPIContextBase( + func, + args, + is_base_method=needs_self, + process_return=process_return, + ) as cm: self_val = args[0] if has_self else None if input_arg_: @@ -171,24 +172,20 @@ def wrapper(*args, **kwargs): return functools.partial(decorator_function, **defaults) -api_return_any = _make_decorator_function(ReturnAnyCM, process_return=False) +api_return_any = _make_decorator_function(process_return=False) api_base_return_any = _make_decorator_function( - ReturnAnyCM, needs_self=True, set_output_type=True, set_n_features_in=True, process_return=False, ) -api_return_array = _make_decorator_function(ReturnArrayCM, process_return=True) +api_return_array = _make_decorator_function(process_return=True) api_base_return_array = _make_decorator_function( - BaseReturnArrayCM, needs_self=True, process_return=True, get_output_type=True, ) api_base_fit_transform = _make_decorator_function( - # TODO: add tests for this decorator( - BaseReturnArrayCM, needs_self=True, process_return=True, get_output_type=True, From ed547db19d0e8ba1a92f0ccfe0e2fb4c8f23af3c Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Fri, 21 Nov 2025 16:35:34 -0600 Subject: [PATCH 10/28] Couple `set_n_features_in` to `set_output_type` These should only be used on fits, but any fit should do all of them. --- python/cuml/cuml/internals/api_decorators.py | 22 ++++++-------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/python/cuml/cuml/internals/api_decorators.py b/python/cuml/cuml/internals/api_decorators.py index ba26791413..15e1712e07 100644 --- a/python/cuml/cuml/internals/api_decorators.py +++ b/python/cuml/cuml/internals/api_decorators.py @@ -94,8 +94,7 @@ def _make_decorator_function( def decorator_function( input_arg: str = ..., get_output_type: bool = False, - set_output_type: bool = False, - set_n_features_in: bool = False, + is_fit: bool = False, ) -> _DecoratorType: def decorator_closure(func): # This function constitutes the closed decorator that will return @@ -110,9 +109,7 @@ def decorator_closure(func): if needs_self and not has_self: raise Exception("No self found on function!") - if input_arg is not None and ( - set_output_type or set_n_features_in or get_output_type - ): + if input_arg is not None and (is_fit or get_output_type): input_arg_ = _find_arg(sig, input_arg or "X", 0) else: input_arg_ = None @@ -142,11 +139,9 @@ def wrapper(*args, **kwargs): else: input_val = None - if set_output_type: + if is_fit: assert self_val is not None self_val._set_output_type(input_val) - if set_n_features_in and len(input_val.shape) >= 2: - assert self_val is not None self_val._set_n_features_in(input_val) if get_output_type: @@ -175,8 +170,7 @@ def wrapper(*args, **kwargs): api_return_any = _make_decorator_function(process_return=False) api_base_return_any = _make_decorator_function( needs_self=True, - set_output_type=True, - set_n_features_in=True, + is_fit=True, process_return=False, ) api_return_array = _make_decorator_function(process_return=True) @@ -189,13 +183,9 @@ def wrapper(*args, **kwargs): needs_self=True, process_return=True, get_output_type=True, - set_output_type=True, - set_n_features_in=True, -) - -api_base_return_any_skipall = api_base_return_any( - set_output_type=False, set_n_features_in=False + is_fit=True, ) +api_base_return_any_skipall = api_base_return_any(is_fit=False) api_base_return_array_skipall = api_base_return_array(get_output_type=False) From dd1b90a36cca4437f083172cd26782833c379bad Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Fri, 21 Nov 2025 16:52:39 -0600 Subject: [PATCH 11/28] Remove `needs_self` This can be easily inferred. Further simplifying. --- python/cuml/cuml/internals/api_decorators.py | 46 ++++++++------------ 1 file changed, 17 insertions(+), 29 deletions(-) diff --git a/python/cuml/cuml/internals/api_decorators.py b/python/cuml/cuml/internals/api_decorators.py index 15e1712e07..9507efda1e 100644 --- a/python/cuml/cuml/internals/api_decorators.py +++ b/python/cuml/cuml/internals/api_decorators.py @@ -2,7 +2,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - import contextlib import functools import inspect @@ -10,6 +9,7 @@ import numpy as np +import cuml import cuml.accel # TODO: Try to resolve circular import that makes this necessary: @@ -72,15 +72,14 @@ def _get_value(args, kwargs, name, index, default_value, accept_lists=False): return value -def _make_decorator_function( +def _make_decorator( process_return=True, - needs_self: bool = False, **defaults, ) -> typing.Callable[..., _DecoratorType]: # This function generates a function to be applied as decorator to a # wrapped function. For example: # - # a_decorator = _make_decorator_function(...) + # a_decorator = _make_decorator(...) # # ... # @@ -106,8 +105,6 @@ def decorator_closure(func): sig = inspect.signature(func, follow_wrapped=True) has_self = _has_self(sig) - if needs_self and not has_self: - raise Exception("No self found on function!") if input_arg is not None and (is_fit or get_output_type): input_arg_ = _find_arg(sig, input_arg or "X", 0) @@ -121,14 +118,13 @@ def wrapper(*args, **kwargs): # Accept list/tuple inputs when accelerator is active accept_lists = cuml.accel.enabled() + self_val = args[0] if has_self else None with InternalAPIContextBase( func, args, - is_base_method=needs_self, + is_base_method=isinstance(self_val, cuml.Base), process_return=process_return, ) as cm: - self_val = args[0] if has_self else None - if input_arg_: input_val = _get_value( args, @@ -167,26 +163,18 @@ def wrapper(*args, **kwargs): return functools.partial(decorator_function, **defaults) -api_return_any = _make_decorator_function(process_return=False) -api_base_return_any = _make_decorator_function( - needs_self=True, - is_fit=True, - process_return=False, -) -api_return_array = _make_decorator_function(process_return=True) -api_base_return_array = _make_decorator_function( - needs_self=True, - process_return=True, - get_output_type=True, -) -api_base_fit_transform = _make_decorator_function( - needs_self=True, - process_return=True, - get_output_type=True, - is_fit=True, -) -api_base_return_any_skipall = api_base_return_any(is_fit=False) -api_base_return_array_skipall = api_base_return_array(get_output_type=False) +# TODO: +# - infer get_output_type from whether return value is Base +# - determine why api_return_any is needed? It should only mark internal API? +# - infer `is_fit` based on method name by default +api_return_array = _make_decorator() +api_return_any = _make_decorator(process_return=False) +api_base_return_any = _make_decorator(is_fit=True, process_return=False) +api_base_return_array = _make_decorator(get_output_type=True) +api_base_fit_transform = _make_decorator(is_fit=True, get_output_type=True) +# TODO: investigate and remove these +api_base_return_any_skipall = api_return_any() +api_base_return_array_skipall = api_return_array() @contextlib.contextmanager From cd48af1e3c105d5676602d876c7e3fd26710d5de Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Mon, 24 Nov 2025 17:19:32 -0600 Subject: [PATCH 12/28] Add `reflect`, simplify frontend --- .../cuml/internals/api_context_managers.py | 16 +- python/cuml/cuml/internals/api_decorators.py | 258 ++++++++---------- python/cuml/cuml/linear_model/base.py | 4 +- python/cuml/tests/test_cuml_descr_decor.py | 4 +- 4 files changed, 123 insertions(+), 159 deletions(-) diff --git a/python/cuml/cuml/internals/api_context_managers.py b/python/cuml/cuml/internals/api_context_managers.py index d98b4fc2bb..8d4ffe1508 100644 --- a/python/cuml/cuml/internals/api_context_managers.py +++ b/python/cuml/cuml/internals/api_context_managers.py @@ -113,14 +113,10 @@ def get_internal_context() -> InternalAPIContext: class InternalAPIContextBase(contextlib.ExitStack): - def __init__( - self, func=None, args=None, is_base_method=False, process_return=True - ): + def __init__(self, base=None, process_return=None): super().__init__() - self._func = func - self._args = args - self._is_base_method = is_base_method + self._base = base self._process_return = process_return self.root_cm = get_internal_context() @@ -128,22 +124,20 @@ def __init__( self.is_root = False self._should_set_output_type_from_base = ( - self._is_base_method + self._base is not None and self.root_cm.prev_output_type in (None, "input") ) def set_output_type_from_base(self, root_cm): output_type = root_cm.output_type - base = self._args[0] - # Check if output_type is None, can happen if no output type has # been set by estimator if output_type is None: - output_type = base.output_type + output_type = self._base.output_type if output_type == "input": - output_type = base._input_type + output_type = self._base._input_type if output_type != root_cm.output_type: set_api_output_type(output_type) diff --git a/python/cuml/cuml/internals/api_decorators.py b/python/cuml/cuml/internals/api_decorators.py index 9507efda1e..dc7d69cba8 100644 --- a/python/cuml/cuml/internals/api_decorators.py +++ b/python/cuml/cuml/internals/api_decorators.py @@ -5,7 +5,6 @@ import contextlib import functools import inspect -import typing import numpy as np @@ -21,160 +20,131 @@ from cuml.internals.constants import CUML_WRAPPED_FLAG from cuml.internals.global_settings import GlobalSettings from cuml.internals.memory_utils import using_output_type -from cuml.internals.type_utils import _DecoratorType -def _wrap_once(wrapped, *args, **kwargs): - """Prevent wrapping functions multiple times.""" - setattr(wrapped, CUML_WRAPPED_FLAG, True) - return functools.wraps(wrapped, *args, **kwargs) +def reflect(func=None, *, on=0, reset=False, skip=False): + """Mark a function or method as participating in the reflection system. + + Parameters + ---------- + func : callable or None + The function to be decorated, or None to curry to be applied later. + on : int, str, or None, default=0 + The parameter to infer the reflected output type from. By default this + will be the first argument to the method or function (excluding + ``self``, unless there are no other arguments). Provide a parameter + position or name to override. May also provide None to disable + this inference entirely; in this case the output type is expected + to be specified manually either internal or external to the method. + reset : bool, default=False + Set to True for methods like ``fit`` that reset the reflected type on + an estimator. + skip : bool, default=False + Set to True to skip output processing for a method. This is mostly + useful if output processing will be handled manually. + """ + if func is None: + return lambda func: reflect(func, on=on, reset=reset, skip=skip) + + # TODO: remove this once auto-decorating is ripped out + setattr(func, CUML_WRAPPED_FLAG, True) + + sig = inspect.signature(func, follow_wrapped=True) + + if on is not None: + has_self = "self" in sig.parameters + + if isinstance(on, str): + param = sig.parameters[on] + elif on == 0 and has_self and len(sig.parameters) == 1: + # Default to self if there are no other parameters + param = sig.parameters["self"] + else: + # Otherwise exclude self, defaulting to first parameter + param = list(sig.parameters.values())[on + has_self] + + if param.kind in ( + inspect.Parameter.VAR_KEYWORD, + inspect.Parameter.VAR_POSITIONAL, + ): + raise ValueError("Cannot reflect on variadic args/kwargs") + + on = param.name + + @functools.wraps(func) + def inner(*args, **kwargs): + # Accept list/tuple inputs when accelerator is active + accept_lists = cuml.accel.enabled() + + # Bind arguments + bound = sig.bind(*args, **kwargs) + bound.apply_defaults() + + if on is None: + base = None + else: + on_arg = bound.arguments[on] + if accept_lists and isinstance(on_arg, (list, tuple)): + on_arg = np.asarray(on_arg) + + # Look for an estimator, first in `on` and then in `self` + if isinstance(on_arg, cuml.Base): + base = on_arg + elif has_self and isinstance(bound.arguments["self"], cuml.Base): + base = bound.arguments["self"] + else: + base = None + if reset and base is None: + raise ValueError("`reset=True` is only valid on estimator methods") -def _has_self(sig): - return "self" in sig.parameters and list(sig.parameters)[0] == "self" + with InternalAPIContextBase(base=base, process_return=not skip) as cm: + if reset: + base._set_output_type(on_arg) + base._set_n_features_in(on_arg) + if on is not None: + if isinstance(on_arg, cuml.Base): + out_type = on_arg._get_output_type() + elif base is not None: + out_type = base._get_output_type(on_arg) + else: + out_type = iu.determine_array_type(on_arg) -def _find_arg(sig, arg_name, default_position): - params = list(sig.parameters) + set_api_output_type(out_type) - # Check for default name in input args - if arg_name in sig.parameters: - param = sig.parameters[arg_name] - return arg_name, params.index(arg_name), param.default - # Otherwise use argument in list by position - elif arg_name is ...: - index = int(_has_self(sig)) + default_position - param = params[index] - return param, index, sig.parameters[param].default - else: - raise ValueError(f"Unable to find parameter '{arg_name}'.") + res = func(*args, **kwargs) + + if skip: + return res + return cm.process_return(res) + + return inner + + +def api_return_array(input_arg=0, get_output_type=False): + return reflect(on=None if not get_output_type else input_arg) + + +def api_return_any(): + return reflect(on=None, skip=True) + + +def api_base_return_any(): + return reflect(reset=True) + + +def api_base_return_array(input_arg=0): + return reflect(on="self" if input_arg is None else input_arg) + + +def api_base_fit_transform(): + return reflect(reset=True) -def _get_value(args, kwargs, name, index, default_value, accept_lists=False): - """Determine value for a given set of args, kwargs, name and index.""" - try: - value = kwargs[name] - except KeyError: - try: - value = args[index] - except IndexError: - if default_value is not inspect._empty: - value = default_value - else: - raise IndexError( - f"Specified arg idx: {index}, and argument name: {name}, " - "were not found in args or kwargs." - ) - # Accept list/tuple inputs when requested - if accept_lists and isinstance(value, (list, tuple)): - return np.asarray(value) - - return value - - -def _make_decorator( - process_return=True, - **defaults, -) -> typing.Callable[..., _DecoratorType]: - # This function generates a function to be applied as decorator to a - # wrapped function. For example: - # - # a_decorator = _make_decorator(...) - # - # ... - # - # @a_decorator(...) # apply decorator where appropriate - # def fit(X, y): - # ... - # - # Note: The decorator function can be partially closed by directly - # providing keyword arguments to this function to be used as defaults. - - def decorator_function( - input_arg: str = ..., - get_output_type: bool = False, - is_fit: bool = False, - ) -> _DecoratorType: - def decorator_closure(func): - # This function constitutes the closed decorator that will return - # the wrapped function. It performs function introspection at - # function definition time. The code within the wrapper function is - # executed at function execution time. - - # Prepare arguments - sig = inspect.signature(func, follow_wrapped=True) - - has_self = _has_self(sig) - - if input_arg is not None and (is_fit or get_output_type): - input_arg_ = _find_arg(sig, input_arg or "X", 0) - else: - input_arg_ = None - - @_wrap_once(func) - def wrapper(*args, **kwargs): - # Wraps the decorated function, executed at runtime. - - # Accept list/tuple inputs when accelerator is active - accept_lists = cuml.accel.enabled() - - self_val = args[0] if has_self else None - with InternalAPIContextBase( - func, - args, - is_base_method=isinstance(self_val, cuml.Base), - process_return=process_return, - ) as cm: - if input_arg_: - input_val = _get_value( - args, - kwargs, - *input_arg_, - accept_lists=accept_lists, - ) - else: - input_val = None - - if is_fit: - assert self_val is not None - self_val._set_output_type(input_val) - self_val._set_n_features_in(input_val) - - if get_output_type: - if self_val is None: - assert input_val is not None - out_type = iu.determine_array_type(input_val) - else: - out_type = self_val._get_output_type(input_val) - - set_api_output_type(out_type) - - if process_return: - ret = func(*args, **kwargs) - else: - return func(*args, **kwargs) - - return cm.process_return(ret) - - return wrapper - - return decorator_closure - - return functools.partial(decorator_function, **defaults) - - -# TODO: -# - infer get_output_type from whether return value is Base -# - determine why api_return_any is needed? It should only mark internal API? -# - infer `is_fit` based on method name by default -api_return_array = _make_decorator() -api_return_any = _make_decorator(process_return=False) -api_base_return_any = _make_decorator(is_fit=True, process_return=False) -api_base_return_array = _make_decorator(get_output_type=True) -api_base_fit_transform = _make_decorator(is_fit=True, get_output_type=True) # TODO: investigate and remove these api_base_return_any_skipall = api_return_any() -api_base_return_array_skipall = api_return_array() +api_base_return_array_skipall = reflect @contextlib.contextmanager diff --git a/python/cuml/cuml/linear_model/base.py b/python/cuml/cuml/linear_model/base.py index b509e76e61..e66dfa1440 100644 --- a/python/cuml/cuml/linear_model/base.py +++ b/python/cuml/cuml/linear_model/base.py @@ -21,7 +21,7 @@ class LinearPredictMixin: "shape": "(n_samples, 1)", } ) - @cuml.internals.api_base_return_array_skipall + @cuml.internals.api_base_return_array() def predict(self, X, *, convert_dtype=True) -> CumlArray: """ Predicts `y` values for `X`. @@ -63,7 +63,7 @@ class LinearClassifierMixin: "shape": "(n_samples,) or (n_samples, n_classes)", }, ) - @cuml.internals.api_base_return_array_skipall + @cuml.internals.api_base_return_array() def decision_function(self, X, *, convert_dtype=True) -> CumlArray: """Predict confidence scores for samples.""" if is_sparse(X): diff --git a/python/cuml/tests/test_cuml_descr_decor.py b/python/cuml/tests/test_cuml_descr_decor.py index 94be686edc..3e9446bc1c 100644 --- a/python/cuml/tests/test_cuml_descr_decor.py +++ b/python/cuml/tests/test_cuml_descr_decor.py @@ -266,7 +266,7 @@ def test_auto_predict(input_type, base_output_type, global_output_type): assert_array_identical(X_in, X_out) -@pytest.mark.parametrize("input_arg", ["X", "y", "bad", ...]) +@pytest.mark.parametrize("input_arg", ["X", "y", "bad", 0]) @pytest.mark.parametrize("get_output_type", [True, False]) def test_return_array(input_arg: str, get_output_type: bool): """ @@ -296,7 +296,7 @@ def test_func(X, y): input_arg=input_arg, get_output_type=get_output_type, )(test_func) - except ValueError: + except KeyError: assert expected_to_fail return else: From 11df678c37c95e6142dadee6e68e5728a9c64e1b Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Mon, 24 Nov 2025 22:17:00 -0600 Subject: [PATCH 13/28] Prepare to rip out `set_api_output_type` --- python/cuml/cuml/cluster/hdbscan/hdbscan.pyx | 15 +- python/cuml/cuml/datasets/arima.pyx | 7 +- python/cuml/cuml/datasets/blobs.py | 8 +- python/cuml/cuml/datasets/classification.py | 4 +- python/cuml/cuml/datasets/regression.pyx | 8 +- python/cuml/cuml/explainer/sampling.py | 3 +- python/cuml/cuml/internals/__init__.py | 1 + python/cuml/cuml/internals/api_decorators.py | 153 +++++++++++------- .../cuml/cuml/neighbors/nearest_neighbors.pyx | 7 +- 9 files changed, 106 insertions(+), 100 deletions(-) diff --git a/python/cuml/cuml/cluster/hdbscan/hdbscan.pyx b/python/cuml/cuml/cluster/hdbscan/hdbscan.pyx index f788246770..ff1e7ffd48 100644 --- a/python/cuml/cuml/cluster/hdbscan/hdbscan.pyx +++ b/python/cuml/cuml/cluster/hdbscan/hdbscan.pyx @@ -1114,7 +1114,7 @@ def _check_clusterer(clusterer): return state -@cuml.internals.api_return_array() +@cuml.internals.reflect(model="clusterer", array=None) def all_points_membership_vectors(clusterer, int batch_size=4096): """ Predict soft cluster membership vectors for all points in the @@ -1145,9 +1145,6 @@ def all_points_membership_vectors(clusterer, int batch_size=4096): if batch_size <= 0: raise ValueError("batch_size must be > 0") - # Reflect the output type from global settings or the clusterer - cuml.internals.set_api_output_type(clusterer._get_output_type()) - n_rows = clusterer._raw_data.shape[0] if clusterer.n_clusters_ == 0: @@ -1186,7 +1183,7 @@ def all_points_membership_vectors(clusterer, int batch_size=4096): return membership_vec -@cuml.internals.api_return_array() +@cuml.internals.reflect(model="clusterer", array="points_to_predict") def membership_vector(clusterer, points_to_predict, int batch_size=4096, convert_dtype=True): """ Predict soft cluster membership. The result produces a vector @@ -1223,9 +1220,6 @@ def membership_vector(clusterer, points_to_predict, int batch_size=4096, convert if batch_size <= 0: raise ValueError("batch_size must be > 0") - # Reflect the output type from global settings, the clusterer, or the input - cuml.internals.set_api_output_type(clusterer._get_output_type(points_to_predict)) - cdef int n_prediction_points points_to_predict_m, n_prediction_points, n_cols, _ = input_to_cuml_array( points_to_predict, @@ -1275,7 +1269,7 @@ def membership_vector(clusterer, points_to_predict, int batch_size=4096, convert return membership_vec -@cuml.internals.api_return_array() +@cuml.internals.reflect(model="clusterer", array="points_to_predict") def approximate_predict(clusterer, points_to_predict, convert_dtype=True): """Predict the cluster label of new points. The returned labels will be those of the original clustering found by ``clusterer``, @@ -1310,9 +1304,6 @@ def approximate_predict(clusterer, points_to_predict, convert_dtype=True): """ _check_clusterer(clusterer) - # Reflect the output type from global settings, the clusterer, or the input - cuml.internals.set_api_output_type(clusterer._get_output_type(points_to_predict)) - if clusterer.n_clusters_ == 0: logger.warn( "Clusterer does not have any defined clusters, new data " diff --git a/python/cuml/cuml/datasets/arima.pyx b/python/cuml/cuml/datasets/arima.pyx index a6948cf89f..561f89f3ac 100644 --- a/python/cuml/cuml/datasets/arima.pyx +++ b/python/cuml/cuml/datasets/arima.pyx @@ -54,7 +54,7 @@ inp_to_dtype = { } -@cuml.internals.api_return_array() +@cuml.internals.reflect(array=None, default_output_type="cupy") def make_arima(batch_size=1000, n_obs=100, order=(1, 1, 1), seasonal_order=(0, 0, 0, 0), intercept=False, random_state=None, dtype='double', @@ -102,11 +102,6 @@ def make_arima(batch_size=1000, n_obs=100, order=(1, 1, 1), cpp_order.k = intercept cpp_order.n_exog = 0 - # Set the default output type to "cupy". This will be ignored if the user - # has set `cuml.global_settings.output_type`. Only necessary for array - # generation methods that do not take an array as input - cuml.internals.set_api_output_type("cupy") - # Define some parameters based on the order scale = 1.0 noise_scale = 0.2 diff --git a/python/cuml/cuml/datasets/blobs.py b/python/cuml/cuml/datasets/blobs.py index 8afe878b68..9a3ee1fd72 100644 --- a/python/cuml/cuml/datasets/blobs.py +++ b/python/cuml/cuml/datasets/blobs.py @@ -71,7 +71,7 @@ def _get_centers(rs, centers, center_box, n_samples, n_features, dtype): @nvtx.annotate(message="datasets.make_blobs", domain="cuml_python") -@cuml.internals.api_return_array() +@cuml.internals.reflect(array=None, default_output_type="cupy") def make_blobs( n_samples=100, n_features=2, @@ -151,12 +151,6 @@ def make_blobs( -------- make_classification: a more intricate variant """ - - # Set the default output type to "cupy". This will be ignored if the user - # has set `cuml.global_settings.output_type`. Only necessary for array - # generation methods that do not take an array as input - cuml.internals.set_api_output_type("cupy") - generator = _create_rs_generator(random_state=random_state) centers, n_centers = _get_centers( diff --git a/python/cuml/cuml/datasets/classification.py b/python/cuml/cuml/datasets/classification.py index af282523cc..d51e15ce26 100644 --- a/python/cuml/cuml/datasets/classification.py +++ b/python/cuml/cuml/datasets/classification.py @@ -33,7 +33,7 @@ def _generate_hypercube(samples, dimensions, random_state): @nvtx.annotate(message="datasets.make_classification", domain="cuml_python") -@cuml.internals.api_return_array() +@cuml.internals.reflect(array=None, default_output_type="cupy") def make_classification( n_samples=100, n_features=20, @@ -202,8 +202,6 @@ def make_classification( selection benchmark", 2003. """ - cuml.internals.set_api_output_type("cupy") - generator = _create_rs_generator(random_state) # Count features, clusters and samples diff --git a/python/cuml/cuml/datasets/regression.pyx b/python/cuml/cuml/datasets/regression.pyx index 5d1986f4e6..09f9b0ea94 100644 --- a/python/cuml/cuml/datasets/regression.pyx +++ b/python/cuml/cuml/datasets/regression.pyx @@ -63,7 +63,7 @@ inp_to_dtype = { @nvtx.annotate(message="datasets.make_regression", domain="cuml_python") -@cuml.internals.api_return_array() +@cuml.internals.reflect(array=None, default_output_type="cupy") def make_regression( n_samples=100, n_features=2, @@ -158,12 +158,6 @@ def make_regression( The coefficient of the underlying linear model. It is returned only if coef is True. """ # noqa: E501 - - # Set the default output type to "cupy". This will be ignored if the user - # has set `cuml.global_settings.output_type`. Only necessary for array - # generation methods that do not take an array as input - cuml.internals.set_api_output_type("cupy") - if dtype not in ['single', 'float', 'double', np.float32, np.float64]: raise TypeError("dtype must be either 'float' or 'double'") else: diff --git a/python/cuml/cuml/explainer/sampling.py b/python/cuml/cuml/explainer/sampling.py index 2f93f6dcd4..668b1d7e37 100644 --- a/python/cuml/cuml/explainer/sampling.py +++ b/python/cuml/cuml/explainer/sampling.py @@ -13,7 +13,7 @@ from cuml.preprocessing import SimpleImputer -@cuml.internals.api_return_array() +@cuml.internals.reflect def kmeans_sampling(X, k, round_values=True, detailed=False, random_state=0): """ Adapted from : @@ -45,7 +45,6 @@ def kmeans_sampling(X, k, round_values=True, detailed=False, random_state=0): """ output_dtype = get_supported_input_type(X) _output_dtype_str = determine_array_type(X) - cuml.internals.set_api_output_type(_output_dtype_str) if output_dtype is None: raise TypeError( diff --git a/python/cuml/cuml/internals/__init__.py b/python/cuml/cuml/internals/__init__.py index c2bc395eed..18d6f737a0 100644 --- a/python/cuml/cuml/internals/__init__.py +++ b/python/cuml/cuml/internals/__init__.py @@ -13,6 +13,7 @@ api_return_any, api_return_array, exit_internal_api, + reflect, ) from cuml.internals.base_helpers import BaseMetaClass, _tags_class_and_instance from cuml.internals.constants import CUML_WRAPPED_FLAG diff --git a/python/cuml/cuml/internals/api_decorators.py b/python/cuml/cuml/internals/api_decorators.py index dc7d69cba8..db31905997 100644 --- a/python/cuml/cuml/internals/api_decorators.py +++ b/python/cuml/cuml/internals/api_decorators.py @@ -21,55 +21,97 @@ from cuml.internals.global_settings import GlobalSettings from cuml.internals.memory_utils import using_output_type - -def reflect(func=None, *, on=0, reset=False, skip=False): +default = type( + "default", + (), + dict.fromkeys(["__repr__", "__reduce__"], lambda s: "default"), +)() + + +def _get_param(sig, name_or_index): + if isinstance(name_or_index, str): + param = sig.parameters[name_or_index] + else: + param = list(sig.parameters.values())[name_or_index] + + if param.kind in ( + inspect.Parameter.VAR_KEYWORD, + inspect.Parameter.VAR_POSITIONAL, + ): + raise ValueError("Cannot reflect variadic args/kwargs") + + return param.name + + +def reflect( + func=None, + *, + array=default, + model=default, + reset=False, + skip=False, + default_output_type=None, +): """Mark a function or method as participating in the reflection system. Parameters ---------- func : callable or None The function to be decorated, or None to curry to be applied later. - on : int, str, or None, default=0 - The parameter to infer the reflected output type from. By default this - will be the first argument to the method or function (excluding - ``self``, unless there are no other arguments). Provide a parameter - position or name to override. May also provide None to disable - this inference entirely; in this case the output type is expected - to be specified manually either internal or external to the method. + model : int, str, or None, default=default + The ``cuml.Base`` parameter to infer the reflected output type from. By + default this will be ``'self'`` (if present), and ``None`` otherwise. + Provide a parameter position or name to override. May also provide + ``None`` to disable this inference entirely. + array : int, str, or None, default=default + The array-like parameter to infer the reflected output type from. By + default this will be the first argument to the method or function + (excluding ``'self'`` or ``model``), or ``None`` if there are no other + arguments. Provide a parameter position or name to override. May also + provide ``None`` to disable this inference entirely; in this case the + output type is expected to be specified manually either internal or + external to the method. reset : bool, default=False Set to True for methods like ``fit`` that reset the reflected type on an estimator. skip : bool, default=False Set to True to skip output processing for a method. This is mostly useful if output processing will be handled manually. + default_output_type : str or None, default=None + The default output type to use for a method when no output type + has been set externally. """ if func is None: - return lambda func: reflect(func, on=on, reset=reset, skip=skip) + return lambda func: reflect( + func, + model=model, + array=array, + reset=reset, + skip=skip, + default_output_type=default_output_type, + ) # TODO: remove this once auto-decorating is ripped out setattr(func, CUML_WRAPPED_FLAG, True) sig = inspect.signature(func, follow_wrapped=True) + has_self = "self" in sig.parameters - if on is not None: - has_self = "self" in sig.parameters + if model is default: + model = "self" if has_self else None + if model is not None: + model = _get_param(sig, model) - if isinstance(on, str): - param = sig.parameters[on] - elif on == 0 and has_self and len(sig.parameters) == 1: - # Default to self if there are no other parameters - param = sig.parameters["self"] + if array is default: + if model is not None and list(sig.parameters).index(model) == 0: + array = 1 else: - # Otherwise exclude self, defaulting to first parameter - param = list(sig.parameters.values())[on + has_self] - - if param.kind in ( - inspect.Parameter.VAR_KEYWORD, - inspect.Parameter.VAR_POSITIONAL, - ): - raise ValueError("Cannot reflect on variadic args/kwargs") - - on = param.name + array = 0 + if len(sig.parameters) <= array: + # Not enough parameters, no array-like param to infer from + array = None + if array is not None: + array = _get_param(sig, array) @functools.wraps(func) def inner(*args, **kwargs): @@ -80,37 +122,34 @@ def inner(*args, **kwargs): bound = sig.bind(*args, **kwargs) bound.apply_defaults() - if on is None: - base = None - else: - on_arg = bound.arguments[on] - if accept_lists and isinstance(on_arg, (list, tuple)): - on_arg = np.asarray(on_arg) - - # Look for an estimator, first in `on` and then in `self` - if isinstance(on_arg, cuml.Base): - base = on_arg - elif has_self and isinstance(bound.arguments["self"], cuml.Base): - base = bound.arguments["self"] - else: - base = None + model_arg = None if model is None else bound.arguments[model] + array_arg = None if array is None else bound.arguments[array] + if accept_lists and isinstance(array_arg, (list, tuple)): + array_arg = np.asarray(array_arg) - if reset and base is None: + if reset and model_arg is None: raise ValueError("`reset=True` is only valid on estimator methods") - with InternalAPIContextBase(base=base, process_return=not skip) as cm: + with InternalAPIContextBase( + base=model_arg, process_return=not skip + ) as cm: if reset: - base._set_output_type(on_arg) - base._set_n_features_in(on_arg) - - if on is not None: - if isinstance(on_arg, cuml.Base): - out_type = on_arg._get_output_type() - elif base is not None: - out_type = base._get_output_type(on_arg) + model_arg._set_output_type(array_arg) + model_arg._set_n_features_in(array_arg) + + if model is not None: + if array is not None: + out_type = model_arg._get_output_type(array_arg) else: - out_type = iu.determine_array_type(on_arg) + out_type = model_arg._get_output_type() + elif array is not None: + out_type = iu.determine_array_type(array_arg) + elif default_output_type is not None: + out_type = default_output_type + else: + out_type = None + if out_type is not None: set_api_output_type(out_type) res = func(*args, **kwargs) @@ -122,20 +161,20 @@ def inner(*args, **kwargs): return inner -def api_return_array(input_arg=0, get_output_type=False): - return reflect(on=None if not get_output_type else input_arg) +def api_return_array(input_arg=default, get_output_type=False): + return reflect(array=None if not get_output_type else input_arg) def api_return_any(): - return reflect(on=None, skip=True) + return reflect(array=None, skip=True) def api_base_return_any(): return reflect(reset=True) -def api_base_return_array(input_arg=0): - return reflect(on="self" if input_arg is None else input_arg) +def api_base_return_array(input_arg=default): + return reflect(array="self" if input_arg is None else input_arg) def api_base_fit_transform(): diff --git a/python/cuml/cuml/neighbors/nearest_neighbors.pyx b/python/cuml/cuml/neighbors/nearest_neighbors.pyx index c183503384..eb755afe08 100644 --- a/python/cuml/cuml/neighbors/nearest_neighbors.pyx +++ b/python/cuml/cuml/neighbors/nearest_neighbors.pyx @@ -1095,7 +1095,7 @@ class NearestNeighbors(Base, return self.metric_params or {} -@cuml.internals.api_return_array() +@cuml.internals.reflect def kneighbors_graph(X=None, n_neighbors=5, mode='connectivity', verbose=False, handle=None, algorithm="brute", metric="euclidean", p=2, include_self=False, metric_params=None): @@ -1174,11 +1174,6 @@ def kneighbors_graph(X=None, n_neighbors=5, mode='connectivity', verbose=False, numpy's CSR sparse graph (host) """ - # Set the default output type to "cupy". This will be ignored if the user - # has set `cuml.global_settings.output_type`. Only necessary for array - # generation methods that do not take an array as input - cuml.internals.set_api_output_type("cupy") - X = NearestNeighbors( n_neighbors=n_neighbors, verbose=verbose, From f7b2c434292293d892326171a04d22892e5e4142 Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Mon, 24 Nov 2025 22:23:28 -0600 Subject: [PATCH 14/28] Rip out last external ref to `root_cm` --- python/cuml/cuml/neighbors/nearest_neighbors.pyx | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/python/cuml/cuml/neighbors/nearest_neighbors.pyx b/python/cuml/cuml/neighbors/nearest_neighbors.pyx index eb755afe08..b795120e15 100644 --- a/python/cuml/cuml/neighbors/nearest_neighbors.pyx +++ b/python/cuml/cuml/neighbors/nearest_neighbors.pyx @@ -1174,7 +1174,7 @@ def kneighbors_graph(X=None, n_neighbors=5, mode='connectivity', verbose=False, numpy's CSR sparse graph (host) """ - X = NearestNeighbors( + model = NearestNeighbors( n_neighbors=n_neighbors, verbose=verbose, handle=handle, @@ -1182,16 +1182,15 @@ def kneighbors_graph(X=None, n_neighbors=5, mode='connectivity', verbose=False, metric=metric, p=p, metric_params=metric_params, - output_type=cuml.global_settings.root_cm.output_type + output_type="cupy", ).fit(X) if include_self == 'auto': include_self = mode == 'connectivity' - with cuml.internals.exit_internal_api(): - if not include_self: - query = None - else: - query = X._fit_X + if not include_self: + query = None + else: + query = model._fit_X - return X.kneighbors_graph(X=query, n_neighbors=n_neighbors, mode=mode) + return model.kneighbors_graph(X=query, n_neighbors=n_neighbors, mode=mode) From c6ddca8202bddeb65d3f98e2e745c8267c5b40a2 Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Tue, 25 Nov 2025 10:27:39 -0600 Subject: [PATCH 15/28] Reorg output handling into `cuml.internals.outputs` Old implementation was scattered between too many files. Also deletes some dead code. --- python/cuml/cuml/__init__.py | 5 +- .../sklearn/preprocessing/_data.py | 2 +- .../sklearn/preprocessing/_discretization.py | 2 +- .../accel/_wrappers/sklearn/linear_model.py | 2 +- python/cuml/cuml/accel/core.py | 2 +- python/cuml/cuml/common/__init__.py | 5 +- python/cuml/cuml/common/classification.py | 3 +- python/cuml/cuml/internals/__init__.py | 10 +- python/cuml/cuml/internals/api_decorators.py | 204 -------------- python/cuml/cuml/internals/array.py | 12 +- python/cuml/cuml/internals/base.py | 2 +- python/cuml/cuml/internals/base_helpers.py | 6 +- python/cuml/cuml/internals/interop.py | 2 +- python/cuml/cuml/internals/mixins.py | 2 +- python/cuml/cuml/internals/output_type.py | 17 -- .../internals/{memory_utils.py => outputs.py} | 251 ++++++++++++++---- python/cuml/cuml/kernel_ridge/kernel_ridge.py | 2 +- .../cuml/linear_model/linear_regression.pyx | 3 +- python/cuml/cuml/linear_model/ridge.pyx | 3 +- .../cuml/cuml/neighbors/nearest_neighbors.pyx | 2 +- .../tests/internals/test_internal_utils.py | 69 ----- python/cuml/tests/test_array.py | 28 +- python/cuml/tests/test_cuml_descr_decor.py | 2 +- python/cuml/tests/test_input_utils.py | 21 +- 24 files changed, 279 insertions(+), 378 deletions(-) delete mode 100644 python/cuml/cuml/internals/api_decorators.py delete mode 100644 python/cuml/cuml/internals/output_type.py rename python/cuml/cuml/internals/{memory_utils.py => outputs.py} (53%) delete mode 100644 python/cuml/tests/internals/test_internal_utils.py diff --git a/python/cuml/cuml/__init__.py b/python/cuml/cuml/__init__.py index ed478f9e59..3fbd69e9be 100644 --- a/python/cuml/cuml/__init__.py +++ b/python/cuml/cuml/__init__.py @@ -42,10 +42,7 @@ GlobalSettings, _global_settings_data, ) -from cuml.internals.memory_utils import ( - set_global_output_type, - using_output_type, -) +from cuml.internals.outputs import set_global_output_type, using_output_type from cuml.kernel_ridge.kernel_ridge import KernelRidge from cuml.linear_model.elastic_net import ElasticNet from cuml.linear_model.lasso import Lasso diff --git a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_data.py b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_data.py index dc8ba64147..b353b0e766 100644 --- a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_data.py +++ b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_data.py @@ -45,7 +45,7 @@ from ....internals import api_return_array from ....internals.array import CumlArray from ....internals.array_sparse import SparseCumlArray -from ....internals.memory_utils import using_output_type +from ....internals.outputs import using_output_type from ....thirdparty_adapters import check_array from ....thirdparty_adapters.sparsefuncs_fast import ( csr_polynomial_expansion, diff --git a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_discretization.py b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_discretization.py index 0efffc5909..f2cb73157c 100644 --- a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_discretization.py +++ b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_discretization.py @@ -27,7 +27,7 @@ from ....common.array_descriptor import CumlArrayDescriptor from ....internals.array_sparse import SparseCumlArray -from ....internals.memory_utils import using_output_type +from ....internals.outputs import using_output_type from ....thirdparty_adapters import check_array from ..utils.skl_dependencies import BaseEstimator, TransformerMixin from ..utils.validation import FLOAT_DTYPES, check_is_fitted diff --git a/python/cuml/cuml/accel/_wrappers/sklearn/linear_model.py b/python/cuml/cuml/accel/_wrappers/sklearn/linear_model.py index 136bcf5b36..6c1f924a45 100644 --- a/python/cuml/cuml/accel/_wrappers/sklearn/linear_model.py +++ b/python/cuml/cuml/accel/_wrappers/sklearn/linear_model.py @@ -9,7 +9,7 @@ import cuml.linear_model from cuml.accel.estimator_proxy import ProxyBase from cuml.internals.array import CumlArray -from cuml.internals.memory_utils import using_output_type +from cuml.internals.outputs import using_output_type __all__ = ( "LinearRegression", diff --git a/python/cuml/cuml/accel/core.py b/python/cuml/cuml/accel/core.py index f358975435..b53648fba3 100644 --- a/python/cuml/cuml/accel/core.py +++ b/python/cuml/cuml/accel/core.py @@ -11,7 +11,7 @@ from cuda.bindings import runtime from cuml.accel.accelerator import Accelerator -from cuml.internals.memory_utils import set_global_output_type +from cuml.internals.outputs import set_global_output_type class Logger: diff --git a/python/cuml/cuml/common/__init__.py b/python/cuml/cuml/common/__init__.py index dadc04903a..e6972119ff 100644 --- a/python/cuml/cuml/common/__init__.py +++ b/python/cuml/cuml/common/__init__.py @@ -13,7 +13,4 @@ input_to_host_array_with_sparse_support, sparse_scipy_to_cp, ) -from cuml.internals.memory_utils import ( - set_global_output_type, - using_output_type, -) +from cuml.internals.outputs import set_global_output_type, using_output_type diff --git a/python/cuml/cuml/common/classification.py b/python/cuml/cuml/common/classification.py index a2617de9c5..ee21a7d2c2 100644 --- a/python/cuml/cuml/common/classification.py +++ b/python/cuml/cuml/common/classification.py @@ -7,9 +7,8 @@ import numpy as np import pandas as pd -from cuml.internals.array import CumlArray +from cuml.internals.array import CumlArray, cuda_ptr from cuml.internals.input_utils import input_to_cuml_array, input_to_cupy_array -from cuml.internals.memory_utils import cuda_ptr from cuml.internals.output_utils import cudf_to_pandas is_integral = cp.ReductionKernel( diff --git a/python/cuml/cuml/internals/__init__.py b/python/cuml/cuml/internals/__init__.py index 18d6f737a0..1b11185ea6 100644 --- a/python/cuml/cuml/internals/__init__.py +++ b/python/cuml/cuml/internals/__init__.py @@ -2,9 +2,10 @@ # SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - -from cuml.internals.api_context_managers import set_api_output_type -from cuml.internals.api_decorators import ( +from cuml.internals.base_helpers import BaseMetaClass, _tags_class_and_instance +from cuml.internals.constants import CUML_WRAPPED_FLAG +from cuml.internals.internals import GraphBasedDimRedCallback +from cuml.internals.outputs import ( api_base_fit_transform, api_base_return_any, api_base_return_any_skipall, @@ -15,6 +16,3 @@ exit_internal_api, reflect, ) -from cuml.internals.base_helpers import BaseMetaClass, _tags_class_and_instance -from cuml.internals.constants import CUML_WRAPPED_FLAG -from cuml.internals.internals import GraphBasedDimRedCallback diff --git a/python/cuml/cuml/internals/api_decorators.py b/python/cuml/cuml/internals/api_decorators.py deleted file mode 100644 index db31905997..0000000000 --- a/python/cuml/cuml/internals/api_decorators.py +++ /dev/null @@ -1,204 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 -# -import contextlib -import functools -import inspect - -import numpy as np - -import cuml -import cuml.accel - -# TODO: Try to resolve circular import that makes this necessary: -from cuml.internals import input_utils as iu -from cuml.internals.api_context_managers import ( - InternalAPIContextBase, - set_api_output_type, -) -from cuml.internals.constants import CUML_WRAPPED_FLAG -from cuml.internals.global_settings import GlobalSettings -from cuml.internals.memory_utils import using_output_type - -default = type( - "default", - (), - dict.fromkeys(["__repr__", "__reduce__"], lambda s: "default"), -)() - - -def _get_param(sig, name_or_index): - if isinstance(name_or_index, str): - param = sig.parameters[name_or_index] - else: - param = list(sig.parameters.values())[name_or_index] - - if param.kind in ( - inspect.Parameter.VAR_KEYWORD, - inspect.Parameter.VAR_POSITIONAL, - ): - raise ValueError("Cannot reflect variadic args/kwargs") - - return param.name - - -def reflect( - func=None, - *, - array=default, - model=default, - reset=False, - skip=False, - default_output_type=None, -): - """Mark a function or method as participating in the reflection system. - - Parameters - ---------- - func : callable or None - The function to be decorated, or None to curry to be applied later. - model : int, str, or None, default=default - The ``cuml.Base`` parameter to infer the reflected output type from. By - default this will be ``'self'`` (if present), and ``None`` otherwise. - Provide a parameter position or name to override. May also provide - ``None`` to disable this inference entirely. - array : int, str, or None, default=default - The array-like parameter to infer the reflected output type from. By - default this will be the first argument to the method or function - (excluding ``'self'`` or ``model``), or ``None`` if there are no other - arguments. Provide a parameter position or name to override. May also - provide ``None`` to disable this inference entirely; in this case the - output type is expected to be specified manually either internal or - external to the method. - reset : bool, default=False - Set to True for methods like ``fit`` that reset the reflected type on - an estimator. - skip : bool, default=False - Set to True to skip output processing for a method. This is mostly - useful if output processing will be handled manually. - default_output_type : str or None, default=None - The default output type to use for a method when no output type - has been set externally. - """ - if func is None: - return lambda func: reflect( - func, - model=model, - array=array, - reset=reset, - skip=skip, - default_output_type=default_output_type, - ) - - # TODO: remove this once auto-decorating is ripped out - setattr(func, CUML_WRAPPED_FLAG, True) - - sig = inspect.signature(func, follow_wrapped=True) - has_self = "self" in sig.parameters - - if model is default: - model = "self" if has_self else None - if model is not None: - model = _get_param(sig, model) - - if array is default: - if model is not None and list(sig.parameters).index(model) == 0: - array = 1 - else: - array = 0 - if len(sig.parameters) <= array: - # Not enough parameters, no array-like param to infer from - array = None - if array is not None: - array = _get_param(sig, array) - - @functools.wraps(func) - def inner(*args, **kwargs): - # Accept list/tuple inputs when accelerator is active - accept_lists = cuml.accel.enabled() - - # Bind arguments - bound = sig.bind(*args, **kwargs) - bound.apply_defaults() - - model_arg = None if model is None else bound.arguments[model] - array_arg = None if array is None else bound.arguments[array] - if accept_lists and isinstance(array_arg, (list, tuple)): - array_arg = np.asarray(array_arg) - - if reset and model_arg is None: - raise ValueError("`reset=True` is only valid on estimator methods") - - with InternalAPIContextBase( - base=model_arg, process_return=not skip - ) as cm: - if reset: - model_arg._set_output_type(array_arg) - model_arg._set_n_features_in(array_arg) - - if model is not None: - if array is not None: - out_type = model_arg._get_output_type(array_arg) - else: - out_type = model_arg._get_output_type() - elif array is not None: - out_type = iu.determine_array_type(array_arg) - elif default_output_type is not None: - out_type = default_output_type - else: - out_type = None - - if out_type is not None: - set_api_output_type(out_type) - - res = func(*args, **kwargs) - - if skip: - return res - return cm.process_return(res) - - return inner - - -def api_return_array(input_arg=default, get_output_type=False): - return reflect(array=None if not get_output_type else input_arg) - - -def api_return_any(): - return reflect(array=None, skip=True) - - -def api_base_return_any(): - return reflect(reset=True) - - -def api_base_return_array(input_arg=default): - return reflect(array="self" if input_arg is None else input_arg) - - -def api_base_fit_transform(): - return reflect(reset=True) - - -# TODO: investigate and remove these -api_base_return_any_skipall = api_return_any() -api_base_return_array_skipall = reflect - - -@contextlib.contextmanager -def exit_internal_api(): - assert GlobalSettings().root_cm is not None - - try: - old_root_cm = GlobalSettings().root_cm - - GlobalSettings().root_cm = None - - # Set the global output type to the previous value to pretend we never - # entered the API - with using_output_type(old_root_cm.prev_output_type): - yield - - finally: - GlobalSettings().root_cm = old_root_cm diff --git a/python/cuml/cuml/internals/array.py b/python/cuml/cuml/internals/array.py index f128fe3816..db87a4427e 100644 --- a/python/cuml/cuml/internals/array.py +++ b/python/cuml/cuml/internals/array.py @@ -15,7 +15,6 @@ from numba import cuda from numba.cuda import is_cuda_array as is_numba_array -import cuml.accel import cuml.internals.nvtx as nvtx from cuml.internals.logger import debug from cuml.internals.mem_type import MemoryType, MemoryTypeError @@ -202,6 +201,9 @@ def __init__( self._mem_type = MemoryType.device self._owner = data else: # Not a CUDA array object + # Local to avoid circular imports + import cuml.accel + if hasattr(data, "__array_interface__"): self._array_interface = data.__array_interface__ self._mem_type = MemoryType.host @@ -974,6 +976,7 @@ def from_input( """ # Local to workaround circular imports + import cuml.accel from cuml.common.sparse_utils import is_sparse if is_sparse(X): @@ -1225,6 +1228,13 @@ def is_array_contiguous(arr): return array_to_memory_order(arr) is not None +def cuda_ptr(X): + """Returns a pointer to a backing device array, or None if not a device array""" + if (interface := getattr(X, "__cuda_array_interface__", None)) is not None: + return interface["data"][0] + return None + + def elements_in_representable_range(arr, dtype): """Return true if all elements of the array can be represented in the available range of the given dtype""" diff --git a/python/cuml/cuml/internals/base.py b/python/cuml/cuml/internals/base.py index 959e4d1b5a..8956357642 100644 --- a/python/cuml/cuml/internals/base.py +++ b/python/cuml/cuml/internals/base.py @@ -16,7 +16,7 @@ import cuml.internals.nvtx as nvtx from cuml.internals.input_utils import determine_array_type from cuml.internals.mixins import TagsMixin -from cuml.internals.output_type import ( +from cuml.internals.outputs import ( INTERNAL_VALID_OUTPUT_TYPES, VALID_OUTPUT_TYPES, ) diff --git a/python/cuml/cuml/internals/base_helpers.py b/python/cuml/cuml/internals/base_helpers.py index 03ae5290f0..92d58b9f7c 100644 --- a/python/cuml/cuml/internals/base_helpers.py +++ b/python/cuml/cuml/internals/base_helpers.py @@ -3,13 +3,13 @@ # SPDX-License-Identifier: Apache-2.0 # -from cuml.internals.api_decorators import ( +from cuml.internals.base_return_types import _get_base_return_type +from cuml.internals.constants import CUML_WRAPPED_FLAG +from cuml.internals.outputs import ( api_base_return_any, api_base_return_array, api_return_any, ) -from cuml.internals.base_return_types import _get_base_return_type -from cuml.internals.constants import CUML_WRAPPED_FLAG def _wrap_attribute(class_name: str, attribute_name: str, attribute, **kwargs): diff --git a/python/cuml/cuml/internals/interop.py b/python/cuml/cuml/internals/interop.py index 83513e014c..8de2d00bb5 100644 --- a/python/cuml/cuml/internals/interop.py +++ b/python/cuml/cuml/internals/interop.py @@ -8,7 +8,7 @@ import numpy as np from cuml.internals.mem_type import MemoryType -from cuml.internals.memory_utils import using_output_type +from cuml.internals.outputs import using_output_type __all__ = ( "UnsupportedOnGPU", diff --git a/python/cuml/cuml/internals/mixins.py b/python/cuml/cuml/internals/mixins.py index ee0c67d2b9..096c99c36b 100644 --- a/python/cuml/cuml/internals/mixins.py +++ b/python/cuml/cuml/internals/mixins.py @@ -8,8 +8,8 @@ from cuml._thirdparty._sklearn_compat import _to_new_tags from cuml.common.doc_utils import generate_docstring -from cuml.internals.api_decorators import api_base_return_any_skipall from cuml.internals.base_helpers import _tags_class_and_instance +from cuml.internals.outputs import api_base_return_any_skipall ############################################################################### # Tag Functionality Mixin # diff --git a/python/cuml/cuml/internals/output_type.py b/python/cuml/cuml/internals/output_type.py deleted file mode 100644 index b1618f9b4f..0000000000 --- a/python/cuml/cuml/internals/output_type.py +++ /dev/null @@ -1,17 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 2022-2023, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 -# -VALID_OUTPUT_TYPES = ( - "array", - "numba", - "dataframe", - "series", - "df_obj", - "cupy", - "numpy", - "cudf", - "pandas", -) - -INTERNAL_VALID_OUTPUT_TYPES = ("input", *VALID_OUTPUT_TYPES) diff --git a/python/cuml/cuml/internals/memory_utils.py b/python/cuml/cuml/internals/outputs.py similarity index 53% rename from python/cuml/cuml/internals/memory_utils.py rename to python/cuml/cuml/internals/outputs.py index 34b81f9941..e51a286109 100644 --- a/python/cuml/cuml/internals/memory_utils.py +++ b/python/cuml/cuml/internals/outputs.py @@ -2,40 +2,40 @@ # SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # +import contextlib import functools -import operator +import inspect -import cudf import numpy as np -import pandas as pd -from cuml.internals.global_settings import GlobalSettings -from cuml.internals.mem_type import MemoryType -from cuml.internals.output_type import ( - INTERNAL_VALID_OUTPUT_TYPES, - VALID_OUTPUT_TYPES, +# TODO: Try to resolve circular import that makes this necessary: +from cuml.internals import input_utils as iu +from cuml.internals.api_context_managers import ( + InternalAPIContextBase, + set_api_output_type, ) +from cuml.internals.constants import CUML_WRAPPED_FLAG +from cuml.internals.global_settings import GlobalSettings +default = type( + "default", + (), + dict.fromkeys(["__repr__", "__reduce__"], lambda s: "default"), +)() + +VALID_OUTPUT_TYPES = ( + "array", + "numba", + "dataframe", + "series", + "df_obj", + "cupy", + "numpy", + "cudf", + "pandas", +) -def _get_size_from_shape(shape, dtype): - """ - Calculates size based on shape and dtype, returns (None, None) if either - shape or dtype are None - """ - - if shape is None or dtype is None: - return (None, None) - - itemsize = np.dtype(dtype).itemsize - if isinstance(shape, int): - size = itemsize * shape - shape = (shape,) - elif isinstance(shape, tuple): - size = functools.reduce(operator.mul, shape) - size = size * itemsize - else: - raise ValueError("Shape must be int or tuple of ints.") - return (size, shape) +INTERNAL_VALID_OUTPUT_TYPES = ("input", *VALID_OUTPUT_TYPES) def set_global_output_type(output_type): @@ -79,7 +79,6 @@ def set_global_output_type(output_type): Examples -------- - >>> import cuml >>> import cupy as cp >>> ary = [[1.0, 4.0, 4.0], [2.0, 2.0, 2.0], [5.0, 1.0, 1.0]] @@ -212,24 +211,180 @@ def __exit__(self, *_): GlobalSettings().output_type = self.prev_output_type -def determine_array_memtype(X): +def _get_param(sig, name_or_index): + if isinstance(name_or_index, str): + param = sig.parameters[name_or_index] + else: + param = list(sig.parameters.values())[name_or_index] + + if param.kind in ( + inspect.Parameter.VAR_KEYWORD, + inspect.Parameter.VAR_POSITIONAL, + ): + raise ValueError("Cannot reflect variadic args/kwargs") + + return param.name + + +def reflect( + func=None, + *, + array=default, + model=default, + reset=False, + skip=False, + default_output_type=None, +): + """Mark a function or method as participating in the reflection system. + + Parameters + ---------- + func : callable or None + The function to be decorated, or None to curry to be applied later. + model : int, str, or None, default=default + The ``cuml.Base`` parameter to infer the reflected output type from. By + default this will be ``'self'`` (if present), and ``None`` otherwise. + Provide a parameter position or name to override. May also provide + ``None`` to disable this inference entirely. + array : int, str, or None, default=default + The array-like parameter to infer the reflected output type from. By + default this will be the first argument to the method or function + (excluding ``'self'`` or ``model``), or ``None`` if there are no other + arguments. Provide a parameter position or name to override. May also + provide ``None`` to disable this inference entirely; in this case the + output type is expected to be specified manually either internal or + external to the method. + reset : bool, default=False + Set to True for methods like ``fit`` that reset the reflected type on + an estimator. + skip : bool, default=False + Set to True to skip output processing for a method. This is mostly + useful if output processing will be handled manually. + default_output_type : str or None, default=None + The default output type to use for a method when no output type + has been set externally. + """ + # Local to avoid circular imports + import cuml.accel + + if func is None: + return lambda func: reflect( + func, + model=model, + array=array, + reset=reset, + skip=skip, + default_output_type=default_output_type, + ) + + # TODO: remove this once auto-decorating is ripped out + setattr(func, CUML_WRAPPED_FLAG, True) + + sig = inspect.signature(func, follow_wrapped=True) + has_self = "self" in sig.parameters + + if model is default: + model = "self" if has_self else None + if model is not None: + model = _get_param(sig, model) + + if array is default: + if model is not None and list(sig.parameters).index(model) == 0: + array = 1 + else: + array = 0 + if len(sig.parameters) <= array: + # Not enough parameters, no array-like param to infer from + array = None + if array is not None: + array = _get_param(sig, array) + + @functools.wraps(func) + def inner(*args, **kwargs): + # Accept list/tuple inputs when accelerator is active + accept_lists = cuml.accel.enabled() + + # Bind arguments + bound = sig.bind(*args, **kwargs) + bound.apply_defaults() + + model_arg = None if model is None else bound.arguments[model] + array_arg = None if array is None else bound.arguments[array] + if accept_lists and isinstance(array_arg, (list, tuple)): + array_arg = np.asarray(array_arg) + + if reset and model_arg is None: + raise ValueError("`reset=True` is only valid on estimator methods") + + with InternalAPIContextBase( + base=model_arg, process_return=not skip + ) as cm: + if reset: + model_arg._set_output_type(array_arg) + model_arg._set_n_features_in(array_arg) + + if model is not None: + if array is not None: + out_type = model_arg._get_output_type(array_arg) + else: + out_type = model_arg._get_output_type() + elif array is not None: + out_type = iu.determine_array_type(array_arg) + elif default_output_type is not None: + out_type = default_output_type + else: + out_type = None + + if out_type is not None: + set_api_output_type(out_type) + + res = func(*args, **kwargs) + + if skip: + return res + return cm.process_return(res) + + return inner + + +def api_return_array(input_arg=default, get_output_type=False): + return reflect(array=None if not get_output_type else input_arg) + + +def api_return_any(): + return reflect(array=None, skip=True) + + +def api_base_return_any(): + return reflect(reset=True) + + +def api_base_return_array(input_arg=default): + return reflect(array="self" if input_arg is None else input_arg) + + +def api_base_fit_transform(): + return reflect(reset=True) + + +# TODO: investigate and remove these +api_base_return_any_skipall = api_return_any() +api_base_return_array_skipall = reflect + + +@contextlib.contextmanager +def exit_internal_api(): + assert GlobalSettings().root_cm is not None + try: - return X.mem_type - except AttributeError: - pass - if hasattr(X, "__cuda_array_interface__"): - return MemoryType.device - if hasattr(X, "__array_interface__"): - return MemoryType.host - if isinstance(X, (cudf.DataFrame, cudf.Series)): - return MemoryType.device - if isinstance(X, (pd.DataFrame, pd.Series)): - return MemoryType.host - return None - - -def cuda_ptr(X): - """Returns a pointer to a backing device array, or None if not a device array""" - if (interface := getattr(X, "__cuda_array_interface__", None)) is not None: - return interface["data"][0] - return None + old_root_cm = GlobalSettings().root_cm + + GlobalSettings().root_cm = None + + # Set the global output type to the previous value to pretend we never + # entered the API + with using_output_type(old_root_cm.prev_output_type): + yield + + finally: + GlobalSettings().root_cm = old_root_cm diff --git a/python/cuml/cuml/kernel_ridge/kernel_ridge.py b/python/cuml/cuml/kernel_ridge/kernel_ridge.py index fd6bebd042..6e5d4c075f 100644 --- a/python/cuml/cuml/kernel_ridge/kernel_ridge.py +++ b/python/cuml/cuml/kernel_ridge/kernel_ridge.py @@ -12,7 +12,7 @@ from cuml.common import input_to_cuml_array from cuml.common.array_descriptor import CumlArrayDescriptor from cuml.common.doc_utils import generate_docstring -from cuml.internals.api_decorators import api_base_return_array +from cuml.internals import api_base_return_array from cuml.internals.array import CumlArray from cuml.internals.base import Base from cuml.internals.interop import ( diff --git a/python/cuml/cuml/linear_model/linear_regression.pyx b/python/cuml/cuml/linear_model/linear_regression.pyx index cd583c6226..d8965f68c3 100644 --- a/python/cuml/cuml/linear_model/linear_regression.pyx +++ b/python/cuml/cuml/linear_model/linear_regression.pyx @@ -12,7 +12,7 @@ from pylibraft.common.handle import Handle from cuml.common import input_to_cuml_array from cuml.common.array_descriptor import CumlArrayDescriptor from cuml.common.doc_utils import generate_docstring -from cuml.internals.array import CumlArray +from cuml.internals.array import CumlArray, cuda_ptr from cuml.internals.base import Base from cuml.internals.interop import ( InteropMixin, @@ -20,7 +20,6 @@ from cuml.internals.interop import ( to_cpu, to_gpu, ) -from cuml.internals.memory_utils import cuda_ptr from cuml.internals.mixins import FMajorInputTagMixin, RegressorMixin from cuml.linear_model.base import ( LinearPredictMixin, diff --git a/python/cuml/cuml/linear_model/ridge.pyx b/python/cuml/cuml/linear_model/ridge.pyx index 70ad41431f..dfdd9f30dc 100644 --- a/python/cuml/cuml/linear_model/ridge.pyx +++ b/python/cuml/cuml/linear_model/ridge.pyx @@ -7,7 +7,7 @@ import numpy as np from cuml.common.array_descriptor import CumlArrayDescriptor from cuml.common.doc_utils import generate_docstring -from cuml.internals.array import CumlArray +from cuml.internals.array import CumlArray, cuda_ptr from cuml.internals.base import Base from cuml.internals.input_utils import input_to_cuml_array from cuml.internals.interop import ( @@ -16,7 +16,6 @@ from cuml.internals.interop import ( to_cpu, to_gpu, ) -from cuml.internals.memory_utils import cuda_ptr from cuml.internals.mixins import FMajorInputTagMixin, RegressorMixin from cuml.linear_model.base import ( LinearPredictMixin, diff --git a/python/cuml/cuml/neighbors/nearest_neighbors.pyx b/python/cuml/cuml/neighbors/nearest_neighbors.pyx index b795120e15..f5f3e3e147 100644 --- a/python/cuml/cuml/neighbors/nearest_neighbors.pyx +++ b/python/cuml/cuml/neighbors/nearest_neighbors.pyx @@ -19,8 +19,8 @@ from cuml.internals.array_sparse import SparseCumlArray from cuml.internals.base import Base from cuml.internals.input_utils import input_to_cuml_array from cuml.internals.interop import InteropMixin, UnsupportedOnGPU, to_gpu -from cuml.internals.memory_utils import using_output_type from cuml.internals.mixins import CMajorInputTagMixin, SparseInputTagMixin +from cuml.internals.outputs import using_output_type from libc.stdint cimport int64_t, uint32_t, uintptr_t from libcpp cimport bool diff --git a/python/cuml/tests/internals/test_internal_utils.py b/python/cuml/tests/internals/test_internal_utils.py deleted file mode 100644 index c4fac48e2c..0000000000 --- a/python/cuml/tests/internals/test_internal_utils.py +++ /dev/null @@ -1,69 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 -# - -import numpy as np - -from cuml.internals.api_decorators import _get_value -from cuml.internals.input_utils import is_array_like - - -def test_is_array_like_with_lists(): - """Test is_array_like function with list/tuple inputs.""" - # Test lists and tuples are accepted when accept_lists=True - assert is_array_like([1, 2, 3], accept_lists=True) - assert is_array_like((1, 2, 3), accept_lists=True) - - # Test lists and tuples are rejected when accept_lists=False - assert not is_array_like([1, 2, 3], accept_lists=False) - assert not is_array_like((1, 2, 3), accept_lists=False) - - # Test numpy arrays are always accepted - assert is_array_like(np.array([1, 2, 3]), accept_lists=True) - assert is_array_like(np.array([1, 2, 3]), accept_lists=False) - - -def test_get_value_with_lists(): - """Test _get_value function with list/tuple inputs.""" - # Test list input is converted to numpy array - value = _get_value( - [], {"test": [1, 2, 3]}, "test", 0, None, accept_lists=True - ) - assert isinstance(value, np.ndarray) - np.testing.assert_array_equal(value, np.array([1, 2, 3])) - - # Test tuple input is converted to numpy array - value = _get_value( - [], {"test": (1, 2, 3)}, "test", 0, None, accept_lists=True - ) - assert isinstance(value, np.ndarray) - np.testing.assert_array_equal(value, np.array([1, 2, 3])) - - # Test non-list/tuple inputs are not converted - value = _get_value( - [], {"test": "string"}, "test", 0, None, accept_lists=True - ) - assert isinstance(value, str) - assert value == "string" - - # Test list input is not converted - value = _get_value( - [], {"test": [1, 2, 3]}, "test", 0, None, accept_lists=False - ) - assert isinstance(value, list) - assert value == [1, 2, 3] - - # Test tuple input is not converted - value = _get_value( - [], {"test": (1, 2, 3)}, "test", 0, None, accept_lists=False - ) - assert isinstance(value, tuple) - assert value == (1, 2, 3) - - # Test non-list/tuple inputs are not converted - value = _get_value( - [], {"test": "string"}, "test", 0, None, accept_lists=False - ) - assert isinstance(value, str) - assert value == "string" diff --git a/python/cuml/tests/test_array.py b/python/cuml/tests/test_array.py index fa5261118e..41677479ce 100644 --- a/python/cuml/tests/test_array.py +++ b/python/cuml/tests/test_array.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # +import functools import gc import operator import pickle @@ -24,10 +25,6 @@ array_to_memory_order, ) from cuml.internals.mem_type import MemoryType -from cuml.internals.memory_utils import ( - _get_size_from_shape, - determine_array_memtype, -) from cuml.testing.strategies import ( UNSUPPORTED_CUDF_DTYPES, create_cuml_array_input, @@ -70,6 +67,22 @@ } +def determine_array_memtype(X): + try: + return X.mem_type + except AttributeError: + pass + if hasattr(X, "__cuda_array_interface__"): + return MemoryType.device + if hasattr(X, "__array_interface__"): + return MemoryType.host + if isinstance(X, (cudf.DataFrame, cudf.Series)): + return MemoryType.device + if isinstance(X, (pd.DataFrame, pd.Series)): + return MemoryType.host + return None + + def _multidimensional(shape): return len(squeezed_shape(normalized_shape(shape))) > 1 @@ -160,7 +173,12 @@ def test_array_init(input_type, dtype, shape, order, force_gc): @settings(deadline=None) def test_array_init_from_bytes(data_type, dtype, shape, order, mem_type): dtype = np.dtype(dtype) - values = bytes(_get_size_from_shape(shape, dtype)[0]) + itemsize = dtype.itemsize + if isinstance(shape, int): + size = itemsize * shape + elif isinstance(shape, tuple): + size = functools.reduce(operator.mul, shape) + values = bytes(size) # Convert to data_type to be tested if needed. if data_type is not bytes: diff --git a/python/cuml/tests/test_cuml_descr_decor.py b/python/cuml/tests/test_cuml_descr_decor.py index 3e9446bc1c..3ab820554f 100644 --- a/python/cuml/tests/test_cuml_descr_decor.py +++ b/python/cuml/tests/test_cuml_descr_decor.py @@ -286,7 +286,7 @@ def test_return_array(input_arg: str, get_output_type: bool): def test_func(X, y): if not get_output_type: - cuml.internals.set_api_output_type(inner_type) + cuml.internals.outputs.set_api_output_type(inner_type) return X expected_to_fail = input_arg == "bad" and get_output_type diff --git a/python/cuml/tests/test_input_utils.py b/python/cuml/tests/test_input_utils.py index 65c7d74fee..4e4b044556 100644 --- a/python/cuml/tests/test_input_utils.py +++ b/python/cuml/tests/test_input_utils.py @@ -13,7 +13,11 @@ from pandas import Series as pdSeries from cuml.common import CumlArray, input_to_cuml_array, input_to_host_array -from cuml.internals.input_utils import convert_dtype, input_to_cupy_array +from cuml.internals.input_utils import ( + convert_dtype, + input_to_cupy_array, + is_array_like, +) from cuml.manifold import umap ############################################################################### @@ -457,3 +461,18 @@ def test_numpy_output(): # Check that this is a cudf.pandas wrapped array assert hasattr(X, "_fsproxy_fast_type") assert isinstance(reducer.fit_transform(X), np.ndarray) + + +def test_is_array_like_with_lists(): + """Test is_array_like function with list/tuple inputs.""" + # Test lists and tuples are accepted when accept_lists=True + assert is_array_like([1, 2, 3], accept_lists=True) + assert is_array_like((1, 2, 3), accept_lists=True) + + # Test lists and tuples are rejected when accept_lists=False + assert not is_array_like([1, 2, 3], accept_lists=False) + assert not is_array_like((1, 2, 3), accept_lists=False) + + # Test numpy arrays are always accepted + assert is_array_like(np.array([1, 2, 3]), accept_lists=True) + assert is_array_like(np.array([1, 2, 3]), accept_lists=False) From ddfb6309b4ce8c156e5d769026f6f6f2b7357065 Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Tue, 25 Nov 2025 10:52:15 -0600 Subject: [PATCH 16/28] Simplify and unify `output_type` validation --- python/cuml/cuml/internals/base.py | 48 ++++++---------------- python/cuml/cuml/internals/outputs.py | 57 +++++++++++++++------------ 2 files changed, 43 insertions(+), 62 deletions(-) diff --git a/python/cuml/cuml/internals/base.py b/python/cuml/cuml/internals/base.py index 8956357642..e1d08f6955 100644 --- a/python/cuml/cuml/internals/base.py +++ b/python/cuml/cuml/internals/base.py @@ -2,7 +2,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - import inspect import os @@ -16,10 +15,7 @@ import cuml.internals.nvtx as nvtx from cuml.internals.input_utils import determine_array_type from cuml.internals.mixins import TagsMixin -from cuml.internals.outputs import ( - INTERNAL_VALID_OUTPUT_TYPES, - VALID_OUTPUT_TYPES, -) +from cuml.internals.outputs import check_output_type class Base(TagsMixin, metaclass=cuml.internals.BaseMetaClass): @@ -159,11 +155,16 @@ def __init__( pylibraft.common.handle.Handle() if handle is None else handle ) self.verbose = verbose - self.output_type = _check_output_type_str( - cuml.global_settings.output_type - if output_type is None - else output_type - ) + if output_type is None: + output_type = cuml.global_settings.output_type or "input" + if output_type == "mirror": + raise ValueError( + "Cannot pass output_type='mirror' to Base.__init__(). Did you forget " + "to pass `output_type=self.output_type` to a child estimator? " + ) + else: + output_type = check_output_type(output_type) + self.output_type = output_type self._input_type = None nvtx_benchmark = os.getenv("NVTX_BENCHMARK") @@ -313,30 +314,3 @@ def set_nvtx_annotations(self): func = getattr(self, func_name) func = nvtx.annotate(message=msg, domain="cuml_python")(func) setattr(self, func_name, func) - - -# Internal, non class owned helper functions -def _check_output_type_str(output_str): - if output_str is None: - return "input" - - assert output_str != "mirror", ( - "Cannot pass output_type='mirror' in Base.__init__(). Did you forget " - "to pass `output_type=self.output_type` to a child estimator? " - "Currently `cuml.global_settings.output_type==`{}`" - ).format(cuml.global_settings.output_type) - - if isinstance(output_str, str): - output_type = output_str.lower() - # Check for valid output types + "input" - if output_type in INTERNAL_VALID_OUTPUT_TYPES: - # Return the original version if nothing has changed, otherwise - # return the lowered. This is to try and keep references the same - # to support sklearn.base.clone() where possible - return output_str if output_type == output_str else output_type - - valid_output_types_str = ", ".join([f"'{x}'" for x in VALID_OUTPUT_TYPES]) - raise ValueError( - f"output_type must be one of {valid_output_types_str}" - f" Got: {output_str}" - ) diff --git a/python/cuml/cuml/internals/outputs.py b/python/cuml/cuml/internals/outputs.py index e51a286109..1ea825cbec 100644 --- a/python/cuml/cuml/internals/outputs.py +++ b/python/cuml/cuml/internals/outputs.py @@ -17,25 +17,48 @@ from cuml.internals.constants import CUML_WRAPPED_FLAG from cuml.internals.global_settings import GlobalSettings +__all__ = ( + "check_output_type", + "set_global_output_type", + "using_output_type", + "reflect", + "exit_internal_api", +) + + default = type( "default", (), dict.fromkeys(["__repr__", "__reduce__"], lambda s: "default"), )() -VALID_OUTPUT_TYPES = ( - "array", +OUTPUT_TYPES = ( + "input", + "numpy", + "cupy", + "cudf", + "pandas", "numba", + "array", "dataframe", "series", "df_obj", - "cupy", - "numpy", - "cudf", - "pandas", ) -INTERNAL_VALID_OUTPUT_TYPES = ("input", *VALID_OUTPUT_TYPES) + +def check_output_type(output_type: str) -> str: + # normalize as lower, keeping original str reference to appease the sklearn + # standard estimator checks as much as possible. + if output_type != (temp := output_type.lower()): + output_type = temp + # Check for allowed types. Allow 'cuml' to support internal estimators + if output_type != "cuml" and output_type not in OUTPUT_TYPES: + valid_output_types = ", ".join(map(repr, OUTPUT_TYPES)) + raise ValueError( + f"output_type must be one of {valid_output_types}" + f" or None. Got: {output_type}" + ) + return output_type def set_global_output_type(output_type): @@ -108,23 +131,8 @@ def set_global_output_type(output_type): CPU memory. """ - if isinstance(output_type, str): - output_type = output_type.lower() - - # Check for allowed types. Allow 'cuml' to support internal estimators - if ( - output_type is not None - and output_type != "cuml" - and output_type not in INTERNAL_VALID_OUTPUT_TYPES - ): - valid_output_types_str = ", ".join( - [f"'{x}'" for x in VALID_OUTPUT_TYPES] - ) - raise ValueError( - f"output_type must be one of {valid_output_types_str}" - f" or None. Got: {output_type}" - ) - + if output_type is not None: + output_type = check_output_type(output_type) GlobalSettings().output_type = output_type @@ -168,7 +176,6 @@ class using_output_type: Examples -------- - >>> import cuml >>> import cupy as cp >>> ary = [[1.0, 4.0, 4.0], [2.0, 2.0, 2.0], [5.0, 1.0, 1.0]] From 4fc664530b2f798b5383e279b4b77a58d67d5943 Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Tue, 25 Nov 2025 12:36:31 -0600 Subject: [PATCH 17/28] Remove `root_cm`, `api_context_managers` Further simplifications - you can actually follow how output type is inferred and propagated now! Now backdoor state. --- python/cuml/cuml/datasets/arima.pyx | 2 +- python/cuml/cuml/datasets/blobs.py | 2 +- python/cuml/cuml/datasets/classification.py | 2 +- python/cuml/cuml/datasets/regression.pyx | 2 +- .../cuml/internals/api_context_managers.py | 207 ------------------ python/cuml/cuml/internals/base.py | 7 +- python/cuml/cuml/internals/global_settings.py | 2 +- python/cuml/cuml/internals/outputs.py | 151 ++++++++----- .../tests/dask/test_dask_global_settings.py | 31 +-- python/cuml/tests/test_array.py | 2 +- python/cuml/tests/test_cuml_descr_decor.py | 35 +-- 11 files changed, 121 insertions(+), 322 deletions(-) delete mode 100644 python/cuml/cuml/internals/api_context_managers.py diff --git a/python/cuml/cuml/datasets/arima.pyx b/python/cuml/cuml/datasets/arima.pyx index 561f89f3ac..591a5bb691 100644 --- a/python/cuml/cuml/datasets/arima.pyx +++ b/python/cuml/cuml/datasets/arima.pyx @@ -54,7 +54,7 @@ inp_to_dtype = { } -@cuml.internals.reflect(array=None, default_output_type="cupy") +@cuml.internals.reflect(array=None) def make_arima(batch_size=1000, n_obs=100, order=(1, 1, 1), seasonal_order=(0, 0, 0, 0), intercept=False, random_state=None, dtype='double', diff --git a/python/cuml/cuml/datasets/blobs.py b/python/cuml/cuml/datasets/blobs.py index 9a3ee1fd72..e85df3ced4 100644 --- a/python/cuml/cuml/datasets/blobs.py +++ b/python/cuml/cuml/datasets/blobs.py @@ -71,7 +71,7 @@ def _get_centers(rs, centers, center_box, n_samples, n_features, dtype): @nvtx.annotate(message="datasets.make_blobs", domain="cuml_python") -@cuml.internals.reflect(array=None, default_output_type="cupy") +@cuml.internals.reflect(array=None) def make_blobs( n_samples=100, n_features=2, diff --git a/python/cuml/cuml/datasets/classification.py b/python/cuml/cuml/datasets/classification.py index d51e15ce26..8ad3f9bc24 100644 --- a/python/cuml/cuml/datasets/classification.py +++ b/python/cuml/cuml/datasets/classification.py @@ -33,7 +33,7 @@ def _generate_hypercube(samples, dimensions, random_state): @nvtx.annotate(message="datasets.make_classification", domain="cuml_python") -@cuml.internals.reflect(array=None, default_output_type="cupy") +@cuml.internals.reflect(array=None) def make_classification( n_samples=100, n_features=20, diff --git a/python/cuml/cuml/datasets/regression.pyx b/python/cuml/cuml/datasets/regression.pyx index 09f9b0ea94..f4924465d3 100644 --- a/python/cuml/cuml/datasets/regression.pyx +++ b/python/cuml/cuml/datasets/regression.pyx @@ -63,7 +63,7 @@ inp_to_dtype = { @nvtx.annotate(message="datasets.make_regression", domain="cuml_python") -@cuml.internals.reflect(array=None, default_output_type="cupy") +@cuml.internals.reflect(array=None) def make_regression( n_samples=100, n_features=2, diff --git a/python/cuml/cuml/internals/api_context_managers.py b/python/cuml/cuml/internals/api_context_managers.py deleted file mode 100644 index 8d4ffe1508..0000000000 --- a/python/cuml/cuml/internals/api_context_managers.py +++ /dev/null @@ -1,207 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 -# -import contextlib -from collections import deque - -import cuml.internals.input_utils as iu -from cuml.internals.array_sparse import SparseCumlArray -from cuml.internals.global_settings import GlobalSettings - - -@contextlib.contextmanager -def _using_mirror_output_type(): - """ - Sets global_settings.output_type to "mirror" for internal API - handling. We need a separate function since `cuml.using_output_type()` - doesn't accept "mirror" - - Yields - ------- - string - Returns the previous value in global_settings.output_type - """ - prev_output_type = GlobalSettings().output_type - try: - GlobalSettings().output_type = "mirror" - yield prev_output_type - finally: - GlobalSettings().output_type = prev_output_type - - -def set_api_output_type(output_type: str): - assert GlobalSettings().root_cm is not None - - # Quick exit - if isinstance(output_type, str): - GlobalSettings().root_cm.output_type = output_type - return - - # Try to convert any array objects to their type - array_type = iu.determine_array_type(output_type) - - # Ensure that this is an array-like object - assert output_type is None or array_type is not None - - GlobalSettings().root_cm.output_type = array_type - - -class InternalAPIContext(contextlib.ExitStack): - def __init__(self): - super().__init__() - - def cleanup(): - GlobalSettings().root_cm = None - - self.callback(cleanup) - - self.prev_output_type = self.enter_context(_using_mirror_output_type()) - - # Set the output type to the prev_output_type. If "input", set to None - # to allow inner functions to specify the input - self.output_type = ( - None if self.prev_output_type == "input" else self.prev_output_type - ) - - self._count = 0 - - self.call_stack = {} - - GlobalSettings().root_cm = self - - def pop_all(self): - """Preserve the context stack by transferring it to a new instance.""" - new_stack = contextlib.ExitStack() - new_stack._exit_callbacks = self._exit_callbacks - self._exit_callbacks = deque() - return new_stack - - def __enter__(self) -> int: - self._count += 1 - - return self._count - - def __exit__(self, *exc_details): - self._count -= 1 - - return - - @contextlib.contextmanager - def push_output_types(self): - try: - old_output_type = self.output_type - self.output_type = None - yield - finally: - self.output_type = ( - old_output_type - if old_output_type is not None - else self.output_type - ) - - -def get_internal_context() -> InternalAPIContext: - """Return the current "root" context manager used to control output type - for external API calls and minimize unnecessary internal output - conversions""" - - if GlobalSettings().root_cm is None: - GlobalSettings().root_cm = InternalAPIContext() - - return GlobalSettings().root_cm - - -class InternalAPIContextBase(contextlib.ExitStack): - def __init__(self, base=None, process_return=None): - super().__init__() - - self._base = base - self._process_return = process_return - - self.root_cm = get_internal_context() - - self.is_root = False - - self._should_set_output_type_from_base = ( - self._base is not None - and self.root_cm.prev_output_type in (None, "input") - ) - - def set_output_type_from_base(self, root_cm): - output_type = root_cm.output_type - - # Check if output_type is None, can happen if no output type has - # been set by estimator - if output_type is None: - output_type = self._base.output_type - - if output_type == "input": - output_type = self._base._input_type - - if output_type != root_cm.output_type: - set_api_output_type(output_type) - - assert output_type != "mirror" - - def __enter__(self): - # Enter the root context to know if we are the root cm - self.is_root = self.enter_context(self.root_cm) == 1 - - # If we are the first, push any callbacks from the root into this CM - # If we are not the first, this will have no effect - self.push(self.root_cm.pop_all()) - - if self._process_return: - self.enter_context(self.root_cm.push_output_types()) - if self._should_set_output_type_from_base: - self.callback(self.set_output_type_from_base, self.root_cm) - - # Only convert output: - # - when returning results from a root api call - # - when the output type is explicitly set - self._convert_output = ( - self.is_root or GlobalSettings().output_type != "mirror" - ) - - return super().__enter__() - - def process_return(self, res): - """Traverse a result, converting it to the proper output type""" - if isinstance(res, tuple): - return tuple(self.process_return(i) for i in res) - elif isinstance(res, list): - return [self.process_return(i) for i in res] - elif isinstance(res, dict): - return {k: self.process_return(v) for k, v in res.items()} - - # Get the output type - arr_type, is_sparse = iu.determine_array_type_full(res) - - if arr_type is None: - # Not an array, just return - return res - - # If we are a supported array and not already cuml, convert to cuml - if arr_type != "cuml": - if is_sparse: - res = SparseCumlArray(res, convert_index=False) - else: - res = iu.input_to_cuml_array(res, order="K").array - - if not self._convert_output: - # Return CumlArray/SparseCumlArray directly - return res - - output_type = GlobalSettings().output_type - - if output_type in (None, "mirror", "input"): - output_type = self.root_cm.output_type - - assert ( - output_type is not None - and output_type != "mirror" - and output_type != "input" - ), ("Invalid root_cm.output_type: '{}'.").format(output_type) - - return res.to_output(output_type=output_type) diff --git a/python/cuml/cuml/internals/base.py b/python/cuml/cuml/internals/base.py index e1d08f6955..75d50249aa 100644 --- a/python/cuml/cuml/internals/base.py +++ b/python/cuml/cuml/internals/base.py @@ -246,15 +246,14 @@ def _get_output_type(self, inp=None): Returns the appropriate output type depending on the type of the input, class output type and global output type. """ - # Default to the global type output_type = cuml.global_settings.output_type - # If its None, default to our type - if output_type is None or output_type == "mirror": + # If not set to an explicit value, use the estimator's setting + if output_type in (None, "input", "mirror"): output_type = self.output_type - # If we are input, get the type from the input (if available) + # If input, get the type from the input (if available) if output_type == "input": if inp is None: # No input value provided, use the estimator input type diff --git a/python/cuml/cuml/internals/global_settings.py b/python/cuml/cuml/internals/global_settings.py index 21872e69ca..237cbfdaa9 100644 --- a/python/cuml/cuml/internals/global_settings.py +++ b/python/cuml/cuml/internals/global_settings.py @@ -16,8 +16,8 @@ class _GlobalSettingsData(threading.local): # pylint: disable=R0903 def __init__(self): super().__init__() self.shared_state = { - "root_cm": None, "_output_type": None, + "_external_output_type": False, "_fil_device_type": DeviceType.device, "_fil_memory_type": MemoryType.device, } diff --git a/python/cuml/cuml/internals/outputs.py b/python/cuml/cuml/internals/outputs.py index 1ea825cbec..6043b8e90c 100644 --- a/python/cuml/cuml/internals/outputs.py +++ b/python/cuml/cuml/internals/outputs.py @@ -10,10 +10,7 @@ # TODO: Try to resolve circular import that makes this necessary: from cuml.internals import input_utils as iu -from cuml.internals.api_context_managers import ( - InternalAPIContextBase, - set_api_output_type, -) +from cuml.internals.array_sparse import SparseCumlArray from cuml.internals.constants import CUML_WRAPPED_FLAG from cuml.internals.global_settings import GlobalSettings @@ -218,6 +215,49 @@ def __exit__(self, *_): GlobalSettings().output_type = self.prev_output_type +@contextlib.contextmanager +def enter_internal_api(): + """Enter an internal API context. + + Returns ``True`` if this is a new internal context, or ``False`` + if the code was already running within an internal context.""" + gs = GlobalSettings() + if gs._external_output_type is False: + # External, this is a new context + gs._external_output_type = gs.output_type + gs.output_type = "mirror" + try: + yield True + finally: + gs.output_type = gs._external_output_type + gs._external_output_type = False + else: + # Already internal, just yield + yield False + + +@contextlib.contextmanager +def exit_internal_api(): + """Exit an internal API context. + + Code run in this context will run under the original + configuration before an internal context was entered""" + gs = GlobalSettings() + if gs._external_output_type is False: + # Already external, nothing to do + yield + else: + orig_external_output_type = gs._external_output_type + orig_output_type = gs.output_type + gs.output_type = orig_external_output_type + gs._external_output_type = False + try: + yield + finally: + gs._external_output_type = orig_external_output_type + gs.output_type = orig_output_type + + def _get_param(sig, name_or_index): if isinstance(name_or_index, str): param = sig.parameters[name_or_index] @@ -233,6 +273,36 @@ def _get_param(sig, name_or_index): return param.name +def coerce_arrays(res, output_type): + """Traverse a result, converting it to the proper output type""" + if isinstance(res, tuple): + return tuple(coerce_arrays(i, output_type) for i in res) + elif isinstance(res, list): + return [coerce_arrays(i, output_type) for i in res] + elif isinstance(res, dict): + return {k: coerce_arrays(v, output_type) for k, v in res.items()} + + # Get the output type + arr_type, is_sparse = iu.determine_array_type_full(res) + + if arr_type is None: + # Not an array, just return + return res + + # If we are a supported array and not already cuml, convert to cuml + if arr_type != "cuml": + if is_sparse: + res = SparseCumlArray(res, convert_index=False) + else: + res = iu.input_to_cuml_array(res, order="K").array + + if output_type == "cuml": + # Return CumlArray/SparseCumlArray directly + return res + + return res.to_output(output_type=output_type) + + def reflect( func=None, *, @@ -240,7 +310,6 @@ def reflect( model=default, reset=False, skip=False, - default_output_type=None, ): """Mark a function or method as participating in the reflection system. @@ -267,9 +336,6 @@ def reflect( skip : bool, default=False Set to True to skip output processing for a method. This is mostly useful if output processing will be handled manually. - default_output_type : str or None, default=None - The default output type to use for a method when no output type - has been set externally. """ # Local to avoid circular imports import cuml.accel @@ -281,7 +347,6 @@ def reflect( array=array, reset=reset, skip=skip, - default_output_type=default_output_type, ) # TODO: remove this once auto-decorating is ripped out @@ -306,6 +371,11 @@ def reflect( if array is not None: array = _get_param(sig, array) + if reset and (model is None or array is None): + raise ValueError( + "`reset=True` is not valid with `array=None` or `model=None`" + ) + @functools.wraps(func) def inner(*args, **kwargs): # Accept list/tuple inputs when accelerator is active @@ -320,36 +390,37 @@ def inner(*args, **kwargs): if accept_lists and isinstance(array_arg, (list, tuple)): array_arg = np.asarray(array_arg) - if reset and model_arg is None: - raise ValueError("`reset=True` is only valid on estimator methods") - - with InternalAPIContextBase( - base=model_arg, process_return=not skip - ) as cm: + with enter_internal_api() as was_external: if reset: model_arg._set_output_type(array_arg) model_arg._set_n_features_in(array_arg) - if model is not None: - if array is not None: - out_type = model_arg._get_output_type(array_arg) + res = func(*args, **kwargs) + + if not skip: + gs = GlobalSettings() + if was_external or gs.output_type != "mirror": + # We're returning to the user, infer the expected output type + if model is not None: + if array is not None: + output_type = model_arg._get_output_type(array_arg) + else: + output_type = model_arg._get_output_type() else: - out_type = model_arg._get_output_type() - elif array is not None: - out_type = iu.determine_array_type(array_arg) - elif default_output_type is not None: - out_type = default_output_type + output_type = gs.output_type + if output_type in ("input", None): + if array is not None: + output_type = iu.determine_array_type(array_arg) + if output_type in ("input", None): + # Nothing to infer from and no explicit type set, default to cupy + output_type = "cupy" else: - out_type = None + # We're internal, return as cuml + output_type = "cuml" - if out_type is not None: - set_api_output_type(out_type) - - res = func(*args, **kwargs) + res = coerce_arrays(res, output_type) - if skip: - return res - return cm.process_return(res) + return res return inner @@ -377,21 +448,3 @@ def api_base_fit_transform(): # TODO: investigate and remove these api_base_return_any_skipall = api_return_any() api_base_return_array_skipall = reflect - - -@contextlib.contextmanager -def exit_internal_api(): - assert GlobalSettings().root_cm is not None - - try: - old_root_cm = GlobalSettings().root_cm - - GlobalSettings().root_cm = None - - # Set the global output type to the previous value to pretend we never - # entered the API - with using_output_type(old_root_cm.prev_output_type): - yield - - finally: - GlobalSettings().root_cm = old_root_cm diff --git a/python/cuml/tests/dask/test_dask_global_settings.py b/python/cuml/tests/dask/test_dask_global_settings.py index 865d524ad1..2e300ad01c 100644 --- a/python/cuml/tests/dask/test_dask_global_settings.py +++ b/python/cuml/tests/dask/test_dask_global_settings.py @@ -11,7 +11,6 @@ import cuml from cuml import set_global_output_type, using_output_type -from cuml.internals.api_context_managers import _using_mirror_output_type from cuml.internals.global_settings import ( GlobalSettings, _global_settings_data, @@ -62,31 +61,6 @@ def check_correct_type(index): assert (delayed(all)(results)).compute() -def test_using_mirror_output_type(): - """Ensure that _using_mirror_output_type is thread-safe""" - - def check_correct_type(index): - # Force a race condition - if index == 0: - sleep(0.1) - if index % 2 == 0: - with _using_mirror_output_type(): - sleep(0.5) - return cuml.global_settings.output_type == "mirror" - else: - output_type = test_output_types_str[index] - with using_output_type(output_type): - sleep(0.5) - return cuml.global_settings.output_type == output_type - - results = [ - delayed(check_correct_type)(index) - for index in range(len(test_output_types_str)) - ] - - assert (delayed(all)(results)).compute() - - def test_global_settings_data(): """Ensure that GlobalSettingsData objects are properly initialized per-thread""" @@ -102,7 +76,10 @@ def check_initialized(index): sleep(0.5) return ( test_global_settings_data_obj.shared_state["_output_type"] is None - and test_global_settings_data_obj.shared_state["root_cm"] is None + and test_global_settings_data_obj.shared_state[ + "_external_output_type" + ] + is False and _global_settings_data.testing_index == index ) diff --git a/python/cuml/tests/test_array.py b/python/cuml/tests/test_array.py index 41677479ce..6ca3469e90 100644 --- a/python/cuml/tests/test_array.py +++ b/python/cuml/tests/test_array.py @@ -177,7 +177,7 @@ def test_array_init_from_bytes(data_type, dtype, shape, order, mem_type): if isinstance(shape, int): size = itemsize * shape elif isinstance(shape, tuple): - size = functools.reduce(operator.mul, shape) + size = itemsize * functools.reduce(operator.mul, shape) values = bytes(size) # Convert to data_type to be tested if needed. diff --git a/python/cuml/tests/test_cuml_descr_decor.py b/python/cuml/tests/test_cuml_descr_decor.py index 3ab820554f..999466334a 100644 --- a/python/cuml/tests/test_cuml_descr_decor.py +++ b/python/cuml/tests/test_cuml_descr_decor.py @@ -266,52 +266,29 @@ def test_auto_predict(input_type, base_output_type, global_output_type): assert_array_identical(X_in, X_out) -@pytest.mark.parametrize("input_arg", ["X", "y", "bad", 0]) -@pytest.mark.parametrize("get_output_type", [True, False]) -def test_return_array(input_arg: str, get_output_type: bool): +@pytest.mark.parametrize("input_arg", ["X", "y", 0]) +def test_return_array(input_arg: str): """ Test autowrapping on predict that will set target_type """ - input_type_X = "numpy" input_dtype_X = np.float64 input_type_Y = "cupy" input_dtype_Y = np.int32 - inner_type = "numba" - X_in = create_input(input_type_X, input_dtype_X, (10, 10), "F") Y_in = create_input(input_type_Y, input_dtype_Y, (10, 10), "F") + @cuml.internals.api_return_array(input_arg=input_arg, get_output_type=True) def test_func(X, y): - if not get_output_type: - cuml.internals.outputs.set_api_output_type(inner_type) return X - expected_to_fail = input_arg == "bad" and get_output_type - - try: - test_func = cuml.internals.api_return_array( - input_arg=input_arg, - get_output_type=get_output_type, - )(test_func) - except KeyError: - assert expected_to_fail - return - else: - assert not expected_to_fail - X_out = test_func(X=X_in, y=Y_in) - target_type = None - - if not get_output_type: - target_type = inner_type + if input_arg == "y": + target_type = input_type_Y else: - if input_arg == "y": - target_type = input_type_Y - else: - target_type = input_type_X + target_type = input_type_X assert determine_array_type(X_out) == target_type From 1353aeb7fe78307d4aaf2b82a45311cd47963119 Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Tue, 25 Nov 2025 14:51:20 -0600 Subject: [PATCH 18/28] Simplify `skip=True` case --- python/cuml/cuml/internals/outputs.py | 30 ++++++++++++++++++++------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/python/cuml/cuml/internals/outputs.py b/python/cuml/cuml/internals/outputs.py index 6043b8e90c..1873e0bb4c 100644 --- a/python/cuml/cuml/internals/outputs.py +++ b/python/cuml/cuml/internals/outputs.py @@ -259,6 +259,10 @@ def exit_internal_api(): def _get_param(sig, name_or_index): + """Get an `inspect.Parameter` instance by name or index from a + signature, and validates it's not variadic. + + Used for normalizing `array`/`model` args to `reflect`.""" if isinstance(name_or_index, str): param = sig.parameters[name_or_index] else: @@ -355,19 +359,28 @@ def reflect( sig = inspect.signature(func, follow_wrapped=True) has_self = "self" in sig.parameters + # Normalize model to str | None if model is default: - model = "self" if has_self else None + if skip and not reset: + # We're skipping output processing and not resetting an estimator, + # there's no need to touch input parameters at all + model = None + else: + model = "self" if has_self else None if model is not None: model = _get_param(sig, model) + # Normalize array to str | None if array is default: - if model is not None and list(sig.parameters).index(model) == 0: - array = 1 - else: - array = 0 - if len(sig.parameters) <= array: - # Not enough parameters, no array-like param to infer from + if skip and not reset: array = None + else: + array = int( + model is not None and list(sig.parameters).index(model) == 0 + ) + if len(sig.parameters) <= array: + # Not enough parameters, no array-like param to infer from + array = None if array is not None: array = _get_param(sig, array) @@ -412,7 +425,8 @@ def inner(*args, **kwargs): if array is not None: output_type = iu.determine_array_type(array_arg) if output_type in ("input", None): - # Nothing to infer from and no explicit type set, default to cupy + # Nothing to infer from and no explicit type set, + # default to cupy output_type = "cupy" else: # We're internal, return as cuml From f3f968e050d58bd5246c28bbf8ff2d055cd633be Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Tue, 25 Nov 2025 15:03:09 -0600 Subject: [PATCH 19/28] Explicit decorators --- .../preprocessing/_column_transformer.py | 3 ++ .../sklearn/preprocessing/_data.py | 47 +++++++++++++++---- .../sklearn/preprocessing/_discretization.py | 17 +++---- .../preprocessing/_function_transformer.py | 6 ++- .../sklearn/preprocessing/_imputation.py | 6 +++ python/cuml/cuml/cluster/agglomerative.pyx | 3 ++ python/cuml/cuml/cluster/dbscan.pyx | 4 +- python/cuml/cuml/cluster/hdbscan/hdbscan.pyx | 13 +++-- python/cuml/cuml/cluster/kmeans.pyx | 7 +++ python/cuml/cuml/common/sparsefuncs.py | 5 -- .../cuml/decomposition/incremental_pca.py | 4 +- python/cuml/cuml/decomposition/pca.pyx | 5 +- python/cuml/cuml/decomposition/tsvd.pyx | 5 +- .../cuml/ensemble/randomforestclassifier.py | 5 +- .../cuml/ensemble/randomforestregressor.py | 4 ++ .../cuml/experimental/linear_model/lars.pyx | 21 +++------ python/cuml/cuml/feature_extraction/_tfidf.py | 6 +-- python/cuml/cuml/fil/fil.pyx | 9 ++-- python/cuml/cuml/internals/mixins.py | 6 +-- python/cuml/cuml/kernel_ridge/kernel_ridge.py | 5 +- python/cuml/cuml/linear_model/base.py | 4 +- python/cuml/cuml/linear_model/elastic_net.py | 2 + .../cuml/linear_model/linear_regression.pyx | 2 + .../cuml/linear_model/logistic_regression.py | 6 ++- .../cuml/linear_model/mbsgd_classifier.py | 3 +- .../cuml/cuml/linear_model/mbsgd_regressor.py | 2 + python/cuml/cuml/linear_model/ridge.pyx | 2 + .../cuml/cuml/manifold/spectral_embedding.pyx | 6 ++- python/cuml/cuml/manifold/t_sne.pyx | 5 +- python/cuml/cuml/manifold/umap/umap.pyx | 10 ++-- python/cuml/cuml/metrics/_classification.py | 3 -- python/cuml/cuml/metrics/_ranking.py | 3 +- .../metrics/cluster/adjusted_rand_index.pyx | 5 -- .../metrics/cluster/completeness_score.pyx | 6 --- python/cuml/cuml/metrics/cluster/entropy.pyx | 32 ++++--------- .../metrics/cluster/homogeneity_score.pyx | 6 --- .../metrics/cluster/mutual_info_score.pyx | 6 --- .../cuml/metrics/cluster/silhouette_score.pyx | 10 ++-- python/cuml/cuml/metrics/cluster/utils.py | 3 -- .../cuml/cuml/metrics/cluster/v_measure.pyx | 6 --- python/cuml/cuml/metrics/confusion_matrix.py | 3 -- python/cuml/cuml/metrics/hinge_loss.py | 3 -- python/cuml/cuml/metrics/kl_divergence.pyx | 3 -- .../cuml/cuml/metrics/pairwise_distances.pyx | 7 +-- python/cuml/cuml/metrics/pairwise_kernels.py | 2 +- python/cuml/cuml/metrics/regression.py | 7 --- python/cuml/cuml/metrics/trustworthiness.pyx | 5 -- python/cuml/cuml/metrics/utils.py | 2 - python/cuml/cuml/multiclass/multiclass.py | 7 ++- python/cuml/cuml/naive_bayes/naive_bayes.py | 12 +++++ python/cuml/cuml/neighbors/kernel_density.py | 5 ++ .../cuml/neighbors/kneighbors_classifier.pyx | 8 ++-- .../cuml/neighbors/kneighbors_regressor.pyx | 8 ++-- .../cuml/cuml/neighbors/nearest_neighbors.pyx | 9 ++-- python/cuml/cuml/preprocessing/label.py | 11 +++-- python/cuml/cuml/prims/label/classlabels.py | 13 +---- python/cuml/cuml/prims/stats/covariance.py | 3 -- .../random_projection/random_projection.py | 6 ++- python/cuml/cuml/solvers/cd.pyx | 3 ++ python/cuml/cuml/solvers/qn.pyx | 4 ++ python/cuml/cuml/solvers/sgd.pyx | 3 ++ python/cuml/cuml/svm/linear_svc.py | 6 ++- python/cuml/cuml/svm/linear_svr.py | 2 + python/cuml/cuml/svm/svc.py | 20 ++++---- python/cuml/cuml/svm/svm_base.pyx | 2 +- python/cuml/cuml/svm/svr.py | 3 ++ python/cuml/cuml/tsa/arima.pyx | 47 +++++++++---------- python/cuml/cuml/tsa/auto_arima.pyx | 34 ++++++-------- python/cuml/cuml/tsa/holtwinters.pyx | 18 +++---- python/cuml/cuml/tsa/seasonality.py | 6 +-- python/cuml/cuml/tsa/stationarity.pyx | 19 +++----- 71 files changed, 295 insertions(+), 289 deletions(-) diff --git a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_column_transformer.py b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_column_transformer.py index d907913081..573af00c02 100644 --- a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_column_transformer.py +++ b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_column_transformer.py @@ -833,6 +833,7 @@ def _fit_transform(self, X, y, func, fitted=False): else: raise + @cuml.internals.reflect def fit(self, X, y=None) -> "ColumnTransformer": """Fit all transformers using X. @@ -856,6 +857,7 @@ def fit(self, X, y=None) -> "ColumnTransformer": self.fit_transform(X, y=y) return self + @cuml.internals.reflect(reset=True) def fit_transform(self, X, y=None) -> SparseCumlArray: """Fit all transformers, transform the data and concatenate results. @@ -913,6 +915,7 @@ def fit_transform(self, X, y=None) -> SparseCumlArray: return self._hstack(list(Xs)) + @cuml.internals.reflect def transform(self, X) -> SparseCumlArray: """Transform X separately by each transformer, concatenate results. diff --git a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_data.py b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_data.py index b353b0e766..36f1b1eba2 100644 --- a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_data.py +++ b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_data.py @@ -42,10 +42,9 @@ ) from ....common.array_descriptor import CumlArrayDescriptor -from ....internals import api_return_array from ....internals.array import CumlArray from ....internals.array_sparse import SparseCumlArray -from ....internals.outputs import using_output_type +from ....internals.outputs import using_output_type, reflect from ....thirdparty_adapters import check_array from ....thirdparty_adapters.sparsefuncs_fast import ( csr_polynomial_expansion, @@ -102,7 +101,7 @@ def _handle_zeros_in_scale(scale, copy=True): return scale -@api_return_array(get_output_type=True) +@reflect def scale(X, *, axis=0, with_mean=True, with_std=True, copy=True): """Standardize a dataset along any axis @@ -330,6 +329,7 @@ def _get_param_names(cls): "copy" ] + @reflect(reset=True) def fit(self, X, y=None) -> "MinMaxScaler": """Compute the minimum and maximum to be used for later scaling. @@ -352,6 +352,7 @@ def fit(self, X, y=None) -> "MinMaxScaler": self._reset() return self.partial_fit(X, y) + @reflect(reset=True) def partial_fit(self, X, y=None) -> "MinMaxScaler": """Online computation of min and max on X for later scaling. @@ -402,6 +403,7 @@ def partial_fit(self, X, y=None) -> "MinMaxScaler": self.data_range_ = data_range return self + @reflect def transform(self, X) -> CumlArray: """Scale features of X according to feature_range. @@ -425,6 +427,7 @@ def transform(self, X) -> CumlArray: return X + @reflect def inverse_transform(self, X) -> CumlArray: """Undo the scaling of X according to feature_range. @@ -448,7 +451,7 @@ def inverse_transform(self, X) -> CumlArray: return X -@api_return_array(get_output_type=True) +@reflect def minmax_scale(X, feature_range=(0, 1), *, axis=0, copy=True): """Transform features by scaling each feature to a given range. @@ -655,6 +658,7 @@ def _get_param_names(cls): "copy" ] + @reflect(reset=True) def fit(self, X, y=None) -> "StandardScaler": """Compute the mean and std to be used for later scaling. @@ -672,6 +676,7 @@ def fit(self, X, y=None) -> "StandardScaler": self._reset() return self.partial_fit(X, y) + @reflect(reset=True) def partial_fit(self, X, y=None) -> "StandardScaler": """ Online computation of mean and std on X for later scaling. @@ -792,6 +797,7 @@ def partial_fit(self, X, y=None) -> "StandardScaler": return self + @reflect def transform(self, X, copy=None) -> SparseCumlArray: """Perform standardization by centering and scaling @@ -827,6 +833,7 @@ def transform(self, X, copy=None) -> SparseCumlArray: return X + @reflect def inverse_transform(self, X, copy=None) -> SparseCumlArray: """Scale back the data to the original representation @@ -957,6 +964,7 @@ def _get_param_names(cls): "copy" ] + @reflect(reset=True) def fit(self, X, y=None) -> "MaxAbsScaler": """Compute the maximum absolute value to be used for later scaling. @@ -971,6 +979,7 @@ def fit(self, X, y=None) -> "MaxAbsScaler": self._reset() return self.partial_fit(X, y) + @reflect(reset=True) def partial_fit(self, X, y=None) -> "MaxAbsScaler": """ Online computation of max absolute value of X for later scaling. @@ -1015,6 +1024,7 @@ def partial_fit(self, X, y=None) -> "MaxAbsScaler": self.scale_ = _handle_zeros_in_scale(max_abs) return self + @reflect def transform(self, X) -> SparseCumlArray: """Scale the data @@ -1036,6 +1046,7 @@ def transform(self, X) -> SparseCumlArray: return X + @reflect def inverse_transform(self, X) -> SparseCumlArray: """Scale back the data to the original representation @@ -1057,7 +1068,7 @@ def inverse_transform(self, X) -> SparseCumlArray: return X -@api_return_array(get_output_type=True) +@reflect def maxabs_scale(X, *, axis=0, copy=True): """Scale each feature to the [-1, 1] range without breaking the sparsity. @@ -1209,6 +1220,7 @@ def _get_param_names(cls): "copy" ] + @reflect(reset=True) def fit(self, X, y=None) -> "RobustScaler": """Compute the median and quantiles to be used for scaling. @@ -1270,6 +1282,7 @@ def fit(self, X, y=None) -> "RobustScaler": return self + @reflect def transform(self, X) -> SparseCumlArray: """Center and scale the data. @@ -1294,6 +1307,7 @@ def transform(self, X) -> SparseCumlArray: X /= self.scale_ return X + @reflect def inverse_transform(self, X) -> SparseCumlArray: """Scale back the data to the original representation @@ -1319,7 +1333,7 @@ def inverse_transform(self, X) -> SparseCumlArray: return X -@api_return_array(get_output_type=True) +@reflect def robust_scale(X, *, axis=0, with_centering=True, with_scaling=True, quantile_range=(25.0, 75.0), copy=True): """ @@ -1529,6 +1543,7 @@ def get_feature_names(self, input_features=None): feature_names.append(name) return feature_names + @reflect(reset=True) def fit(self, X, y=None) -> "PolynomialFeatures": """ Compute number of output features. @@ -1552,6 +1567,7 @@ def fit(self, X, y=None) -> "PolynomialFeatures": self.n_output_features_ = sum(1 for _ in combinations) return self + @reflect def transform(self, X) -> SparseCumlArray: """Transform data to polynomial features @@ -1679,7 +1695,7 @@ def transform(self, X) -> SparseCumlArray: return XP # TODO keep order -@api_return_array(get_output_type=True) +@reflect def normalize(X, norm='l2', *, axis=1, copy=True, return_norm=False): """Scale input vectors individually to unit norm (vector length). @@ -1830,6 +1846,7 @@ def __init__(self, norm='l2', *, copy=True): self.norm = norm self.copy = copy + @reflect(reset=True) def fit(self, X, y=None) -> "Normalizer": """Do nothing and return the estimator unchanged @@ -1843,6 +1860,7 @@ def fit(self, X, y=None) -> "Normalizer": self._validate_data(X, accept_sparse='csr') return self + @reflect def transform(self, X, copy=None) -> SparseCumlArray: """Scale each non zero row of X to unit norm @@ -1859,7 +1877,7 @@ def transform(self, X, copy=None) -> SparseCumlArray: return normalize(X, norm=self.norm, axis=1, copy=copy) -@api_return_array(get_output_type=True) +@reflect def binarize(X, *, threshold=0.0, copy=True): """Boolean thresholding of array-like or sparse matrix @@ -1959,6 +1977,7 @@ def __init__(self, *, threshold=0.0, copy=True): self.threshold = threshold self.copy = copy + @reflect(reset=True) def fit(self, X, y=None) -> "Binarizer": """Do nothing and return the estimator unchanged @@ -1972,6 +1991,7 @@ def fit(self, X, y=None) -> "Binarizer": self._validate_data(X, accept_sparse=['csr', 'csc']) return self + @reflect def transform(self, X, copy=None) -> SparseCumlArray: """Binarize each element of X @@ -1988,7 +2008,7 @@ def transform(self, X, copy=None) -> SparseCumlArray: return binarize(X, threshold=self.threshold, copy=copy) -@api_return_array(get_output_type=True) +@reflect def add_dummy_feature(X, value=1.0): """Augment dataset with an additional dummy feature. @@ -2098,6 +2118,7 @@ def __init__(self): # Needed for backported inspect.signature compatibility with PyPy pass + @reflect(reset=True) def fit(self, K, y=None) -> 'KernelCenterer': """Fit KernelCenterer @@ -2123,6 +2144,7 @@ def fit(self, K, y=None) -> 'KernelCenterer': self.K_fit_all_ = self.K_fit_rows_.sum() / n_samples return self + @reflect def transform(self, K, copy=True) -> CumlArray: """Center kernel matrix. @@ -2355,6 +2377,7 @@ def _sparse_fit(self, X, random_state): # https://github.com/numpy/numpy/issues/14685 self.quantiles_ = np.array(cpu_np.maximum.accumulate(self.quantiles_)) + @reflect(reset=True) def fit(self, X, y=None) -> 'QuantileTransformer': """Compute the quantiles used for transforming. @@ -2548,6 +2571,7 @@ def _transform(self, X, inverse=False): return X + @reflect def transform(self, X) -> SparseCumlArray: """Feature-wise transformation of the data. @@ -2569,6 +2593,7 @@ def transform(self, X) -> SparseCumlArray: return self._transform(X, inverse=False) + @reflect def inverse_transform(self, X) -> SparseCumlArray: """Back-projection to the original space. @@ -2795,6 +2820,7 @@ def _get_param_names(cls): "copy" ] + @reflect(reset=True) def fit(self, X, y=None) -> 'PowerTransformer': """Estimate the optimal parameter lambda for each feature. @@ -2815,6 +2841,7 @@ def fit(self, X, y=None) -> 'PowerTransformer': self._fit(X, y=y, force_transform=False) return self + @reflect(reset=True) def fit_transform(self, X, y=None) -> CumlArray: return self._fit(X, y, force_transform=True) @@ -2853,6 +2880,7 @@ def _fit(self, X, y=None, force_transform=False): return X + @reflect def transform(self, X) -> CumlArray: """Apply the power transform to each feature using the fitted lambdas. @@ -2887,6 +2915,7 @@ def transform(self, X) -> CumlArray: return X + @reflect def inverse_transform(self, X) -> CumlArray: """Apply the inverse power transformation using the fitted lambdas. diff --git a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_discretization.py b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_discretization.py index f2cb73157c..bc190edd95 100644 --- a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_discretization.py +++ b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_discretization.py @@ -27,7 +27,7 @@ from ....common.array_descriptor import CumlArrayDescriptor from ....internals.array_sparse import SparseCumlArray -from ....internals.outputs import using_output_type +from ....internals.outputs import using_output_type, reflect from ....thirdparty_adapters import check_array from ..utils.skl_dependencies import BaseEstimator, TransformerMixin from ..utils.validation import FLOAT_DTYPES, check_is_fitted @@ -137,8 +137,6 @@ class KBinsDiscretizer(TransformerMixin, [ 0.5, 3.5, -1.5, 1.5]]) """ - - bin_edges_internal_ = CumlArrayDescriptor() n_bins_ = CumlArrayDescriptor() def __init__(self, n_bins=5, *, encode='onehot', strategy='quantile'): @@ -154,6 +152,7 @@ def _get_param_names(cls): "strategy" ] + @reflect(reset=True) def fit(self, X, y=None) -> "KBinsDiscretizer": """ Fit the estimator. @@ -237,7 +236,7 @@ def fit(self, X, y=None) -> "KBinsDiscretizer": 'decreasing the number of bins.' % jj) n_bins[jj] = len(bin_edges[jj]) - 1 - self.bin_edges_internal_ = bin_edges + self.bin_edges_ = bin_edges self.n_bins_ = n_bins if 'onehot' in self.encode: @@ -284,6 +283,7 @@ def _validate_n_bins(self, n_features): .format(KBinsDiscretizer.__name__, indices)) return n_bins + @reflect def transform(self, X) -> SparseCumlArray: """ Discretize the data. @@ -306,7 +306,7 @@ def transform(self, X) -> SparseCumlArray: raise ValueError("Incorrect number of features. Expecting {}, " "received {}.".format(n_features, Xt.shape[1])) - bin_edges = self.bin_edges_internal_ + bin_edges = self.bin_edges_ for jj in range(Xt.shape[1]): # Values which are close to a bin edge are susceptible to numeric # instability. Add eps to X so these values are binned correctly @@ -326,6 +326,7 @@ def transform(self, X) -> SparseCumlArray: Xt = self._encoder.transform(Xt) return Xt + @reflect def inverse_transform(self, Xt) -> SparseCumlArray: """ Transform discretized data back to original feature space. @@ -356,13 +357,9 @@ def inverse_transform(self, Xt) -> SparseCumlArray: "received {}.".format(n_features, Xinv.shape[1])) for jj in range(n_features): - bin_edges = self.bin_edges_internal_[jj] + bin_edges = self.bin_edges_[jj] bin_centers = (bin_edges[1:] + bin_edges[:-1]) * 0.5 idxs = np.asnumpy(Xinv[:, jj]) Xinv[:, jj] = bin_centers[idxs.astype(np.int32)] return Xinv - - @property - def bin_edges_(self): - return self.bin_edges_internal_ diff --git a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_function_transformer.py b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_function_transformer.py index 7e3a4cc166..c85d275696 100644 --- a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_function_transformer.py +++ b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_function_transformer.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2019-2024, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: BSD-3-Clause # This code originates from the Scikit-Learn library, @@ -12,6 +12,7 @@ import cuml from ....internals.array_sparse import SparseCumlArray +from ....internals.outputs import reflect from ..utils.skl_dependencies import BaseEstimator, TransformerMixin from ..utils.validation import _allclose_dense_sparse @@ -97,6 +98,7 @@ def _check_inverse_transform(self, X): " want to proceed regardless, set" " 'check_inverse=False'.", UserWarning) + @reflect(reset=True) def fit(self, X, y=None) -> "FunctionTransformer": """Fit transformer by checking X. @@ -115,6 +117,7 @@ def fit(self, X, y=None) -> "FunctionTransformer": self._check_inverse_transform(X) return self + @reflect def transform(self, X) -> SparseCumlArray: """Transform X using the forward function. @@ -130,6 +133,7 @@ def transform(self, X) -> SparseCumlArray: """ return self._transform(X, func=self.func, kw_args=self.kw_args) + @reflect def inverse_transform(self, X) -> SparseCumlArray: """Transform X using the inverse function. diff --git a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_imputation.py b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_imputation.py index 6df3d3e87a..82e771a2a8 100644 --- a/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_imputation.py +++ b/python/cuml/cuml/_thirdparty/sklearn/preprocessing/_imputation.py @@ -31,6 +31,7 @@ from ....common.array_descriptor import CumlArrayDescriptor from ....internals.array_sparse import SparseCumlArray +from ....internals.outputs import reflect from ....thirdparty_adapters import ( _get_mask, _masked_column_mean, @@ -299,6 +300,7 @@ def _validate_input(self, X, in_fit): return X + @reflect(reset=True) def fit(self, X, y=None) -> "SimpleImputer": """Fit the imputer on X. @@ -411,6 +413,7 @@ def _dense_fit(self, X, strategy, missing_values, fill_value): elif strategy == "constant": return np.full(X.shape[1], fill_value, dtype=X.dtype) + @reflect def transform(self, X) -> SparseCumlArray: """Impute all missing values in X. @@ -674,6 +677,7 @@ def _fit(self, X, y=None): return missing_features_info[0] + @reflect(reset=True) def fit(self, X, y=None) -> "MissingIndicator": """Fit the transformer on X. @@ -692,6 +696,7 @@ def fit(self, X, y=None) -> "MissingIndicator": return self + @reflect def transform(self, X) -> SparseCumlArray: """Generate missing values indicator for X. @@ -731,6 +736,7 @@ def transform(self, X) -> SparseCumlArray: return imputer_mask + @reflect(reset=True) def fit_transform(self, X, y=None) -> SparseCumlArray: """Generate missing values indicator for X. diff --git a/python/cuml/cuml/cluster/agglomerative.pyx b/python/cuml/cuml/cluster/agglomerative.pyx index d33d0a5e50..d5dc3f499a 100644 --- a/python/cuml/cuml/cluster/agglomerative.pyx +++ b/python/cuml/cuml/cluster/agglomerative.pyx @@ -12,6 +12,7 @@ from cuml.common.doc_utils import generate_docstring from cuml.internals.array import CumlArray from cuml.internals.base import Base from cuml.internals.mixins import ClusterMixin, CMajorInputTagMixin +from cuml.internals.outputs import reflect from libc.stdint cimport uintptr_t from libcpp cimport bool @@ -150,6 +151,7 @@ class AgglomerativeClustering(Base, ClusterMixin, CMajorInputTagMixin): self.n_neighbors = n_neighbors @generate_docstring() + @reflect(reset=True) def fit(self, X, y=None, *, convert_dtype=True) -> "AgglomerativeClustering": """ Fit the hierarchical clustering from features. @@ -260,6 +262,7 @@ class AgglomerativeClustering(Base, ClusterMixin, CMajorInputTagMixin): "type": "dense", "description": "Cluster indexes", "shape": "(n_samples, 1)"}) + @reflect def fit_predict(self, X, y=None) -> CumlArray: """ Fit and return the assigned cluster labels. diff --git a/python/cuml/cuml/cluster/dbscan.pyx b/python/cuml/cuml/cluster/dbscan.pyx index 0b042a98f8..680a2f130f 100644 --- a/python/cuml/cuml/cluster/dbscan.pyx +++ b/python/cuml/cuml/cluster/dbscan.pyx @@ -8,7 +8,7 @@ import numpy as np from cuml.common.array_descriptor import CumlArrayDescriptor from cuml.common.doc_utils import generate_docstring -from cuml.internals import logger +from cuml.internals import logger, reflect from cuml.internals.array import CumlArray from cuml.internals.base import Base from cuml.internals.input_utils import input_to_cuml_array @@ -308,6 +308,7 @@ class DBSCAN(Base, self.algorithm = algorithm @generate_docstring(skip_parameters_heading=True) + @reflect(reset=True) def fit( self, X, @@ -497,6 +498,7 @@ class DBSCAN(Base, 'type': 'dense', 'description': 'Cluster labels', 'shape': '(n_samples, 1)'}) + @reflect def fit_predict(self, X, y=None, sample_weight=None, *, out_dtype="int32") -> CumlArray: """ Performs clustering on X and returns cluster labels. diff --git a/python/cuml/cuml/cluster/hdbscan/hdbscan.pyx b/python/cuml/cuml/cluster/hdbscan/hdbscan.pyx index ff1e7ffd48..a603bee043 100644 --- a/python/cuml/cuml/cluster/hdbscan/hdbscan.pyx +++ b/python/cuml/cuml/cluster/hdbscan/hdbscan.pyx @@ -1,8 +1,5 @@ # SPDX-FileCopyrightText: Copyright (c) 2021-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -# - -# distutils: language = c++ import cupy as cp import numpy as np from pylibraft.common.handle import Handle @@ -11,7 +8,7 @@ import cuml from cuml.common import input_to_cuml_array from cuml.common.array_descriptor import CumlArrayDescriptor from cuml.common.doc_utils import generate_docstring -from cuml.internals import logger +from cuml.internals import logger, reflect from cuml.internals.array import CumlArray from cuml.internals.base import Base from cuml.internals.interop import ( @@ -913,6 +910,7 @@ class HDBSCAN(Base, InteropMixin, ClusterMixin, CMajorInputTagMixin): self.prediction_data = True @generate_docstring() + @reflect(reset=True) def fit(self, X, y=None, *, convert_dtype=True) -> "HDBSCAN": """ Fit HDBSCAN model from features. @@ -1064,6 +1062,7 @@ class HDBSCAN(Base, InteropMixin, ClusterMixin, CMajorInputTagMixin): 'type': 'dense', 'description': 'Cluster indexes', 'shape': '(n_samples, 1)'}) + @reflect def fit_predict(self, X, y=None) -> CumlArray: """ Fit the HDBSCAN model from features and return @@ -1114,7 +1113,7 @@ def _check_clusterer(clusterer): return state -@cuml.internals.reflect(model="clusterer", array=None) +@reflect(model="clusterer", array=None) def all_points_membership_vectors(clusterer, int batch_size=4096): """ Predict soft cluster membership vectors for all points in the @@ -1183,7 +1182,7 @@ def all_points_membership_vectors(clusterer, int batch_size=4096): return membership_vec -@cuml.internals.reflect(model="clusterer", array="points_to_predict") +@reflect(model="clusterer", array="points_to_predict") def membership_vector(clusterer, points_to_predict, int batch_size=4096, convert_dtype=True): """ Predict soft cluster membership. The result produces a vector @@ -1269,7 +1268,7 @@ def membership_vector(clusterer, points_to_predict, int batch_size=4096, convert return membership_vec -@cuml.internals.reflect(model="clusterer", array="points_to_predict") +@reflect(model="clusterer", array="points_to_predict") def approximate_predict(clusterer, points_to_predict, convert_dtype=True): """Predict the cluster label of new points. The returned labels will be those of the original clustering found by ``clusterer``, diff --git a/python/cuml/cuml/cluster/kmeans.pyx b/python/cuml/cuml/cluster/kmeans.pyx index 7c4e3af371..440ea5dfdb 100644 --- a/python/cuml/cuml/cluster/kmeans.pyx +++ b/python/cuml/cuml/cluster/kmeans.pyx @@ -18,6 +18,7 @@ from cuml.internals.interop import ( to_gpu, ) from cuml.internals.mixins import ClusterMixin, CMajorInputTagMixin +from cuml.internals.outputs import reflect from cuml.internals.utils import check_random_seed from libc.stdint cimport int64_t, uintptr_t @@ -516,6 +517,7 @@ class KMeans(Base, return self.n_clusters @generate_docstring() + @reflect(reset=True) def fit(self, X, y=None, sample_weight=None, *, convert_dtype=True) -> "KMeans": """ Compute k-means clustering with X. @@ -598,6 +600,7 @@ class KMeans(Base, 'type': 'dense', 'description': 'Cluster indexes', 'shape': '(n_samples, 1)'}) + @reflect def fit_predict(self, X, y=None, sample_weight=None) -> CumlArray: """ Compute cluster centers and predict cluster index for each sample. @@ -671,6 +674,7 @@ class KMeans(Base, 'type': 'dense', 'description': 'Cluster indexes', 'shape': '(n_samples, 1)'}) + @reflect def predict( self, X, @@ -688,6 +692,7 @@ class KMeans(Base, 'type': 'dense', 'description': 'Transformed data', 'shape': '(n_samples, n_clusters)'}) + @reflect def transform(self, X, *, convert_dtype=True) -> CumlArray: """ Transform X to a cluster-distance space. @@ -772,6 +777,7 @@ class KMeans(Base, 'description': 'Opposite of the value \ of X on the K-means \ objective.'}) + @reflect(skip=True) def score(self, X, y=None, sample_weight=None, *, convert_dtype=True): """ Opposite of the value of X on the K-means objective. @@ -787,6 +793,7 @@ class KMeans(Base, 'type': 'dense', 'description': 'Transformed data', 'shape': '(n_samples, n_clusters)'}) + @reflect def fit_transform( self, X, y=None, sample_weight=None, *, convert_dtype=False ) -> CumlArray: diff --git a/python/cuml/cuml/common/sparsefuncs.py b/python/cuml/cuml/common/sparsefuncs.py index e7c3fb5dd0..3a858ffb25 100644 --- a/python/cuml/cuml/common/sparsefuncs.py +++ b/python/cuml/cuml/common/sparsefuncs.py @@ -13,7 +13,6 @@ from cupyx.scipy.sparse import csr_matrix as cp_csr_matrix from scipy.sparse import coo_matrix, csc_matrix, csr_matrix -import cuml from cuml.common.kernel_utils import cuda_kernel_factory from cuml.internals.input_utils import input_to_cuml_array, input_to_cupy_array @@ -72,7 +71,6 @@ def _map_l2_norm_kernel(dtype): return cuda_kernel_factory(map_kernel_str, dtype, "map_l2_norm_kernel") -@cuml.internals.api_return_any() def csr_row_normalize_l1(X, inplace=True): """Row normalize for csr matrix using the l1 norm""" if not inplace: @@ -88,7 +86,6 @@ def csr_row_normalize_l1(X, inplace=True): return X -@cuml.internals.api_return_any() def csr_row_normalize_l2(X, inplace=True): """Row normalize for csr matrix using the l2 norm""" if not inplace: @@ -104,7 +101,6 @@ def csr_row_normalize_l2(X, inplace=True): return X -@cuml.internals.api_return_any() def csr_diag_mul(X, y, inplace=True): """Multiply a sparse X matrix with diagonal matrix y""" if not inplace: @@ -115,7 +111,6 @@ def csr_diag_mul(X, y, inplace=True): return X -@cuml.internals.api_return_any() def create_csr_matrix_from_count_df( count_df, empty_doc_ids, n_doc, n_features, dtype=np.float32 ): diff --git a/python/cuml/cuml/decomposition/incremental_pca.py b/python/cuml/cuml/decomposition/incremental_pca.py index d7b8e2320a..2bb485fb50 100644 --- a/python/cuml/cuml/decomposition/incremental_pca.py +++ b/python/cuml/cuml/decomposition/incremental_pca.py @@ -203,6 +203,7 @@ def __init__( ) self.batch_size = batch_size + @cuml.internals.reflect(reset=True) def fit(self, X, y=None, *, convert_dtype=True) -> "IncrementalPCA": """ Fit the model with X, using minibatches of size batch_size. @@ -258,7 +259,7 @@ def fit(self, X, y=None, *, convert_dtype=True) -> "IncrementalPCA": return self - @cuml.internals.api_base_return_any_skipall + @cuml.internals.reflect(skip=True) def partial_fit(self, X, y=None, *, check_input=True) -> "IncrementalPCA": """ Incremental fit with X. All of X is processed as a single batch. @@ -400,6 +401,7 @@ def partial_fit(self, X, y=None, *, check_input=True) -> "IncrementalPCA": return self + @cuml.internals.reflect def transform(self, X, *, convert_dtype=False) -> CumlArray: """ Apply dimensionality reduction to X. diff --git a/python/cuml/cuml/decomposition/pca.pyx b/python/cuml/cuml/decomposition/pca.pyx index 4bbae2b3f9..e4357d4a12 100644 --- a/python/cuml/cuml/decomposition/pca.pyx +++ b/python/cuml/cuml/decomposition/pca.pyx @@ -475,6 +475,7 @@ class PCA(Base, self.noise_variance_ = noise_variance @generate_docstring(X='dense_sparse') + @cuml.internals.reflect(reset=True) def fit(self, X, y=None, *, convert_dtype=True) -> "PCA": """ Fit the model with X. y is currently ignored. @@ -513,7 +514,7 @@ class PCA(Base, 'type': 'dense_sparse', 'description': 'Transformed values', 'shape': '(n_samples, n_components)'}) - @cuml.internals.api_base_return_array_skipall + @cuml.internals.reflect def fit_transform(self, X, y=None) -> CumlArray: """ Fit the model with X and apply the dimensionality reduction on X. @@ -592,6 +593,7 @@ class PCA(Base, 'type': 'dense_sparse', 'description': 'Transformed values', 'shape': '(n_samples, n_features)'}) + @cuml.internals.reflect def inverse_transform( self, X, @@ -687,6 +689,7 @@ class PCA(Base, 'type': 'dense_sparse', 'description': 'Transformed values', 'shape': '(n_samples, n_components)'}) + @cuml.internals.reflect def transform(self, X, *, convert_dtype=True) -> CumlArray: """ Apply dimensionality reduction to X. diff --git a/python/cuml/cuml/decomposition/tsvd.pyx b/python/cuml/cuml/decomposition/tsvd.pyx index eb30ec1f8a..6496bd0d1a 100644 --- a/python/cuml/cuml/decomposition/tsvd.pyx +++ b/python/cuml/cuml/decomposition/tsvd.pyx @@ -284,6 +284,7 @@ class TruncatedSVD(Base, return self.components_.shape[0] @generate_docstring() + @cuml.internals.reflect def fit(self, X, y=None) -> "TruncatedSVD": """ Fit model on training cudf DataFrame X. y is currently ignored. @@ -296,7 +297,7 @@ class TruncatedSVD(Base, 'type': 'dense', 'description': 'Reduced version of X', 'shape': '(n_samples, n_components)'}) - @cuml.internals.api_base_fit_transform() + @cuml.internals.reflect(reset=True) def fit_transform(self, X, y=None, *, convert_dtype=True) -> CumlArray: """ Fit model to X and perform dimensionality reduction on X. @@ -390,6 +391,7 @@ class TruncatedSVD(Base, 'type': 'dense', 'description': 'X in original space', 'shape': '(n_samples, n_features)'}) + @cuml.internals.reflect def inverse_transform(self, X, *, convert_dtype=False) -> CumlArray: """ Transform X back to its original space. @@ -444,6 +446,7 @@ class TruncatedSVD(Base, 'type': 'dense', 'description': 'Reduced version of X', 'shape': '(n_samples, n_components)'}) + @cuml.internals.reflect def transform(self, X, *, convert_dtype=True) -> CumlArray: """ Perform dimensionality reduction on X. diff --git a/python/cuml/cuml/ensemble/randomforestclassifier.py b/python/cuml/cuml/ensemble/randomforestclassifier.py index d81eff2fbe..bb2a03e867 100644 --- a/python/cuml/cuml/ensemble/randomforestclassifier.py +++ b/python/cuml/cuml/ensemble/randomforestclassifier.py @@ -218,6 +218,7 @@ def __init__( y="dense_intdtype", convert_dtype_cast="np.float32", ) + @cuml.internals.reflect(reset=True) def fit(self, X, y, *, convert_dtype=True) -> "RandomForestClassifier": """ Perform Random Forest Classification on the input data @@ -252,7 +253,7 @@ def fit(self, X, y, *, convert_dtype=True) -> "RandomForestClassifier": parameters=[("dense", "(n_samples, n_features)")], return_values=[("dense", "(n_samples, 1)")], ) - @cuml.internals.api_base_return_any_skipall + @cuml.internals.reflect(skip=True) def predict( self, X, @@ -302,6 +303,7 @@ def predict( parameters=[("dense", "(n_samples, n_features)")], return_values=[("dense", "(n_samples, 1)")], ) + @cuml.internals.reflect def predict_proba( self, X, @@ -355,6 +357,7 @@ def predict_proba( ("dense_intdtype", "(n_samples, 1)"), ] ) + @cuml.internals.reflect(skip=True) def score( self, X, diff --git a/python/cuml/cuml/ensemble/randomforestregressor.py b/python/cuml/cuml/ensemble/randomforestregressor.py index 60bd416f84..35e1eaecdc 100644 --- a/python/cuml/cuml/ensemble/randomforestregressor.py +++ b/python/cuml/cuml/ensemble/randomforestregressor.py @@ -9,6 +9,7 @@ from cuml.ensemble.randomforest_common import BaseRandomForestModel from cuml.internals.array import CumlArray from cuml.internals.mixins import RegressorMixin +from cuml.internals.outputs import reflect from cuml.metrics import r2_score @@ -185,6 +186,7 @@ def __init__( domain="cuml_python", ) @generate_docstring() + @reflect(reset=True) def fit(self, X, y, *, convert_dtype=True) -> "RandomForestRegressor": """ Perform Random Forest Regression on the input data @@ -215,6 +217,7 @@ def fit(self, X, y, *, convert_dtype=True) -> "RandomForestRegressor": parameters=[("dense", "(n_samples, n_features)")], return_values=[("dense", "(n_samples, 1)")], ) + @reflect def predict( self, X, @@ -274,6 +277,7 @@ def predict( ("dense", "(n_samples, 1)"), ] ) + @reflect(skip=True) def score( self, X, diff --git a/python/cuml/cuml/experimental/linear_model/lars.pyx b/python/cuml/cuml/experimental/linear_model/lars.pyx index 6455fe17ee..cc3d57d613 100644 --- a/python/cuml/cuml/experimental/linear_model/lars.pyx +++ b/python/cuml/cuml/experimental/linear_model/lars.pyx @@ -2,34 +2,25 @@ # SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - import cupy as cp import numpy as np -from cuml.internals import logger - -from cuml.internals cimport logger - import cuml.internals - -from libc.stdint cimport uintptr_t -from libcpp cimport nullptr - from cuml.common import input_to_cuml_array from cuml.common.array_descriptor import CumlArrayDescriptor from cuml.common.doc_utils import generate_docstring +from cuml.internals import logger, reflect from cuml.internals.array import CumlArray from cuml.internals.base import Base from cuml.internals.mixins import RegressorMixin from cuml.linear_model.base import check_deprecated_normalize +from libc.stdint cimport uintptr_t +from libcpp cimport nullptr from pylibraft.common.handle cimport handle_t +from cuml.internals cimport logger + cdef extern from "cuml/solvers/lars.hpp" namespace "ML::Solver::Lars" nogil: @@ -285,6 +276,7 @@ class Lars(Base, RegressorMixin): self.beta_ = self.beta_ / x_scale[self.active_] @generate_docstring(y='dense_anydtype') + @reflect(reset=True) def fit(self, X, y, convert_dtype=True) -> 'Lars': """ Fit the model with X and y. @@ -337,6 +329,7 @@ class Lars(Base, RegressorMixin): return self + @reflect def predict(self, X, convert_dtype=True) -> CumlArray: """ Predicts `y` values for `X`. diff --git a/python/cuml/cuml/feature_extraction/_tfidf.py b/python/cuml/cuml/feature_extraction/_tfidf.py index 89a2cfc728..9671f14a46 100644 --- a/python/cuml/cuml/feature_extraction/_tfidf.py +++ b/python/cuml/cuml/feature_extraction/_tfidf.py @@ -171,7 +171,7 @@ def _set_idf_diag(self): # Free up memory occupied by below del self.__df - @cuml.internals.api_base_return_any_skipall + @cuml.internals.reflect(skip=True) def fit(self, X, y=None) -> "TfidfTransformer": """Learn the idf vector (global term weights). @@ -188,7 +188,7 @@ def fit(self, X, y=None) -> "TfidfTransformer": return self - @cuml.internals.api_base_return_any_skipall + @cuml.internals.reflect(skip=True) def transform(self, X, copy=True): """Transform a count matrix to a tf or tf-idf representation @@ -240,7 +240,7 @@ def transform(self, X, copy=True): return X - @cuml.internals.api_base_return_any_skipall + @cuml.internals.reflect(skip=True) def fit_transform(self, X, y=None, copy=True): """ Fit TfidfTransformer to X, then transform X. diff --git a/python/cuml/cuml/fil/fil.pyx b/python/cuml/cuml/fil/fil.pyx index 1406beb8d1..d3040329f0 100644 --- a/python/cuml/cuml/fil/fil.pyx +++ b/python/cuml/cuml/fil/fil.pyx @@ -17,6 +17,7 @@ from cuml.internals.global_settings import GlobalSettings from cuml.internals.input_utils import input_to_cuml_array from cuml.internals.mem_type import MemoryType from cuml.internals.mixins import CMajorInputTagMixin +from cuml.internals.outputs import reflect from cuml.internals.treelite import safe_treelite_call from libc.stdint cimport uint32_t, uintptr_t @@ -987,6 +988,7 @@ class ForestInference(Base, CMajorInputTagMixin): message='ForestInference.predict_proba', domain='cuml_python' ) + @reflect def predict_proba( self, X, @@ -1043,6 +1045,7 @@ class ForestInference(Base, CMajorInputTagMixin): message='ForestInference.predict', domain='cuml_python' ) + @reflect def predict( self, X, @@ -1132,6 +1135,7 @@ class ForestInference(Base, CMajorInputTagMixin): message='ForestInference.predict_per_tree', domain='cuml_python' ) + @reflect def predict_per_tree( self, X, @@ -1186,6 +1190,7 @@ class ForestInference(Base, CMajorInputTagMixin): message='ForestInference.apply', domain='cuml_python' ) + @reflect def apply( self, X, @@ -1373,7 +1378,3 @@ class ForestInference(Base, CMajorInputTagMixin): "precision", "device_id", ] - - def set_params(self, **params): - super().set_params(**params) - return self diff --git a/python/cuml/cuml/internals/mixins.py b/python/cuml/cuml/internals/mixins.py index 096c99c36b..62853f5696 100644 --- a/python/cuml/cuml/internals/mixins.py +++ b/python/cuml/cuml/internals/mixins.py @@ -9,7 +9,7 @@ from cuml._thirdparty._sklearn_compat import _to_new_tags from cuml.common.doc_utils import generate_docstring from cuml.internals.base_helpers import _tags_class_and_instance -from cuml.internals.outputs import api_base_return_any_skipall +from cuml.internals.outputs import reflect ############################################################################### # Tag Functionality Mixin # @@ -190,7 +190,7 @@ class RegressorMixin: "description": "R^2 of self.predict(X) wrt. y.", } ) - @api_base_return_any_skipall + @reflect(skip=True) def score(self, X, y, sample_weight=None, **kwargs): """ Scoring function for regression estimators @@ -225,7 +225,7 @@ class ClassifierMixin: ), } ) - @api_base_return_any_skipall + @reflect(skip=True) def score(self, X, y, sample_weight=None, **kwargs): """ Scoring function for classifier estimators based on mean accuracy. diff --git a/python/cuml/cuml/kernel_ridge/kernel_ridge.py b/python/cuml/cuml/kernel_ridge/kernel_ridge.py index 6e5d4c075f..56ed66bec8 100644 --- a/python/cuml/cuml/kernel_ridge/kernel_ridge.py +++ b/python/cuml/cuml/kernel_ridge/kernel_ridge.py @@ -12,7 +12,7 @@ from cuml.common import input_to_cuml_array from cuml.common.array_descriptor import CumlArrayDescriptor from cuml.common.doc_utils import generate_docstring -from cuml.internals import api_base_return_array +from cuml.internals import reflect from cuml.internals.array import CumlArray from cuml.internals.base import Base from cuml.internals.interop import ( @@ -289,6 +289,7 @@ def _get_kernel(self, X, Y=None): ) @generate_docstring() + @reflect(reset=True) def fit( self, X, y, sample_weight=None, *, convert_dtype=True ) -> "KernelRidge": @@ -324,7 +325,7 @@ def fit( self.dual_coef_ = CumlArray(data=dual_coef) return self - @api_base_return_array() + @reflect def predict(self, X, *, convert_dtype=True): """ Predict using the kernel ridge model. diff --git a/python/cuml/cuml/linear_model/base.py b/python/cuml/cuml/linear_model/base.py index e66dfa1440..c85b4d669b 100644 --- a/python/cuml/cuml/linear_model/base.py +++ b/python/cuml/cuml/linear_model/base.py @@ -21,7 +21,7 @@ class LinearPredictMixin: "shape": "(n_samples, 1)", } ) - @cuml.internals.api_base_return_array() + @cuml.internals.reflect def predict(self, X, *, convert_dtype=True) -> CumlArray: """ Predicts `y` values for `X`. @@ -63,7 +63,7 @@ class LinearClassifierMixin: "shape": "(n_samples,) or (n_samples, n_classes)", }, ) - @cuml.internals.api_base_return_array() + @cuml.internals.reflect def decision_function(self, X, *, convert_dtype=True) -> CumlArray: """Predict confidence scores for samples.""" if is_sparse(X): diff --git a/python/cuml/cuml/linear_model/elastic_net.py b/python/cuml/cuml/linear_model/elastic_net.py index b6dfe2c40e..911e34c3c4 100644 --- a/python/cuml/cuml/linear_model/elastic_net.py +++ b/python/cuml/cuml/linear_model/elastic_net.py @@ -13,6 +13,7 @@ to_gpu, ) from cuml.internals.mixins import FMajorInputTagMixin, RegressorMixin +from cuml.internals.outputs import reflect from cuml.linear_model.base import ( LinearPredictMixin, check_deprecated_normalize, @@ -230,6 +231,7 @@ def __init__( self.normalize = normalize @generate_docstring() + @reflect(reset=True) def fit( self, X, y, sample_weight=None, *, convert_dtype=True ) -> "ElasticNet": diff --git a/python/cuml/cuml/linear_model/linear_regression.pyx b/python/cuml/cuml/linear_model/linear_regression.pyx index d8965f68c3..d770b56b4d 100644 --- a/python/cuml/cuml/linear_model/linear_regression.pyx +++ b/python/cuml/cuml/linear_model/linear_regression.pyx @@ -21,6 +21,7 @@ from cuml.internals.interop import ( to_gpu, ) from cuml.internals.mixins import FMajorInputTagMixin, RegressorMixin +from cuml.internals.outputs import reflect from cuml.linear_model.base import ( LinearPredictMixin, check_deprecated_normalize, @@ -305,6 +306,7 @@ class LinearRegression(Base, return algo @generate_docstring() + @reflect(reset=True) def fit(self, X, y, sample_weight=None, *, convert_dtype=True) -> "LinearRegression": """ Fit the model with X and y. diff --git a/python/cuml/cuml/linear_model/logistic_regression.py b/python/cuml/cuml/linear_model/logistic_regression.py index 8dd1921d6f..6482a858b8 100644 --- a/python/cuml/cuml/linear_model/logistic_regression.py +++ b/python/cuml/cuml/linear_model/logistic_regression.py @@ -281,7 +281,7 @@ def _get_l1_l2_strength(self): return l1_strength, l2_strength @generate_docstring(X="dense_sparse") - @cuml.internals.api_base_return_any() + @cuml.internals.reflect(reset=True) def fit( self, X, y, sample_weight=None, *, convert_dtype=True ) -> "LogisticRegression": @@ -334,7 +334,7 @@ def fit( "shape": "(n_samples, 1)", }, ) - @cuml.internals.api_base_return_any_skipall + @cuml.internals.reflect(skip=True) def predict(self, X, *, convert_dtype=True): """ Predicts the y for X. @@ -362,6 +362,7 @@ def predict(self, X, *, convert_dtype=True): "shape": "(n_samples, n_classes)", }, ) + @cuml.internals.reflect def predict_proba(self, X, *, convert_dtype=True) -> CumlArray: """ Predicts the class probabilities for each class in X @@ -391,6 +392,7 @@ def predict_proba(self, X, *, convert_dtype=True) -> CumlArray: "shape": "(n_samples, n_classes)", }, ) + @cuml.internals.reflect def predict_log_proba(self, X, *, convert_dtype=True) -> CumlArray: """ Predicts the log class probabilities for each class in X diff --git a/python/cuml/cuml/linear_model/mbsgd_classifier.py b/python/cuml/cuml/linear_model/mbsgd_classifier.py index a95adad373..1073611644 100644 --- a/python/cuml/cuml/linear_model/mbsgd_classifier.py +++ b/python/cuml/cuml/linear_model/mbsgd_classifier.py @@ -183,6 +183,7 @@ def __init__( self.n_iter_no_change = n_iter_no_change @generate_docstring() + @cuml.internals.reflect(reset=True) def fit(self, X, y, *, convert_dtype=True) -> "MBSGDClassifier": """ Fit the model with X and y. @@ -227,7 +228,7 @@ def fit(self, X, y, *, convert_dtype=True) -> "MBSGDClassifier": "shape": "(n_samples, 1)", } ) - @cuml.internals.api_base_return_any_skipall + @cuml.internals.reflect(skip=True) def predict(self, X, *, convert_dtype=True): """ Predicts the y for X. diff --git a/python/cuml/cuml/linear_model/mbsgd_regressor.py b/python/cuml/cuml/linear_model/mbsgd_regressor.py index 8c7a52e695..8ad22fa95a 100644 --- a/python/cuml/cuml/linear_model/mbsgd_regressor.py +++ b/python/cuml/cuml/linear_model/mbsgd_regressor.py @@ -4,6 +4,7 @@ from cuml.common.doc_utils import generate_docstring from cuml.internals.base import Base from cuml.internals.mixins import FMajorInputTagMixin, RegressorMixin +from cuml.internals.outputs import reflect from cuml.linear_model.base import LinearPredictMixin from cuml.solvers.sgd import fit_sgd @@ -170,6 +171,7 @@ def __init__( self.n_iter_no_change = n_iter_no_change @generate_docstring() + @reflect(reset=True) def fit(self, X, y, *, convert_dtype=True) -> "MBSGDRegressor": """ Fit the model with X and y. diff --git a/python/cuml/cuml/linear_model/ridge.pyx b/python/cuml/cuml/linear_model/ridge.pyx index dfdd9f30dc..fe5642c0aa 100644 --- a/python/cuml/cuml/linear_model/ridge.pyx +++ b/python/cuml/cuml/linear_model/ridge.pyx @@ -17,6 +17,7 @@ from cuml.internals.interop import ( to_gpu, ) from cuml.internals.mixins import FMajorInputTagMixin, RegressorMixin +from cuml.internals.outputs import reflect from cuml.linear_model.base import ( LinearPredictMixin, check_deprecated_normalize, @@ -428,6 +429,7 @@ class Ridge(Base, return coef, intercept @generate_docstring() + @reflect(reset=True) def fit(self, X, y, sample_weight=None, *, convert_dtype=True) -> "Ridge": """ Fit the model with X and y. diff --git a/python/cuml/cuml/manifold/spectral_embedding.pyx b/python/cuml/cuml/manifold/spectral_embedding.pyx index 337d418742..d4883c150a 100644 --- a/python/cuml/cuml/manifold/spectral_embedding.pyx +++ b/python/cuml/cuml/manifold/spectral_embedding.pyx @@ -8,7 +8,6 @@ import numpy as np import scipy.sparse as sp from pylibraft.common.handle import Handle -import cuml from cuml.common.array_descriptor import CumlArrayDescriptor from cuml.internals.array import CumlArray from cuml.internals.base import Base @@ -20,6 +19,7 @@ from cuml.internals.interop import ( to_gpu, ) from cuml.internals.mixins import CMajorInputTagMixin +from cuml.internals.outputs import reflect from cuml.internals.utils import check_random_seed from libc.stdint cimport uint64_t, uintptr_t @@ -60,7 +60,7 @@ cdef extern from "cuml/manifold/spectral_embedding.hpp" \ device_matrix_view[float, int, col_major] embedding) except + -@cuml.internals.api_return_array(get_output_type=True) +@reflect def spectral_embedding(A, *, int n_components=8, @@ -379,6 +379,7 @@ class SpectralEmbedding(Base, **super()._attrs_to_cpu(model), } + @reflect def fit_transform(self, X, y=None) -> CumlArray: """Fit the model from data in X and transform X. @@ -402,6 +403,7 @@ class SpectralEmbedding(Base, self.fit(X, y) return self.embedding_ + @reflect(reset=True) def fit(self, X, y=None) -> "SpectralEmbedding": """Fit the model from data in X. diff --git a/python/cuml/cuml/manifold/t_sne.pyx b/python/cuml/cuml/manifold/t_sne.pyx index ee1a9029e2..cee3d1801b 100644 --- a/python/cuml/cuml/manifold/t_sne.pyx +++ b/python/cuml/cuml/manifold/t_sne.pyx @@ -8,7 +8,6 @@ import numpy as np import sklearn from packaging.version import Version -import cuml.internals from cuml.common import input_to_cuml_array from cuml.common.array_descriptor import CumlArrayDescriptor from cuml.common.doc_utils import generate_docstring @@ -24,6 +23,7 @@ from cuml.internals.interop import ( to_gpu, ) from cuml.internals.mixins import CMajorInputTagMixin, SparseInputTagMixin +from cuml.internals.outputs import reflect from cuml.internals.utils import check_random_seed from libc.stdint cimport int64_t, uintptr_t @@ -594,6 +594,7 @@ class TSNE(Base, @generate_docstring(skip_parameters_heading=True, X='dense_sparse', convert_dtype_cast='np.float32') + @reflect(reset=True) def fit(self, X, y=None, *, convert_dtype=True, knn_graph=None) -> "TSNE": """ Fit X into an embedded space. @@ -714,7 +715,7 @@ class TSNE(Base, data in \ low-dimensional space.', 'shape': '(n_samples, n_components)'}) - @cuml.internals.api_base_fit_transform() + @reflect def fit_transform(self, X, y=None, *, convert_dtype=True, knn_graph=None) -> CumlArray: """ Fit X into an embedded space and return that transformed output. diff --git a/python/cuml/cuml/manifold/umap/umap.pyx b/python/cuml/cuml/manifold/umap/umap.pyx index acac8abda2..d0c6032b05 100644 --- a/python/cuml/cuml/manifold/umap/umap.pyx +++ b/python/cuml/cuml/manifold/umap/umap.pyx @@ -12,13 +12,11 @@ import numpy as np import scipy.sparse from pylibraft.common.handle import Handle -import cuml.accel -import cuml.internals from cuml.common.array_descriptor import CumlArrayDescriptor from cuml.common.doc_utils import generate_docstring from cuml.common.sparse_utils import is_sparse from cuml.common.sparsefuncs import extract_knn_graph -from cuml.internals import logger +from cuml.internals import logger, reflect from cuml.internals.array import CumlArray from cuml.internals.array_sparse import SparseCumlArray from cuml.internals.base import Base @@ -888,6 +886,7 @@ class UMAP(Base, InteropMixin, CMajorInputTagMixin, SparseInputTagMixin): X="dense_sparse", skip_parameters_heading=True, ) + @reflect(reset=True) def fit(self, X, y=None, *, convert_dtype=True, knn_graph=None) -> "UMAP": """ Fit X into an embedded space. @@ -1053,7 +1052,7 @@ class UMAP(Base, InteropMixin, CMajorInputTagMixin, SparseInputTagMixin): "shape": "(n_samples, n_components)" } ) - @cuml.internals.api_base_fit_transform() + @reflect def fit_transform( self, X, y=None, *, convert_dtype=True, knn_graph=None ) -> CumlArray: @@ -1093,6 +1092,7 @@ class UMAP(Base, InteropMixin, CMajorInputTagMixin, SparseInputTagMixin): "shape": "(n_samples, n_components)" } ) + @reflect def transform(self, X, *, convert_dtype=True) -> CumlArray: """ Transform X into the existing embedded space and return that @@ -1357,7 +1357,7 @@ def fuzzy_simplicial_set( return fss_graph.view_cupy_coo() -@cuml.internals.api_return_array(get_output_type=True) +@reflect def simplicial_set_embedding( data, graph, diff --git a/python/cuml/cuml/metrics/_classification.py b/python/cuml/cuml/metrics/_classification.py index 1afc6dcd69..ae0e245b64 100644 --- a/python/cuml/cuml/metrics/_classification.py +++ b/python/cuml/cuml/metrics/_classification.py @@ -6,7 +6,6 @@ import cupy as cp import numpy as np -import cuml.internals from cuml.internals.input_utils import input_to_cupy_array @@ -45,7 +44,6 @@ def _input_to_cupy_or_cudf_series(x, check_rows=None): return out -@cuml.internals.api_return_any() def accuracy_score(y_true, y_pred, *, sample_weight=None, normalize=True): """ Accuracy classification score. @@ -104,7 +102,6 @@ def accuracy_score(y_true, y_pred, *, sample_weight=None, normalize=True): return float(cp.count_nonzero(correct)) -@cuml.internals.api_return_any() def log_loss( y_true, y_pred, eps=1e-15, normalize=True, sample_weight=None ) -> float: diff --git a/python/cuml/cuml/metrics/_ranking.py b/python/cuml/cuml/metrics/_ranking.py index ff0d9cebce..355f86d3b5 100644 --- a/python/cuml/cuml/metrics/_ranking.py +++ b/python/cuml/cuml/metrics/_ranking.py @@ -14,7 +14,7 @@ from cuml.internals.input_utils import input_to_cupy_array -@cuml.internals.api_return_array(get_output_type=True) +@cuml.internals.reflect def precision_recall_curve( y_true, probs_pred ) -> typing.Tuple[CumlArray, CumlArray, CumlArray]: @@ -107,7 +107,6 @@ def precision_recall_curve( return precision, recall, thresholds -@cuml.internals.api_return_any() def roc_auc_score(y_true, y_score): """ Compute Area Under the Receiver Operating Characteristic Curve (ROC AUC) diff --git a/python/cuml/cuml/metrics/cluster/adjusted_rand_index.pyx b/python/cuml/cuml/metrics/cluster/adjusted_rand_index.pyx index fb20e437d7..1e912fd911 100644 --- a/python/cuml/cuml/metrics/cluster/adjusted_rand_index.pyx +++ b/python/cuml/cuml/metrics/cluster/adjusted_rand_index.pyx @@ -2,13 +2,9 @@ # SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - -# distutils: language = c++ - import cupy as cp from pylibraft.common.handle import Handle -import cuml.internals from cuml.common import input_to_cuml_array from libc.stdint cimport uintptr_t @@ -23,7 +19,6 @@ cdef extern from "cuml/metrics/metrics.hpp" namespace "ML::Metrics" nogil: int n) except + -@cuml.internals.api_return_any() def adjusted_rand_score(labels_true, labels_pred, handle=None, convert_dtype=True) -> float: """ diff --git a/python/cuml/cuml/metrics/cluster/completeness_score.pyx b/python/cuml/cuml/metrics/cluster/completeness_score.pyx index 9699df3baa..a5eeafb16c 100644 --- a/python/cuml/cuml/metrics/cluster/completeness_score.pyx +++ b/python/cuml/cuml/metrics/cluster/completeness_score.pyx @@ -2,11 +2,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - -# distutils: language = c++ - -import cuml.internals - from libc.stdint cimport uintptr_t from pylibraft.common.handle cimport handle_t @@ -22,7 +17,6 @@ cdef extern from "cuml/metrics/metrics.hpp" namespace "ML::Metrics" nogil: const int upper_class_range) except + -@cuml.internals.api_return_any() def cython_completeness_score(labels_true, labels_pred, handle=None) -> float: """ Completeness metric of a cluster labeling given a ground truth. diff --git a/python/cuml/cuml/metrics/cluster/entropy.pyx b/python/cuml/cuml/metrics/cluster/entropy.pyx index e336f515af..1e444c6c9c 100644 --- a/python/cuml/cuml/metrics/cluster/entropy.pyx +++ b/python/cuml/cuml/metrics/cluster/entropy.pyx @@ -2,17 +2,12 @@ # SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - -# distutils: language = c++ import math -import typing import cupy as cp import numpy as np from pylibraft.common.handle import Handle -import cuml.internals -from cuml.common import CumlArray from cuml.internals.input_utils import input_to_cupy_array from libc.stdint cimport uintptr_t @@ -27,22 +22,6 @@ cdef extern from "cuml/metrics/metrics.hpp" namespace "ML::Metrics" nogil: const int upper_class_range) except + -@cuml.internals.api_return_array() -def _prepare_cluster_input(cluster) -> typing.Tuple[CumlArray, int, int, int]: - """Helper function to avoid code duplication for clustering metrics.""" - cluster_m, n_rows, _, _ = input_to_cupy_array( - cluster, - check_dtype=np.int32, - check_cols=1 - ) - - lower_class_range = cp.min(cluster_m).item() - upper_class_range = cp.max(cluster_m).item() - - return cluster_m, n_rows, lower_class_range, upper_class_range - - -@cuml.internals.api_return_any() def cython_entropy(clustering, base=None, handle=None) -> float: """ Computes the entropy of a distribution for given probability values. @@ -72,10 +51,15 @@ def cython_entropy(clustering, base=None, handle=None) -> float: handle = Handle() if handle is None else handle cdef handle_t *handle_ = handle.getHandle() - (clustering, n_rows, - lower_class_range, upper_class_range) = _prepare_cluster_input(clustering) + clustering, n_rows, _, _ = input_to_cupy_array( + clustering, + check_dtype=np.int32, + check_cols=1 + ) + lower_class_range = cp.min(clustering).item() + upper_class_range = cp.max(clustering).item() - cdef uintptr_t clustering_ptr = clustering.ptr + cdef uintptr_t clustering_ptr = clustering.data.ptr S = entropy(handle_[0], clustering_ptr, diff --git a/python/cuml/cuml/metrics/cluster/homogeneity_score.pyx b/python/cuml/cuml/metrics/cluster/homogeneity_score.pyx index 73f70f5c2f..bcdf23dc43 100644 --- a/python/cuml/cuml/metrics/cluster/homogeneity_score.pyx +++ b/python/cuml/cuml/metrics/cluster/homogeneity_score.pyx @@ -2,11 +2,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - -# distutils: language = c++ - -import cuml.internals - from libc.stdint cimport uintptr_t from pylibraft.common.handle cimport handle_t @@ -22,7 +17,6 @@ cdef extern from "cuml/metrics/metrics.hpp" namespace "ML::Metrics" nogil: const int upper_class_range) except + -@cuml.internals.api_return_any() def cython_homogeneity_score(labels_true, labels_pred, handle=None) -> float: """ Computes the homogeneity metric of a cluster labeling given a ground truth. diff --git a/python/cuml/cuml/metrics/cluster/mutual_info_score.pyx b/python/cuml/cuml/metrics/cluster/mutual_info_score.pyx index 57fb0525c3..f438d69088 100644 --- a/python/cuml/cuml/metrics/cluster/mutual_info_score.pyx +++ b/python/cuml/cuml/metrics/cluster/mutual_info_score.pyx @@ -2,11 +2,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - -# distutils: language = c++ - -import cuml.internals - from libc.stdint cimport uintptr_t from pylibraft.common.handle cimport handle_t @@ -24,7 +19,6 @@ cdef extern from "cuml/metrics/metrics.hpp" namespace "ML::Metrics" nogil: const int upper_class_range) except + -@cuml.internals.api_return_any() def cython_mutual_info_score(labels_true, labels_pred, handle=None) -> float: """ Computes the Mutual Information between two clusterings. diff --git a/python/cuml/cuml/metrics/cluster/silhouette_score.pyx b/python/cuml/cuml/metrics/cluster/silhouette_score.pyx index 7e46e55b8b..5b6fcb68cc 100644 --- a/python/cuml/cuml/metrics/cluster/silhouette_score.pyx +++ b/python/cuml/cuml/metrics/cluster/silhouette_score.pyx @@ -2,23 +2,19 @@ # SPDX-FileCopyrightText: Copyright (c) 2021-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - import cupy as cp import numpy as np - -from libc.stdint cimport uintptr_t +from pylibraft.common.handle import Handle from cuml.common import input_to_cuml_array from cuml.metrics.pairwise_distances import _determine_metric +from cuml.prims.label.classlabels import check_labels, make_monotonic +from libc.stdint cimport uintptr_t from pylibraft.common.handle cimport handle_t -from pylibraft.common.handle import Handle - from cuml.metrics.distance_type cimport DistanceType -from cuml.prims.label.classlabels import check_labels, make_monotonic - cdef extern from "cuml/metrics/metrics.hpp" namespace "ML::Metrics::Batched" nogil: float silhouette_score( diff --git a/python/cuml/cuml/metrics/cluster/utils.py b/python/cuml/cuml/metrics/cluster/utils.py index a709cc58a6..fdddaa1005 100644 --- a/python/cuml/cuml/metrics/cluster/utils.py +++ b/python/cuml/cuml/metrics/cluster/utils.py @@ -2,16 +2,13 @@ # SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - import cupy as cp -import cuml.internals from cuml.common import input_to_cuml_array from cuml.metrics.utils import sorted_unique_labels from cuml.prims.label import make_monotonic -@cuml.internals.api_return_array(get_output_type=True) def prepare_cluster_metric_inputs(labels_true, labels_pred): """Helper function to avoid code duplication for homogeneity score, mutual info score and completeness score. diff --git a/python/cuml/cuml/metrics/cluster/v_measure.pyx b/python/cuml/cuml/metrics/cluster/v_measure.pyx index 3cc8ca8467..7b85d9e636 100644 --- a/python/cuml/cuml/metrics/cluster/v_measure.pyx +++ b/python/cuml/cuml/metrics/cluster/v_measure.pyx @@ -2,11 +2,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2022-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - -# distutils: language = c++ - -import cuml.internals - from libc.stdint cimport uintptr_t from pylibraft.common.handle cimport handle_t @@ -25,7 +20,6 @@ cdef extern from "cuml/metrics/metrics.hpp" namespace "ML::Metrics" nogil: const double beta) except + -@cuml.internals.api_return_any() def cython_v_measure(labels_true, labels_pred, beta=1.0, handle=None) -> float: """ V-measure metric of a cluster labeling given a ground truth. diff --git a/python/cuml/cuml/metrics/confusion_matrix.py b/python/cuml/cuml/metrics/confusion_matrix.py index 17e84411ee..9e4a5934b3 100644 --- a/python/cuml/cuml/metrics/confusion_matrix.py +++ b/python/cuml/cuml/metrics/confusion_matrix.py @@ -2,12 +2,10 @@ # SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - import cupy as cp import cupyx import numpy as np -import cuml.internals from cuml.common import input_to_cuml_array, using_output_type from cuml.internals.array import CumlArray from cuml.internals.input_utils import input_to_cupy_array @@ -15,7 +13,6 @@ from cuml.prims.label import make_monotonic -@cuml.internals.api_return_any() def confusion_matrix( y_true, y_pred, diff --git a/python/cuml/cuml/metrics/hinge_loss.py b/python/cuml/cuml/metrics/hinge_loss.py index 8f416fb5d3..19a4d0f822 100644 --- a/python/cuml/cuml/metrics/hinge_loss.py +++ b/python/cuml/cuml/metrics/hinge_loss.py @@ -2,16 +2,13 @@ # SPDX-FileCopyrightText: Copyright (c) 2021-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - import cudf import cupy as cp -import cuml.internals from cuml.internals.input_utils import determine_array_type from cuml.preprocessing import LabelBinarizer, LabelEncoder -@cuml.internals.api_return_any() def hinge_loss( y_true, pred_decision, labels=None, sample_weights=None ) -> float: diff --git a/python/cuml/cuml/metrics/kl_divergence.pyx b/python/cuml/cuml/metrics/kl_divergence.pyx index 25c1dd4e92..3e9d0b320f 100644 --- a/python/cuml/cuml/metrics/kl_divergence.pyx +++ b/python/cuml/cuml/metrics/kl_divergence.pyx @@ -2,11 +2,9 @@ # SPDX-FileCopyrightText: Copyright (c) 2021-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - import numpy as np from pylibraft.common.handle import Handle -import cuml.internals from cuml.common import input_to_cuml_array from libc.stdint cimport uintptr_t @@ -26,7 +24,6 @@ cdef extern from "cuml/metrics/metrics.hpp" namespace "ML::Metrics" nogil: int n) except + -@cuml.internals.api_return_any() def kl_divergence(P, Q, handle=None, convert_dtype=True): """ Calculates the "Kullback-Leibler" Divergence diff --git a/python/cuml/cuml/metrics/pairwise_distances.pyx b/python/cuml/cuml/metrics/pairwise_distances.pyx index ee2b2d36dd..8e457bee6b 100644 --- a/python/cuml/cuml/metrics/pairwise_distances.pyx +++ b/python/cuml/cuml/metrics/pairwise_distances.pyx @@ -2,9 +2,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - -# distutils: language = c++ - import warnings from libc.stdint cimport uintptr_t @@ -250,7 +247,7 @@ def nan_euclidean_distances( return distances -@cuml.internals.api_return_array(get_output_type=True) +@cuml.internals.reflect def pairwise_distances(X, Y=None, metric="euclidean", handle=None, convert_dtype=True, metric_arg=2, **kwds): """ @@ -441,7 +438,7 @@ def pairwise_distances(X, Y=None, metric="euclidean", handle=None, return dest_m -@cuml.internals.api_return_array(get_output_type=True) +@cuml.internals.reflect def sparse_pairwise_distances(X, Y=None, metric="euclidean", handle=None, convert_dtype=True, metric_arg=2, **kwds): """ diff --git a/python/cuml/cuml/metrics/pairwise_kernels.py b/python/cuml/cuml/metrics/pairwise_kernels.py index c460db4e5f..28897dbaea 100644 --- a/python/cuml/cuml/metrics/pairwise_kernels.py +++ b/python/cuml/cuml/metrics/pairwise_kernels.py @@ -177,7 +177,7 @@ def evaluate_pairwise_kernels(X, Y, K): return K -@cuml.internals.api_return_array(get_output_type=True) +@cuml.internals.reflect def pairwise_kernels( X, Y=None, diff --git a/python/cuml/cuml/metrics/regression.py b/python/cuml/cuml/metrics/regression.py index 6c71998922..cb57b44e1b 100644 --- a/python/cuml/cuml/metrics/regression.py +++ b/python/cuml/cuml/metrics/regression.py @@ -2,11 +2,9 @@ # SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - import cupy as cp import numpy as np -import cuml.internals from cuml.internals.input_utils import input_to_cupy_array @@ -63,7 +61,6 @@ def _normalize_regression_metric_args( return y_true, y_pred, sample_weight, multioutput -@cuml.internals.api_return_any() def r2_score( y_true, y_pred, @@ -178,7 +175,6 @@ def _mse(y_true, y_pred, sample_weight, multioutput, squared): return float(out if squared else cp.sqrt(out)) -@cuml.internals.api_return_any() def mean_squared_error( y_true, y_pred, @@ -231,7 +227,6 @@ def mean_squared_error( return _mse(y_true, y_pred, sample_weight, multioutput, squared) -@cuml.internals.api_return_any() def mean_absolute_error( y_true, y_pred, sample_weight=None, multioutput="uniform_average" ): @@ -310,7 +305,6 @@ def _weighted_median(X, weights): return X[sorted_idx[median_idx, col_idx], col_idx] -@cuml.internals.api_return_any() def median_absolute_error( y_true, y_pred, @@ -374,7 +368,6 @@ def median_absolute_error( return float(cp.average(output_errors, weights=multioutput)) -@cuml.internals.api_return_any() def mean_squared_log_error( y_true, y_pred, diff --git a/python/cuml/cuml/metrics/trustworthiness.pyx b/python/cuml/cuml/metrics/trustworthiness.pyx index 0bd60d3afe..9259de1038 100644 --- a/python/cuml/cuml/metrics/trustworthiness.pyx +++ b/python/cuml/cuml/metrics/trustworthiness.pyx @@ -2,13 +2,9 @@ # SPDX-FileCopyrightText: Copyright (c) 2018-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - -# distutils: language = c++ - import numpy as np from pylibraft.common.handle import Handle -import cuml.internals from cuml.internals.input_utils import input_to_cuml_array from libc.stdint cimport uintptr_t @@ -39,7 +35,6 @@ def _get_array_ptr(obj): return obj.device_ctypes_pointer.value -@cuml.internals.api_return_any() def trustworthiness(X, X_embedded, handle=None, n_neighbors=5, metric='euclidean', convert_dtype=True, batch_size=512) -> float: diff --git a/python/cuml/cuml/metrics/utils.py b/python/cuml/cuml/metrics/utils.py index fa9c7c9d02..6910ba0ae7 100644 --- a/python/cuml/cuml/metrics/utils.py +++ b/python/cuml/cuml/metrics/utils.py @@ -2,8 +2,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - - import cupy as cp diff --git a/python/cuml/cuml/multiclass/multiclass.py b/python/cuml/cuml/multiclass/multiclass.py index 565a720253..5933299ce4 100644 --- a/python/cuml/cuml/multiclass/multiclass.py +++ b/python/cuml/cuml/multiclass/multiclass.py @@ -3,7 +3,7 @@ # import warnings -import cuml.internals +import cuml from cuml.common import ( input_to_host_array, input_to_host_array_with_sparse_support, @@ -35,11 +35,12 @@ def _get_param_names(cls): return [*super()._get_param_names(), "estimator"] @property - @cuml.internals.api_base_return_array_skipall + @cuml.internals.reflect def classes_(self): return self.multiclass_estimator.classes_ @generate_docstring(y="dense_anydtype") + @cuml.internals.reflect(reset=True) def fit(self, X, y) -> "_BaseMulticlassClassifier": """ Fit a multiclass classifier. @@ -71,6 +72,7 @@ def fit(self, X, y) -> "_BaseMulticlassClassifier": "shape": "(n_samples, 1)", } ) + @cuml.internals.reflect def predict(self, X) -> CumlArray: """ Predict using multi class classifier. @@ -88,6 +90,7 @@ def predict(self, X) -> CumlArray: "shape": "(n_samples, 1)", } ) + @cuml.internals.reflect def decision_function(self, X) -> CumlArray: """ Calculate the decision function. diff --git a/python/cuml/cuml/naive_bayes/naive_bayes.py b/python/cuml/cuml/naive_bayes/naive_bayes.py index 9afb1b6392..5591e0d535 100644 --- a/python/cuml/cuml/naive_bayes/naive_bayes.py +++ b/python/cuml/cuml/naive_bayes/naive_bayes.py @@ -17,6 +17,7 @@ from cuml.internals.base import Base from cuml.internals.input_utils import input_to_cuml_array, input_to_cupy_array from cuml.internals.mixins import ClassifierMixin +from cuml.internals.outputs import reflect from cuml.prims.array import binarize from cuml.prims.label import check_labels, invert_labels, make_monotonic @@ -158,6 +159,7 @@ def _check_X(self, X): "shape": "(n_rows, 1)", }, ) + @reflect def predict(self, X, *, convert_dtype=True) -> CumlArray: """ Perform classification on an array of test vectors X. @@ -201,6 +203,7 @@ def predict(self, X, *, convert_dtype=True) -> CumlArray: "shape": "(n_rows, 1)", }, ) + @reflect def predict_log_proba(self, X, *, convert_dtype=True) -> CumlArray: """ Return log-probability estimates for the test vector X. @@ -258,6 +261,7 @@ def predict_log_proba(self, X, *, convert_dtype=True) -> CumlArray: "shape": "(n_rows, 1)", }, ) + @reflect def predict_proba(self, X) -> CumlArray: """ Return probability estimates for the test vector X. @@ -339,6 +343,7 @@ def __init__( self.fit_called_ = False self.classes_ = None + @reflect(reset=True) def fit(self, X, y, sample_weight=None) -> "GaussianNB": """ Fit Gaussian Naive Bayes classifier according to X, y @@ -366,6 +371,7 @@ def fit(self, X, y, sample_weight=None) -> "GaussianNB": @nvtx.annotate( message="naive_bayes.GaussianNB._partial_fit", domain="cuml_python" ) + @reflect(reset=True) def _partial_fit( self, X, @@ -472,6 +478,7 @@ def var_sparse(X, axis=0): return self + @reflect(reset=True) def partial_fit( self, X, y, classes=None, sample_weight=None ) -> "GaussianNB": @@ -741,6 +748,7 @@ def _update_class_log_prior(self, class_prior=None): self.n_classes_, -math.log(self.n_classes_) ) + @reflect(reset=True) def partial_fit( self, X, y, classes=None, sample_weight=None ) -> "_BaseDiscreteNB": @@ -788,6 +796,7 @@ def partial_fit( message="naive_bayes._BaseDiscreteNB._partial_fit", domain="cuml_python", ) + @reflect(reset=True) def _partial_fit( self, X, y, sample_weight=None, _classes=None, convert_dtype=True ) -> "_BaseDiscreteNB": @@ -844,6 +853,7 @@ def _partial_fit( return self + @reflect(reset=True) def fit(self, X, y, sample_weight=None) -> "_BaseDiscreteNB": """ Fit Naive Bayes classifier according to X, y @@ -1633,6 +1643,7 @@ def _check_X(self, X): raise ValueError("Negative values in data passed to CategoricalNB") return X + @reflect(reset=True) def fit(self, X, y, sample_weight=None) -> "CategoricalNB": """Fit Naive Bayes classifier according to X, y @@ -1658,6 +1669,7 @@ def fit(self, X, y, sample_weight=None) -> "CategoricalNB": """ return super().fit(X, y, sample_weight=sample_weight) + @reflect(reset=True) def partial_fit( self, X, y, classes=None, sample_weight=None ) -> "CategoricalNB": diff --git a/python/cuml/cuml/neighbors/kernel_density.py b/python/cuml/cuml/neighbors/kernel_density.py index e6e0c176e4..043217c8c5 100644 --- a/python/cuml/cuml/neighbors/kernel_density.py +++ b/python/cuml/cuml/neighbors/kernel_density.py @@ -14,6 +14,7 @@ from cuml.internals.base import Base from cuml.internals.input_utils import input_to_cuml_array, input_to_cupy_array from cuml.internals.interop import InteropMixin, UnsupportedOnGPU +from cuml.internals.outputs import reflect from cuml.internals.utils import check_random_seed from cuml.metrics import pairwise_distances from cuml.metrics.pairwise_distances import ( @@ -271,6 +272,7 @@ def __init__( self.metric = metric self.metric_params = metric_params + @reflect(reset=True) def fit( self, X, y=None, sample_weight=None, *, convert_dtype=True ) -> "KernelDensity": @@ -343,6 +345,7 @@ def fit( return self + @reflect def score_samples(self, X, *, convert_dtype=True) -> CumlArray: """Compute the log-likelihood of each sample under the model. @@ -430,6 +433,7 @@ def score_samples(self, X, *, convert_dtype=True) -> CumlArray: return log_probabilities + @reflect(skip=True) def score(self, X, y=None) -> float: """Compute the total log-likelihood under the model. @@ -450,6 +454,7 @@ def score(self, X, y=None) -> float: """ return float(cp.sum(self.score_samples(X).to_output("cupy"))) + @reflect def sample(self, n_samples=1, random_state=None) -> CumlArray: """Generate random samples from the model. diff --git a/python/cuml/cuml/neighbors/kneighbors_classifier.pyx b/python/cuml/cuml/neighbors/kneighbors_classifier.pyx index aedc5e1f6f..2e8c123757 100644 --- a/python/cuml/cuml/neighbors/kneighbors_classifier.pyx +++ b/python/cuml/cuml/neighbors/kneighbors_classifier.pyx @@ -7,13 +7,14 @@ from __future__ import annotations import cupy as cp import numpy as np -import cuml.internals +import cuml from cuml.common import input_to_cuml_array from cuml.common.classification import decode_labels, preprocess_labels from cuml.common.doc_utils import generate_docstring from cuml.internals.array import CumlArray from cuml.internals.interop import UnsupportedOnGPU from cuml.internals.mixins import ClassifierMixin, FMajorInputTagMixin +from cuml.internals.outputs import reflect from cuml.neighbors.nearest_neighbors import NearestNeighbors from cuml.neighbors.weights import compute_weights @@ -173,6 +174,7 @@ class KNeighborsClassifier(ClassifierMixin, self.weights = weights @generate_docstring(convert_dtype_cast='np.float32') + @reflect(reset=True) def fit(self, X, y, *, convert_dtype=True) -> "KNeighborsClassifier": """ Fit a GPU index for k-nearest neighbors classifier model. @@ -205,7 +207,7 @@ class KNeighborsClassifier(ClassifierMixin, 'type': 'dense', 'description': 'Labels predicted', 'shape': '(n_samples, 1)'}) - @cuml.internals.api_base_return_any_skipall + @reflect(skip=True) def predict(self, X, *, convert_dtype=True): """ Use the trained k-nearest neighbors classifier to @@ -277,7 +279,7 @@ class KNeighborsClassifier(ClassifierMixin, 'type': 'dense', 'description': 'Labels probabilities', 'shape': '(n_samples, 1)'}) - @cuml.internals.api_base_return_array() + @reflect def predict_proba(self, X, *, convert_dtype=True) -> CumlArray | list[CumlArray]: """ Use the trained k-nearest neighbors classifier to diff --git a/python/cuml/cuml/neighbors/kneighbors_regressor.pyx b/python/cuml/cuml/neighbors/kneighbors_regressor.pyx index 340a03fd85..4719db56ae 100644 --- a/python/cuml/cuml/neighbors/kneighbors_regressor.pyx +++ b/python/cuml/cuml/neighbors/kneighbors_regressor.pyx @@ -9,6 +9,7 @@ from cuml.common.doc_utils import generate_docstring from cuml.internals.array import CumlArray from cuml.internals.interop import UnsupportedOnGPU, to_cpu, to_gpu from cuml.internals.mixins import FMajorInputTagMixin, RegressorMixin +from cuml.internals.outputs import reflect from cuml.neighbors.nearest_neighbors import NearestNeighbors from cuml.neighbors.weights import compute_weights @@ -31,11 +32,8 @@ cdef extern from "cuml/neighbors/knn.hpp" namespace "ML" nogil: ) except + -class KNeighborsRegressor(RegressorMixin, - FMajorInputTagMixin, - NearestNeighbors): +class KNeighborsRegressor(RegressorMixin, FMajorInputTagMixin, NearestNeighbors): """ - K-Nearest Neighbors Regressor is an instance-based learning technique, that keeps training samples around for prediction, rather than trying to learn a generalizable set of model parameters. @@ -176,6 +174,7 @@ class KNeighborsRegressor(RegressorMixin, self.weights = weights @generate_docstring(convert_dtype_cast='np.float32') + @reflect(reset=True) def fit(self, X, y, *, convert_dtype=True) -> "KNeighborsRegressor": """ Fit a GPU index for k-nearest neighbors regression model. @@ -202,6 +201,7 @@ class KNeighborsRegressor(RegressorMixin, 'type': 'dense', 'description': 'Predicted values', 'shape': '(n_samples, n_features)'}) + @reflect def predict(self, X, *, convert_dtype=True) -> CumlArray: """ Use the trained k-nearest neighbors regression model to diff --git a/python/cuml/cuml/neighbors/nearest_neighbors.pyx b/python/cuml/cuml/neighbors/nearest_neighbors.pyx index f5f3e3e147..f16024e75a 100644 --- a/python/cuml/cuml/neighbors/nearest_neighbors.pyx +++ b/python/cuml/cuml/neighbors/nearest_neighbors.pyx @@ -10,7 +10,7 @@ import cupyx import numpy as np import scipy.sparse -import cuml.internals +import cuml from cuml.common.array_descriptor import CumlArrayDescriptor from cuml.common.doc_utils import generate_docstring, insert_into_docstring from cuml.common.sparse_utils import is_dense, is_sparse @@ -20,7 +20,7 @@ from cuml.internals.base import Base from cuml.internals.input_utils import input_to_cuml_array from cuml.internals.interop import InteropMixin, UnsupportedOnGPU, to_gpu from cuml.internals.mixins import CMajorInputTagMixin, SparseInputTagMixin -from cuml.internals.outputs import using_output_type +from cuml.internals.outputs import reflect, using_output_type from libc.stdint cimport int64_t, uint32_t, uintptr_t from libcpp cimport bool @@ -691,6 +691,7 @@ class NearestNeighbors(Base, ) @generate_docstring(X='dense_sparse') + @reflect(reset=True) def fit(self, X, y=None, *, convert_dtype=True) -> "NearestNeighbors": """ Fit GPU index for performing nearest neighbor queries. @@ -757,6 +758,7 @@ class NearestNeighbors(Base, return_values=[('dense', '(n_samples, n_features)'), ('dense', '(n_samples, n_features)')]) + @reflect def kneighbors( self, X=None, @@ -1017,6 +1019,7 @@ class NearestNeighbors(Base, return distances, indices @insert_into_docstring(parameters=[('dense', '(n_samples, n_features)')]) + @reflect def kneighbors_graph( self, X=None, n_neighbors=None, mode='connectivity' ) -> SparseCumlArray: @@ -1095,7 +1098,7 @@ class NearestNeighbors(Base, return self.metric_params or {} -@cuml.internals.reflect +@reflect def kneighbors_graph(X=None, n_neighbors=5, mode='connectivity', verbose=False, handle=None, algorithm="brute", metric="euclidean", p=2, include_self=False, metric_params=None): diff --git a/python/cuml/cuml/preprocessing/label.py b/python/cuml/cuml/preprocessing/label.py index 2da54f1e68..8b888f3c4f 100644 --- a/python/cuml/cuml/preprocessing/label.py +++ b/python/cuml/cuml/preprocessing/label.py @@ -14,7 +14,7 @@ from cuml.prims.label import check_labels, invert_labels, make_monotonic -@cuml.internals.api_return_array() +@cuml.internals.reflect def label_binarize( y, classes, neg_label=0, pos_label=1, sparse_output=False ) -> SparseCumlArray: @@ -40,9 +40,6 @@ def label_binarize( row_ind = cp.arange(0, labels.shape[0], 1, dtype=y.dtype) col_ind, _ = make_monotonic(labels, classes, copy=True) - # Convert from CumlArray to cupy - col_ind = cp.asarray(col_ind) - val = cp.full(row_ind.shape[0], pos_label, dtype=y.dtype) sp = cupyx.scipy.sparse.coo_matrix( @@ -169,6 +166,7 @@ def __init__( self.sparse_output = sparse_output self.classes_ = None + @cuml.internals.reflect(reset=True) def fit(self, y) -> "LabelBinarizer": """ Fit label binarizer @@ -200,6 +198,7 @@ def fit(self, y) -> "LabelBinarizer": return self + @cuml.internals.reflect def fit_transform(self, y) -> SparseCumlArray: """ Fit label binarizer and transform multi-class labels to their @@ -216,6 +215,7 @@ def fit_transform(self, y) -> SparseCumlArray: """ return self.fit(y).transform(y) + @cuml.internals.reflect def transform(self, y) -> SparseCumlArray: """ Transform multi-class labels to their dummy-encoded representation @@ -237,6 +237,7 @@ def transform(self, y) -> SparseCumlArray: sparse_output=self.sparse_output, ) + @cuml.internals.reflect def inverse_transform(self, y, *, threshold=None) -> CumlArray: """ Transform binary labels back to original multi-class labels @@ -263,7 +264,7 @@ def inverse_transform(self, y, *, threshold=None) -> CumlArray: y.dtype ) - return invert_labels(y_mapped, self.classes_) + return CumlArray(data=invert_labels(y_mapped, self.classes_)) @classmethod def _get_param_names(cls): diff --git a/python/cuml/cuml/prims/label/classlabels.py b/python/cuml/cuml/prims/label/classlabels.py index 273cc8f251..172009333d 100644 --- a/python/cuml/cuml/prims/label/classlabels.py +++ b/python/cuml/cuml/prims/label/classlabels.py @@ -2,15 +2,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - import math -import typing import cupy as cp -import cuml.internals from cuml.common.kernel_utils import cuda_kernel_factory -from cuml.internals.array import CumlArray from cuml.internals.input_utils import input_to_cupy_array map_kernel_str = r""" @@ -99,10 +95,7 @@ def _validate_kernel(dtype): ) -@cuml.internals.api_return_array(input_arg="labels", get_output_type=True) -def make_monotonic( - labels, classes=None, copy=False -) -> typing.Tuple[CumlArray, CumlArray]: +def make_monotonic(labels, classes=None, copy=False): """ Takes a set of labels that might not be drawn from the set [0, n-1] and renumbers them to be drawn that @@ -148,7 +141,6 @@ def make_monotonic( return labels, classes -@cuml.internals.api_return_any() def check_labels(labels, classes) -> bool: """ Validates that a set of labels is drawn from the unique @@ -193,8 +185,7 @@ def check_labels(labels, classes) -> bool: return valid[0] == 1 -@cuml.internals.api_return_array(input_arg="labels", get_output_type=True) -def invert_labels(labels, classes, copy=False) -> CumlArray: +def invert_labels(labels, classes, copy=False): """ Takes a set of labels that have been mapped to be drawn from a monotonically increasing set and inverts them to diff --git a/python/cuml/cuml/prims/stats/covariance.py b/python/cuml/cuml/prims/stats/covariance.py index 5d47b4ad17..de9081fba2 100644 --- a/python/cuml/cuml/prims/stats/covariance.py +++ b/python/cuml/cuml/prims/stats/covariance.py @@ -8,7 +8,6 @@ import cupy as cp import cupyx -import cuml.internals from cuml.common.kernel_utils import cuda_kernel_factory cov_kernel_str = r""" @@ -95,7 +94,6 @@ def _copy_kernel(dtype): return cuda_kernel_factory(copy_kernel, (dtype,), "copy_kernel") -@cuml.internals.api_return_any() def cov(x, y, mean_x=None, mean_y=None, return_gram=False, return_mean=False): """ Computes a covariance between two matrices using @@ -222,7 +220,6 @@ def cov(x, y, mean_x=None, mean_y=None, return_gram=False, return_mean=False): return cov_result, gram_matrix, mean_x, mean_y -@cuml.internals.api_return_any() def _cov_sparse(x, return_gram=False, return_mean=False): """ Computes the mean and the covariance of matrix X of diff --git a/python/cuml/cuml/random_projection/random_projection.py b/python/cuml/cuml/random_projection/random_projection.py index 4d352a4fdf..b6ea3a7de5 100644 --- a/python/cuml/cuml/random_projection/random_projection.py +++ b/python/cuml/cuml/random_projection/random_projection.py @@ -5,7 +5,6 @@ import numpy as np import scipy.sparse as sp -import cuml from cuml.common.array_descriptor import CumlArrayDescriptor from cuml.common.doc_utils import generate_docstring from cuml.internals.array import CumlArray @@ -13,6 +12,7 @@ from cuml.internals.base import Base from cuml.internals.input_utils import input_to_cuml_array from cuml.internals.mixins import SparseInputTagMixin +from cuml.internals.outputs import reflect from cuml.internals.utils import check_random_seed @@ -81,7 +81,7 @@ def _gen_random_matrix(self, n_components, n_features, dtype): raise NotImplementedError @generate_docstring() - @cuml.internals.api_base_return_any() + @reflect(reset=True) def fit(self, X, y=None, *, convert_dtype=True): """Generate a random projection matrix.""" n_samples, n_features = X.shape @@ -119,6 +119,7 @@ def fit(self, X, y=None, *, convert_dtype=True): return self @generate_docstring() + @reflect def transform(self, X, *, convert_dtype=True) -> CumlArray: """Project the data by taking the matrix product with the random matrix.""" # Coerce X to a cupy array or cupyx sparse matrix @@ -154,6 +155,7 @@ def transform(self, X, *, convert_dtype=True) -> CumlArray: return CumlArray(data=out, index=index) @generate_docstring() + @reflect def fit_transform(self, X, y=None, *, convert_dtype=True) -> CumlArray: """Fit to data, then transform it.""" return self.fit(X, convert_dtype=convert_dtype).transform( diff --git a/python/cuml/cuml/solvers/cd.pyx b/python/cuml/cuml/solvers/cd.pyx index 5db6c67bff..eceeae6a95 100644 --- a/python/cuml/cuml/solvers/cd.pyx +++ b/python/cuml/cuml/solvers/cd.pyx @@ -9,6 +9,7 @@ from cuml.common.doc_utils import generate_docstring from cuml.internals.base import Base from cuml.internals.input_utils import input_to_cuml_array from cuml.internals.mixins import FMajorInputTagMixin +from cuml.internals.outputs import reflect from libc.stdint cimport uintptr_t from libcpp cimport bool @@ -347,6 +348,7 @@ class CD(Base, FMajorInputTagMixin): self.shuffle = shuffle @generate_docstring() + @reflect(reset=True) def fit(self, X, y, convert_dtype=True, sample_weight=None) -> "CD": """ Fit the model with X and y. @@ -379,6 +381,7 @@ class CD(Base, FMajorInputTagMixin): 'type': 'dense', 'description': 'Predicted values', 'shape': '(n_samples, 1)'}) + @reflect def predict(self, X, convert_dtype=True) -> CumlArray: """ Predicts the y for X. diff --git a/python/cuml/cuml/solvers/qn.pyx b/python/cuml/cuml/solvers/qn.pyx index ccc548ef4f..ba052212c1 100644 --- a/python/cuml/cuml/solvers/qn.pyx +++ b/python/cuml/cuml/solvers/qn.pyx @@ -12,6 +12,7 @@ from cuml.common.sparse_utils import is_sparse from cuml.internals.array import CumlArray from cuml.internals.array_sparse import SparseCumlArray from cuml.internals.base import Base +from cuml.internals.outputs import reflect from cuml.metrics import accuracy_score from libc.stdint cimport uintptr_t @@ -526,6 +527,7 @@ class QN(Base): self.penalty_normalized = penalty_normalized @generate_docstring(X="dense_sparse") + @reflect(reset=True) def fit(self, X, y, sample_weight=None, convert_dtype=True) -> "QN": """ Fit the model with X and y. @@ -572,6 +574,7 @@ class QN(Base): return self @generate_docstring(X="dense_sparse") + @reflect def predict(self, X, *, convert_dtype=True) -> CumlArray: """Predicts the y for X.""" if is_sparse(X): @@ -605,5 +608,6 @@ class QN(Base): return CumlArray(data=out, index=out_index) + @reflect(skip=True) def score(self, X, y): return accuracy_score(y, self.predict(X)) diff --git a/python/cuml/cuml/solvers/sgd.pyx b/python/cuml/cuml/solvers/sgd.pyx index 29fef8f178..b66c2f98b3 100644 --- a/python/cuml/cuml/solvers/sgd.pyx +++ b/python/cuml/cuml/solvers/sgd.pyx @@ -9,6 +9,7 @@ from cuml.common.doc_utils import generate_docstring from cuml.internals.array import CumlArray from cuml.internals.base import Base from cuml.internals.mixins import FMajorInputTagMixin +from cuml.internals.outputs import reflect from libc.stdint cimport uintptr_t from libcpp cimport bool @@ -401,6 +402,7 @@ class SGD(Base, FMajorInputTagMixin): self.n_iter_no_change = n_iter_no_change @generate_docstring() + @reflect(reset=True) def fit(self, X, y, *, convert_dtype=True) -> "SGD": """ Fit the model with X and y. @@ -437,6 +439,7 @@ class SGD(Base, FMajorInputTagMixin): "shape": "(n_samples,)" } ) + @reflect def predict(self, X, *, convert_dtype=True) -> CumlArray: """ Predicts the y for X. diff --git a/python/cuml/cuml/svm/linear_svc.py b/python/cuml/cuml/svm/linear_svc.py index 45f2b48de6..19ade77d62 100644 --- a/python/cuml/cuml/svm/linear_svc.py +++ b/python/cuml/cuml/svm/linear_svc.py @@ -23,6 +23,7 @@ to_gpu, ) from cuml.internals.mixins import ClassifierMixin +from cuml.internals.outputs import reflect from cuml.linear_model.base import LinearClassifierMixin __all__ = ("LinearSVC",) @@ -243,6 +244,7 @@ def __init__( self.multi_class = multi_class @generate_docstring() + @reflect(reset=True) def fit( self, X, y, sample_weight=None, *, convert_dtype=True ) -> "LinearSVC": @@ -298,7 +300,7 @@ def fit( "shape": "(n_samples,)", }, ) - @cuml.internals.api_base_return_any_skipall + @reflect(skip=True) def predict(self, X, *, convert_dtype=True): """Predict class labels for samples in X.""" if self.probability: @@ -326,6 +328,7 @@ def predict(self, X, *, convert_dtype=True): "shape": "(n_samples, n_classes)", }, ) + @reflect def predict_proba(self, X, *, convert_dtype=True) -> CumlArray: """Compute probabilities of possible outcomes for samples in X. @@ -357,6 +360,7 @@ def predict_proba(self, X, *, convert_dtype=True) -> CumlArray: "shape": "(n_samples, n_classes)", }, ) + @reflect def predict_log_proba(self, X, *, convert_dtype=True) -> CumlArray: """Compute log probabilities of possible outcomes for samples in X. diff --git a/python/cuml/cuml/svm/linear_svr.py b/python/cuml/cuml/svm/linear_svr.py index 62798eef57..b89bfefd81 100644 --- a/python/cuml/cuml/svm/linear_svr.py +++ b/python/cuml/cuml/svm/linear_svr.py @@ -15,6 +15,7 @@ to_gpu, ) from cuml.internals.mixins import RegressorMixin +from cuml.internals.outputs import reflect from cuml.linear_model.base import LinearPredictMixin __all__ = ["LinearSVR"] @@ -209,6 +210,7 @@ def __init__( self.lbfgs_memory = lbfgs_memory @generate_docstring() + @reflect(reset=True) def fit( self, X, y, sample_weight=None, *, convert_dtype=True ) -> "LinearSVR": diff --git a/python/cuml/cuml/svm/svc.py b/python/cuml/cuml/svm/svc.py index 74c2fb859d..4b7966b5f5 100644 --- a/python/cuml/cuml/svm/svc.py +++ b/python/cuml/cuml/svm/svc.py @@ -4,7 +4,6 @@ import cupy as cp import numpy as np -import cuml.internals from cuml.common.classification import ( decode_labels, preprocess_labels, @@ -23,6 +22,7 @@ from cuml.internals.interop import UnsupportedOnCPU, UnsupportedOnGPU from cuml.internals.logger import warn from cuml.internals.mixins import ClassifierMixin +from cuml.internals.outputs import exit_internal_api, reflect from cuml.internals.utils import check_random_seed from cuml.multiclass import OneVsOneClassifier, OneVsRestClassifier from cuml.svm.svm_base import SVMBase @@ -298,7 +298,7 @@ def __init__( self.decision_function_shape = decision_function_shape @property - @cuml.internals.api_base_return_array_skipall + @reflect def support_(self): if hasattr(self, "_multiclass"): estimators = self._multiclass.multiclass_estimator.estimators_ @@ -313,7 +313,7 @@ def support_(self, value): self._support_ = value @property - @cuml.internals.api_base_return_array_skipall + @reflect def intercept_(self): if hasattr(self, "_multiclass"): estimators = self._multiclass.multiclass_estimator.estimators_ @@ -327,7 +327,7 @@ def intercept_(self): def intercept_(self, value): self._intercept_ = value - def _fit_multiclass(self, X, y, sample_weight) -> "SVC": + def _fit_multiclass(self, X, y, sample_weight): if sample_weight is not None: warn( "Sample weights are currently ignored for multi class classification" @@ -380,7 +380,7 @@ def _fit_multiclass(self, X, y, sample_weight) -> "SVC": ) return self - def _fit_proba(self, X, y, sample_weight) -> "SVC": + def _fit_proba(self, X, y, sample_weight): from sklearn.calibration import CalibratedClassifierCV from sklearn.model_selection import StratifiedKFold @@ -407,7 +407,7 @@ def _fit_proba(self, X, y, sample_weight) -> "SVC": ) cccv = CalibratedClassifierCV(SVC(**params), cv=cv, ensemble=False) - with cuml.internals.exit_internal_api(): + with exit_internal_api(): cccv.fit(X, y, sample_weight=sample_weight) cal_clf = cccv.calibrated_classifiers_[0] @@ -439,6 +439,7 @@ def _fit_proba(self, X, y, sample_weight) -> "SVC": return self @generate_docstring(y="dense_anydtype") + @reflect(reset=True) def fit(self, X, y, sample_weight=None, *, convert_dtype=True) -> "SVC": """ Fit the model with X and y. @@ -502,7 +503,7 @@ def fit(self, X, y, sample_weight=None, *, convert_dtype=True) -> "SVC": "shape": "(n_samples, 1)", } ) - @cuml.internals.api_base_return_any_skipall + @reflect(skip=True) def predict(self, X, *, convert_dtype=True): """ Predicts the class labels for X. The returned y values are the class @@ -517,7 +518,7 @@ def predict(self, X, *, convert_dtype=True): res = self.decision_function(X, convert_dtype=convert_dtype) inds = (res.to_output("cupy") >= 0).view(cp.int8) - with cuml.internals.exit_internal_api(): + with exit_internal_api(): output_type = self._get_output_type(X) return decode_labels(inds, self.classes_, output_type=output_type) @@ -530,6 +531,7 @@ def predict(self, X, *, convert_dtype=True): "shape": "(n_samples, n_classes)", }, ) + @reflect def predict_proba(self, X, *, log=False) -> CumlArray: """ Predicts the class probabilities for X. @@ -586,6 +588,7 @@ def predict_proba(self, X, *, log=False) -> CumlArray: "shape": "(n_samples, n_classes)", } ) + @reflect def predict_log_proba(self, X) -> CumlArray: """ Predicts the log probabilities for X (returns log(predict_proba(x)). @@ -603,6 +606,7 @@ def predict_log_proba(self, X) -> CumlArray: "shape": "(n_samples, 1)", } ) + @reflect def decision_function(self, X, *, convert_dtype=True) -> CumlArray: """ Calculates the decision function values for X. diff --git a/python/cuml/cuml/svm/svm_base.pyx b/python/cuml/cuml/svm/svm_base.pyx index 33fc453f31..273d7ada81 100644 --- a/python/cuml/cuml/svm/svm_base.pyx +++ b/python/cuml/cuml/svm/svm_base.pyx @@ -316,7 +316,7 @@ class SVMBase(Base, self.nochange_steps = nochange_steps @property - @cuml.internals.api_base_return_array_skipall + @cuml.internals.reflect def coef_(self): if self.kernel != "linear": raise AttributeError("coef_ is only available for linear kernels") diff --git a/python/cuml/cuml/svm/svr.py b/python/cuml/cuml/svm/svr.py index 4d9c5516a9..7ff7eded1e 100644 --- a/python/cuml/cuml/svm/svr.py +++ b/python/cuml/cuml/svm/svr.py @@ -9,6 +9,7 @@ from cuml.internals.array_sparse import SparseCumlArray from cuml.internals.input_utils import input_to_cuml_array from cuml.internals.mixins import RegressorMixin +from cuml.internals.outputs import reflect from cuml.svm.svm_base import SVMBase @@ -145,6 +146,7 @@ class SVR(SVMBase, RegressorMixin): _cpu_class_path = "sklearn.svm.SVR" @generate_docstring() + @reflect(reset=True) def fit(self, X, y, sample_weight=None, *, convert_dtype=True) -> "SVR": """ Fit the model with X and y. @@ -194,6 +196,7 @@ def fit(self, X, y, sample_weight=None, *, convert_dtype=True) -> "SVR": "shape": "(n_samples, 1)", } ) + @reflect def predict(self, X, *, convert_dtype=True) -> CumlArray: """ Predicts the values for X. diff --git a/python/cuml/cuml/tsa/arima.pyx b/python/cuml/cuml/tsa/arima.pyx index 16f6ee1718..2f5776d11b 100644 --- a/python/cuml/cuml/tsa/arima.pyx +++ b/python/cuml/cuml/tsa/arima.pyx @@ -2,30 +2,22 @@ # SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - -# distutils: language = c++ +from typing import Dict, Mapping, Optional, Tuple, Union import numpy as np -import cuml.internals.nvtx as nvtx - -from libc.stdint cimport uintptr_t -from libcpp cimport bool -from libcpp.vector cimport vector - -from typing import Dict, Mapping, Optional, Tuple, Union - -import cuml.internals from cuml.common.array_descriptor import CumlArrayDescriptor +from cuml.internals import logger, nvtx, reflect from cuml.internals.array import CumlArray from cuml.internals.base import Base - -from pylibraft.common.handle cimport handle_t - -import cuml.internals.logger as logger from cuml.internals.input_utils import input_to_cuml_array from cuml.tsa.batched_lbfgs import batched_fmin_lbfgs_b +from libc.stdint cimport uintptr_t +from libcpp cimport bool +from libcpp.vector cimport vector +from pylibraft.common.handle cimport handle_t + cdef extern from "cuml/tsa/arima_common.h" namespace "ML" nogil: cdef cppclass ARIMAParams[DataT]: @@ -387,7 +379,7 @@ class ARIMA(Base): self._initial_calc() - @cuml.internals.api_base_return_any_skipall + @reflect(skip=True) def _initial_calc(self): """ This separates the initial calculation from the initialization to make @@ -441,7 +433,6 @@ class ARIMA(Base): order.p, order.d, order.q, intercept_str, self.batch_size) @nvtx.annotate(message="tsa.arima.ARIMA._ic", domain="cuml_python") - @cuml.internals.api_base_return_any_skipall def _ic(self, ic_type: str): """Wrapper around C++ information_criterion """ @@ -488,16 +479,19 @@ class ARIMA(Base): return ic @property + @reflect def aic(self) -> CumlArray: """Akaike Information Criterion""" return self._ic("aic") @property + @reflect def aicc(self) -> CumlArray: """Corrected Akaike Information Criterion""" return self._ic("aicc") @property + @reflect def bic(self) -> CumlArray: """Bayesian Information Criterion""" return self._ic("bic") @@ -509,7 +503,7 @@ class ARIMA(Base): return (order.p + order.P + order.q + order.Q + order.k + order.n_exog + 1) - @cuml.internals.api_base_return_array(input_arg=None) + @reflect def get_fit_params(self) -> Dict[str, CumlArray]: """Get all the fit parameters. Not to be confused with get_params Note: pack() can be used to get a compact vector of the parameters @@ -585,7 +579,7 @@ class ARIMA(Base): raise NotImplementedError("ARIMA is unable to be cloned via " "`get_params` and `set_params`.") - @cuml.internals.api_base_return_array(input_arg=None) + @reflect(array=None) def predict( self, start=0, @@ -734,7 +728,7 @@ class ARIMA(Base): d_upper) @nvtx.annotate(message="tsa.arima.ARIMA.forecast", domain="cuml_python") - @cuml.internals.api_base_return_array_skipall + @reflect(array=None) def forecast( self, nsteps: int, @@ -780,7 +774,6 @@ class ARIMA(Base): return self.predict(self.n_obs, self.n_obs + nsteps, level, exog) - @cuml.internals.api_base_return_any_skipall def _create_arrays(self): """Create the parameter arrays if non-existing""" cdef ARIMAOrder order = self.order @@ -807,7 +800,7 @@ class ARIMA(Base): @nvtx.annotate(message="tsa.arima.ARIMA._estimate_x0", domain="cuml_python") - @cuml.internals.api_base_return_any_skipall + @reflect(skip=True) def _estimate_x0(self): """Internal method. Estimate initial parameters of the model. """ @@ -827,7 +820,7 @@ class ARIMA(Base): d_exog_ptr, self.batch_size, self.n_obs, order, self.missing) - @cuml.internals.api_base_return_any_skipall + @reflect(skip=True) def fit(self, start_params: Optional[Mapping[str, object]] = None, opt_disp: int = -1, @@ -929,7 +922,7 @@ class ARIMA(Base): return self @nvtx.annotate(message="tsa.arima.ARIMA._loglike", domain="cuml_python") - @cuml.internals.api_base_return_any_skipall + @reflect(skip=True) def _loglike(self, x, trans=True, method="ml", truncate=0, convert_dtype=True): """Compute the batched log-likelihood for the given parameters. @@ -997,7 +990,7 @@ class ARIMA(Base): @nvtx.annotate(message="tsa.arima.ARIMA._loglike_grad", domain="cuml_python") - @cuml.internals.api_base_return_any_skipall + @reflect(skip=True) def _loglike_grad(self, x, h=1e-8, trans=True, method="ml", truncate=0, convert_dtype=True): """Compute the gradient (via finite differencing) of the batched @@ -1073,6 +1066,7 @@ class ARIMA(Base): return grad.to_output("numpy") @property + @reflect(skip=True) def llf(self): """Log-likelihood of a fit model. Shape: (batch_size,) """ @@ -1119,6 +1113,7 @@ class ARIMA(Base): return np.array(vec_loglike, dtype=np.float64) @nvtx.annotate(message="tsa.arima.ARIMA.unpack", domain="cuml_python") + @reflect(skip=True) def unpack(self, x: Union[list, np.ndarray], convert_dtype=True): """Unpack linearized parameter vector `x` into the separate parameter arrays of the model @@ -1148,6 +1143,7 @@ class ARIMA(Base): d_x_ptr) @nvtx.annotate(message="tsa.arima.ARIMA.pack", domain="cuml_python") + @reflect(skip=True) def pack(self) -> np.ndarray: """Pack parameters of the model into a linearized vector `x` @@ -1173,7 +1169,6 @@ class ARIMA(Base): @nvtx.annotate(message="tsa.arima.ARIMA._batched_transform", domain="cuml_python") - @cuml.internals.api_base_return_any_skipall def _batched_transform(self, x, isInv=False): """Applies Jones transform or inverse transform to a parameter vector diff --git a/python/cuml/cuml/tsa/auto_arima.pyx b/python/cuml/cuml/tsa/auto_arima.pyx index 9427a9bc8f..ba24000e92 100644 --- a/python/cuml/cuml/tsa/auto_arima.pyx +++ b/python/cuml/cuml/tsa/auto_arima.pyx @@ -2,34 +2,27 @@ # SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - -# distutils: language = c++ - import itertools import typing -from libc.stdint cimport uintptr_t -from libcpp cimport bool -from libcpp.vector cimport vector - import cupy as cp import numpy as np +from pylibraft.common.handle import Handle -import cuml.internals +from cuml.common import input_to_cuml_array, using_output_type from cuml.common.array_descriptor import CumlArrayDescriptor -from cuml.internals import logger +from cuml.internals import logger, reflect from cuml.internals.array import CumlArray from cuml.internals.base import Base - -from pylibraft.common.handle cimport handle_t - -from pylibraft.common.handle import Handle - -from cuml.common import input_to_cuml_array, using_output_type from cuml.tsa.arima import ARIMA from cuml.tsa.seasonality import seas_test from cuml.tsa.stationarity import kpss_test +from libc.stdint cimport uintptr_t +from libcpp cimport bool +from libcpp.vector cimport vector +from pylibraft.common.handle cimport handle_t + # TODO: # - Box-Cox transformations? (parameter lambda) # - Would a "one-fits-all" method be useful? @@ -195,7 +188,7 @@ class AutoARIMA(Base): self._initial_calc() - @cuml.internals.api_base_return_any_skipall + @reflect(skip=True) def _initial_calc(self): cdef uintptr_t d_y_ptr = self.d_y.ptr cdef handle_t* handle_ = self.handle.getHandle() @@ -208,7 +201,7 @@ class AutoARIMA(Base): raise ValueError( "Missing observations are not supported in AutoARIMA yet") - @cuml.internals.api_return_any() + @reflect(skip=True) def search(self, s=None, d=range(3), @@ -425,7 +418,7 @@ class AutoARIMA(Base): self.id_to_model, self.id_to_pos = _build_division_map(id_tracker, self.batch_size) - @cuml.internals.api_base_return_any_skipall + @reflect(skip=True) def fit(self, h: float = 1e-8, maxiter: int = 1000, @@ -452,7 +445,7 @@ class AutoARIMA(Base): logger.debug("Fitting {} ({})".format(model, method)) model.fit(h=h, maxiter=maxiter, method=method, truncate=truncate) - @cuml.internals.api_base_return_array_skipall + @reflect(array=None) def predict( self, start=0, @@ -512,7 +505,7 @@ class AutoARIMA(Base): else: return y_p, lower, upper - @cuml.internals.api_base_return_array_skipall + @reflect(array=None) def forecast(self, nsteps: int, level=None) -> typing.Union[CumlArray, @@ -542,6 +535,7 @@ class AutoARIMA(Base): """ return self.predict(self.n_obs, self.n_obs + nsteps, level) + @reflect(skip=True) def summary(self): """Display a quick summary of the models selected by `search` """ diff --git a/python/cuml/cuml/tsa/holtwinters.pyx b/python/cuml/cuml/tsa/holtwinters.pyx index 2dab0e653f..75b5055abd 100644 --- a/python/cuml/cuml/tsa/holtwinters.pyx +++ b/python/cuml/cuml/tsa/holtwinters.pyx @@ -1,22 +1,18 @@ # SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - -# distutils: language = c++ - import cudf import cupy as cp import numpy as np -from libc.stdint cimport uintptr_t - -import cuml.internals from cuml.common import using_output_type from cuml.common.array_descriptor import CumlArrayDescriptor from cuml.internals.array import CumlArray from cuml.internals.base import Base from cuml.internals.input_utils import input_to_cupy_array +from cuml.internals.outputs import reflect +from libc.stdint cimport uintptr_t from pylibraft.common.handle cimport handle_t @@ -264,9 +260,9 @@ class ExponentialSmoothing(Base): raise ValueError(err_mess + str(d2)) else: raise ValueError("Data input must have 1 or 2 dimensions.") - return mod_ts_input + return CumlArray(data=mod_ts_input) - @cuml.internals.api_base_return_any_skipall + @reflect(skip=True) def fit(self) -> "ExponentialSmoothing": """ Perform fitting on the given `endog` dataset. @@ -349,9 +345,9 @@ class ExponentialSmoothing(Base): self.handle.sync() self.fit_executed_flag = True - del X_m return self + @reflect(skip=True) def forecast(self, h=1, index=None): """ Forecasts future points based on the fitted model. @@ -437,6 +433,7 @@ class ExponentialSmoothing(Base): else: raise ValueError("Fit() the model before forecast()") + @reflect(skip=True) def score(self, index=None): """ Returns the score of the model. @@ -468,6 +465,7 @@ class ExponentialSmoothing(Base): else: raise ValueError("Fit() the model before score()") + @reflect(skip=True) def get_level(self, index=None): """ Returns the level component of the model. @@ -499,6 +497,7 @@ class ExponentialSmoothing(Base): else: raise ValueError("Fit() the model to get level values") + @reflect(skip=True) def get_trend(self, index=None): """ Returns the trend component of the model. @@ -530,6 +529,7 @@ class ExponentialSmoothing(Base): else: raise ValueError("Fit() the model to get trend values") + @reflect(skip=True) def get_season(self, index=None): """ Returns the season component of the model. diff --git a/python/cuml/cuml/tsa/seasonality.py b/python/cuml/cuml/tsa/seasonality.py index 4ff94e906e..f48023e1b8 100644 --- a/python/cuml/cuml/tsa/seasonality.py +++ b/python/cuml/cuml/tsa/seasonality.py @@ -1,12 +1,10 @@ # SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -# - import numpy as np -import cuml.internals from cuml.internals.array import CumlArray from cuml.internals.input_utils import input_to_cuml_array, input_to_host_array +from cuml.internals.outputs import reflect # TODO: #2234 and #2235 @@ -29,7 +27,7 @@ def python_seas_test(y, batch_size, n_obs, s, threshold=0.64): return results -@cuml.internals.api_return_array(input_arg="y", get_output_type=True) +@reflect def seas_test(y, s, handle=None, convert_dtype=True) -> CumlArray: """ Perform Wang, Smith & Hyndman's test to decide whether seasonal diff --git a/python/cuml/cuml/tsa/stationarity.pyx b/python/cuml/cuml/tsa/stationarity.pyx index 2730063893..e578dd8198 100644 --- a/python/cuml/cuml/tsa/stationarity.pyx +++ b/python/cuml/cuml/tsa/stationarity.pyx @@ -1,23 +1,16 @@ # SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -# - -# distutils: language = c++ - import numpy as np +from pylibraft.common.handle import Handle -from libc.stdint cimport uintptr_t -from libcpp cimport bool as boolcpp - -import cuml.internals from cuml.internals.array import CumlArray +from cuml.internals.input_utils import input_to_cuml_array +from cuml.internals.outputs import reflect +from libc.stdint cimport uintptr_t +from libcpp cimport bool as boolcpp from pylibraft.common.handle cimport handle_t -from pylibraft.common.handle import Handle - -from cuml.internals.input_utils import input_to_cuml_array - cdef extern from "cuml/tsa/stationarity.h" namespace "ML" nogil: int cpp_kpss "ML::Stationarity::kpss_test" ( @@ -39,7 +32,7 @@ cdef extern from "cuml/tsa/stationarity.h" namespace "ML" nogil: double pval_threshold) except + -@cuml.internals.api_return_array(input_arg="y", get_output_type=True) +@reflect def kpss_test(y, d=0, D=0, s=0, pval_threshold=0.05, handle=None, convert_dtype=True) -> CumlArray: """ From 65df8e49131f4f37b56870f1cbba8bad47e3b43d Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Tue, 25 Nov 2025 18:14:00 -0600 Subject: [PATCH 20/28] Explicit decorators for MG models --- python/cuml/cuml/decomposition/base_mg.py | 5 ++--- python/cuml/cuml/decomposition/pca_mg.pyx | 5 ++--- python/cuml/cuml/decomposition/tsvd_mg.pyx | 5 ++--- python/cuml/cuml/linear_model/base_mg.pyx | 8 ++------ .../linear_model/linear_regression_mg.pyx | 4 ++-- .../linear_model/logistic_regression_mg.pyx | 4 ++-- python/cuml/cuml/linear_model/ridge_mg.pyx | 4 ++-- .../neighbors/kneighbors_classifier_mg.pyx | 20 +++++++------------ .../neighbors/kneighbors_regressor_mg.pyx | 8 ++------ .../cuml/neighbors/nearest_neighbors_mg.pyx | 18 ++++++----------- .../cuml/preprocessing/onehotencoder_mg.py | 2 ++ .../cuml/preprocessing/ordinalencoder_mg.py | 2 ++ python/cuml/cuml/solvers/cd_mg.pyx | 4 ++-- 13 files changed, 35 insertions(+), 54 deletions(-) diff --git a/python/cuml/cuml/decomposition/base_mg.py b/python/cuml/cuml/decomposition/base_mg.py index cb0c990772..720abc3438 100644 --- a/python/cuml/cuml/decomposition/base_mg.py +++ b/python/cuml/cuml/decomposition/base_mg.py @@ -2,16 +2,15 @@ # SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - import numpy as np import cuml.common.opg_data_utils_mg as opg -import cuml.internals from cuml.common import input_to_cuml_array +from cuml.internals import reflect class BaseDecompositionMG: - @cuml.internals.api_base_return_any_skipall + @reflect(skip=True) def fit(self, X, total_rows, n_cols, partsToRanks, rank, _transform=False): """ Fit function for PCA/TSVD MG. This not meant to be used as diff --git a/python/cuml/cuml/decomposition/pca_mg.pyx b/python/cuml/cuml/decomposition/pca_mg.pyx index 95265de458..ad619a1592 100644 --- a/python/cuml/cuml/decomposition/pca_mg.pyx +++ b/python/cuml/cuml/decomposition/pca_mg.pyx @@ -2,14 +2,13 @@ # SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - import numpy as np import sklearn from packaging.version import Version -import cuml.internals from cuml.decomposition import PCA from cuml.decomposition.base_mg import BaseDecompositionMG +from cuml.internals import reflect from cuml.internals.array import CumlArray from cython.operator cimport dereference as deref @@ -56,7 +55,7 @@ cdef extern from "cuml/decomposition/pca_mg.hpp" namespace "ML::PCA::opg" nogil: class PCAMG(BaseDecompositionMG, PCA): - @cuml.internals.api_base_return_any_skipall + @reflect(skip=True) def _mg_fit(self, X_ptr, n_rows, n_cols, dtype, input_desc_ptr): # Validate and initialize parameters cdef paramsPCAMG params diff --git a/python/cuml/cuml/decomposition/tsvd_mg.pyx b/python/cuml/cuml/decomposition/tsvd_mg.pyx index da7ce70c60..b091760f77 100644 --- a/python/cuml/cuml/decomposition/tsvd_mg.pyx +++ b/python/cuml/cuml/decomposition/tsvd_mg.pyx @@ -2,14 +2,13 @@ # SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - import numpy as np import sklearn from packaging.version import Version -import cuml.internals from cuml.decomposition import TruncatedSVD from cuml.decomposition.base_mg import BaseDecompositionMG +from cuml.internals import reflect from cuml.internals.array import CumlArray from cython.operator cimport dereference as deref @@ -56,7 +55,7 @@ cdef extern from "cuml/decomposition/tsvd_mg.hpp" namespace "ML::TSVD::opg" nogi class TSVDMG(BaseDecompositionMG, TruncatedSVD): - @cuml.internals.api_base_return_any_skipall + @reflect(skip=True) def _mg_fit_transform( self, X_ptr, n_rows, n_cols, dtype, trans_ptr, input_desc_ptr, trans_desc_ptr ): diff --git a/python/cuml/cuml/linear_model/base_mg.pyx b/python/cuml/cuml/linear_model/base_mg.pyx index 2b95bdb9bb..1b9b8cba4d 100644 --- a/python/cuml/cuml/linear_model/base_mg.pyx +++ b/python/cuml/cuml/linear_model/base_mg.pyx @@ -2,14 +2,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # -# distutils: language = c++ - - import numpy as np import cuml.common.opg_data_utils_mg as opg -import cuml.internals from cuml.common.sparse_utils import is_sparse +from cuml.internals import reflect from cuml.internals.array import CumlArray from cuml.internals.array_sparse import SparseCumlArray from cuml.internals.input_utils import input_to_cuml_array @@ -20,8 +17,7 @@ from cuml.common.opg_data_utils_mg cimport * class MGFitMixin(object): - - @cuml.internals.api_base_return_any_skipall + @reflect(skip=True) def fit( self, input_data, diff --git a/python/cuml/cuml/linear_model/linear_regression_mg.pyx b/python/cuml/cuml/linear_model/linear_regression_mg.pyx index 961763e66b..dc7c5b6300 100644 --- a/python/cuml/cuml/linear_model/linear_regression_mg.pyx +++ b/python/cuml/cuml/linear_model/linear_regression_mg.pyx @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 import numpy as np -import cuml.internals +from cuml.internals import reflect from cuml.linear_model.base import check_deprecated_normalize from cuml.linear_model.base_mg import MGFitMixin from cuml.linear_model.linear_regression import Algo, LinearRegression @@ -42,7 +42,7 @@ cdef extern from "cuml/linear_model/ols_mg.hpp" namespace "ML::OLS::opg" nogil: class LinearRegressionMG(MGFitMixin, LinearRegression): - @cuml.internals.api_base_return_any_skipall + @reflect(skip=True) def _fit(self, X, y, coef_ptr, input_desc): check_deprecated_normalize(self) diff --git a/python/cuml/cuml/linear_model/logistic_regression_mg.pyx b/python/cuml/cuml/linear_model/logistic_regression_mg.pyx index 890e274669..3847ca93d8 100644 --- a/python/cuml/cuml/linear_model/logistic_regression_mg.pyx +++ b/python/cuml/cuml/linear_model/logistic_regression_mg.pyx @@ -4,7 +4,7 @@ # import numpy as np -import cuml.internals +from cuml.internals import reflect from cuml.internals.array import CumlArray from cuml.linear_model import LogisticRegression from cuml.linear_model.base_mg import MGFitMixin @@ -144,7 +144,7 @@ class LogisticRegressionMG(MGFitMixin, LogisticRegression): convert_index=self._convert_index, ) - @cuml.internals.api_base_return_any_skipall + @reflect(skip=True) def _fit(self, X, uintptr_t y, uintptr_t coef_ptr, uintptr_t input_desc): cdef handle_t* handle_ = self.handle.getHandle() diff --git a/python/cuml/cuml/linear_model/ridge_mg.pyx b/python/cuml/cuml/linear_model/ridge_mg.pyx index a043330add..858d7787a8 100644 --- a/python/cuml/cuml/linear_model/ridge_mg.pyx +++ b/python/cuml/cuml/linear_model/ridge_mg.pyx @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 import numpy as np -import cuml.internals +from cuml.internals import reflect from cuml.linear_model import Ridge from cuml.linear_model.base import check_deprecated_normalize from cuml.linear_model.base_mg import MGFitMixin @@ -46,7 +46,7 @@ cdef extern from "cuml/linear_model/ridge_mg.hpp" namespace "ML::Ridge::opg" nog class RidgeMG(MGFitMixin, Ridge): - @cuml.internals.api_base_return_any_skipall + @reflect(skip=True) def _fit(self, X, y, coef_ptr, input_desc): check_deprecated_normalize(self) diff --git a/python/cuml/cuml/neighbors/kneighbors_classifier_mg.pyx b/python/cuml/cuml/neighbors/kneighbors_classifier_mg.pyx index 271469a577..39c917d5c4 100644 --- a/python/cuml/cuml/neighbors/kneighbors_classifier_mg.pyx +++ b/python/cuml/cuml/neighbors/kneighbors_classifier_mg.pyx @@ -2,27 +2,21 @@ # SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - -# distutils: language = c++ - import typing -import cuml.internals.logger as logger -from cuml.internals import api_base_return_array_skipall +from cuml.common import input_to_cuml_array +from cuml.internals import logger, reflect from cuml.internals.array import CumlArray from cuml.neighbors.nearest_neighbors_mg import NearestNeighborsMG -from pylibraft.common.handle cimport handle_t - -from cuml.common.opg_data_utils_mg cimport * - -from cuml.common import input_to_cuml_array - from cython.operator cimport dereference as deref from libc.stdint cimport uintptr_t from libc.stdlib cimport free from libcpp cimport bool from libcpp.vector cimport vector +from pylibraft.common.handle cimport handle_t + +from cuml.common.opg_data_utils_mg cimport * cdef extern from "cuml/neighbors/knn_mg.hpp" namespace "ML::KNN::opg" nogil: @@ -58,7 +52,7 @@ class KNeighborsClassifierMG(NearestNeighborsMG): def __init__(self, **kwargs): super(KNeighborsClassifierMG, self).__init__(**kwargs) - @api_base_return_array_skipall + @reflect(array=None) def predict( self, index, @@ -178,7 +172,7 @@ class KNeighborsClassifierMG(NearestNeighborsMG): return output_cais - @api_base_return_array_skipall + @reflect(array=None) def predict_proba(self, index, index_parts_to_ranks, index_nrows, query, query_parts_to_ranks, query_nrows, uniq_labels, n_unique, ncols, rank, diff --git a/python/cuml/cuml/neighbors/kneighbors_regressor_mg.pyx b/python/cuml/cuml/neighbors/kneighbors_regressor_mg.pyx index e539315560..e07efab869 100644 --- a/python/cuml/cuml/neighbors/kneighbors_regressor_mg.pyx +++ b/python/cuml/cuml/neighbors/kneighbors_regressor_mg.pyx @@ -2,13 +2,9 @@ # SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - -# distutils: language = c++ - import typing -import cuml.internals.logger as logger -from cuml.internals import api_base_return_array_skipall +from cuml.internals import logger, reflect from cuml.internals.array import CumlArray from cuml.neighbors.nearest_neighbors_mg import NearestNeighborsMG @@ -52,7 +48,7 @@ class KNeighborsRegressorMG(NearestNeighborsMG): def __init__(self, **kwargs): super(KNeighborsRegressorMG, self).__init__(**kwargs) - @api_base_return_array_skipall + @reflect(array=None) def predict( self, index, diff --git a/python/cuml/cuml/neighbors/nearest_neighbors_mg.pyx b/python/cuml/cuml/neighbors/nearest_neighbors_mg.pyx index 555d9ac16c..2275a94830 100644 --- a/python/cuml/cuml/neighbors/nearest_neighbors_mg.pyx +++ b/python/cuml/cuml/neighbors/nearest_neighbors_mg.pyx @@ -2,28 +2,22 @@ # SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - -# distutils: language = c++ - import typing -import cuml.internals.logger as logger from cuml.common import input_to_cuml_array -from cuml.internals import api_base_return_array_skipall +from cuml.common.opg_data_utils_mg import _build_part_inputs +from cuml.internals import logger, reflect from cuml.internals.array import CumlArray from cuml.neighbors import NearestNeighbors -from pylibraft.common.handle cimport handle_t - -from cuml.common.opg_data_utils_mg cimport * - -from cuml.common.opg_data_utils_mg import _build_part_inputs - from cython.operator cimport dereference as deref from libc.stdint cimport uintptr_t from libc.stdlib cimport free from libcpp cimport bool from libcpp.vector cimport vector +from pylibraft.common.handle cimport handle_t + +from cuml.common.opg_data_utils_mg cimport * cdef extern from "cuml/neighbors/knn_mg.hpp" namespace "ML::KNN::opg" nogil: @@ -60,7 +54,7 @@ class NearestNeighborsMG(NearestNeighbors): super().__init__(**kwargs) self.batch_size = batch_size - @api_base_return_array_skipall + @reflect(array=None) def kneighbors( self, index, diff --git a/python/cuml/cuml/preprocessing/onehotencoder_mg.py b/python/cuml/cuml/preprocessing/onehotencoder_mg.py index e04063246f..4afd5a6791 100644 --- a/python/cuml/cuml/preprocessing/onehotencoder_mg.py +++ b/python/cuml/cuml/preprocessing/onehotencoder_mg.py @@ -27,6 +27,8 @@ def _check_input_fit(self, X, is_categories=False): from cuml.dask.common.dask_arr_utils import to_dask_cudf + self._check_n_features(X, reset=True) + if isinstance(X, (dask.array.core.Array, cp.ndarray)): self._set_input_type("array") if is_categories: diff --git a/python/cuml/cuml/preprocessing/ordinalencoder_mg.py b/python/cuml/cuml/preprocessing/ordinalencoder_mg.py index 53bbff0ea4..6438afa3d2 100644 --- a/python/cuml/cuml/preprocessing/ordinalencoder_mg.py +++ b/python/cuml/cuml/preprocessing/ordinalencoder_mg.py @@ -19,6 +19,8 @@ def _check_input_fit(self, X, is_categories=False): from cuml.dask.common.dask_arr_utils import to_dask_cudf + self._check_n_features(X, reset=True) + if isinstance(X, (dask.array.core.Array, cp.ndarray)): self._set_input_type("array") if is_categories: diff --git a/python/cuml/cuml/solvers/cd_mg.pyx b/python/cuml/cuml/solvers/cd_mg.pyx index a907f61eb4..63216b4cf6 100644 --- a/python/cuml/cuml/solvers/cd_mg.pyx +++ b/python/cuml/cuml/solvers/cd_mg.pyx @@ -4,7 +4,7 @@ # import numpy as np -import cuml.internals +from cuml.internals import reflect from cuml.linear_model.base_mg import MGFitMixin from cuml.solvers import CD @@ -61,7 +61,7 @@ class CDMG(MGFitMixin, CD): """ Cython class for MNMG code usage. Not meant for end user consumption. """ - @cuml.internals.api_base_return_any_skipall + @reflect(skip=True) def _fit(self, uintptr_t X, uintptr_t y, uintptr_t coef_ptr, uintptr_t input_desc): cdef handle_t* handle_ = self.handle.getHandle() cdef bool use_f32 = self.dtype == np.float32 From 2b3929c1468ef82da945145026d88b63dc8a9d4e Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Tue, 25 Nov 2025 18:23:32 -0600 Subject: [PATCH 21/28] Remove base metaclass and old decorator names --- python/cuml/cuml/dask/common/base.py | 3 +- python/cuml/cuml/internals/__init__.py | 14 +--- python/cuml/cuml/internals/base.py | 2 +- python/cuml/cuml/internals/base_helpers.py | 94 ---------------------- python/cuml/cuml/internals/constants.py | 5 -- python/cuml/cuml/internals/mixins.py | 29 ++++++- python/cuml/cuml/internals/outputs.py | 29 ------- python/cuml/tests/test_cuml_descr_decor.py | 19 +++-- 8 files changed, 42 insertions(+), 153 deletions(-) delete mode 100644 python/cuml/cuml/internals/base_helpers.py delete mode 100644 python/cuml/cuml/internals/constants.py diff --git a/python/cuml/cuml/dask/common/base.py b/python/cuml/cuml/dask/common/base.py index 25abcb9445..3b3681a8a4 100644 --- a/python/cuml/cuml/dask/common/base.py +++ b/python/cuml/cuml/dask/common/base.py @@ -20,12 +20,11 @@ from cuml.dask.common import parts_to_ranks from cuml.dask.common.input_utils import DistributedDataHandler from cuml.dask.common.utils import get_client, wait_and_raise_from_futures -from cuml.internals import BaseMetaClass from cuml.internals.array import CumlArray from cuml.internals.base import Base -class BaseEstimator(object, metaclass=BaseMetaClass): +class BaseEstimator: def __init__(self, *, client=None, verbose=False, **kwargs): """ Constructor for distributed estimators. diff --git a/python/cuml/cuml/internals/__init__.py b/python/cuml/cuml/internals/__init__.py index 1b11185ea6..33944e598f 100644 --- a/python/cuml/cuml/internals/__init__.py +++ b/python/cuml/cuml/internals/__init__.py @@ -2,17 +2,5 @@ # SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # -from cuml.internals.base_helpers import BaseMetaClass, _tags_class_and_instance -from cuml.internals.constants import CUML_WRAPPED_FLAG from cuml.internals.internals import GraphBasedDimRedCallback -from cuml.internals.outputs import ( - api_base_fit_transform, - api_base_return_any, - api_base_return_any_skipall, - api_base_return_array, - api_base_return_array_skipall, - api_return_any, - api_return_array, - exit_internal_api, - reflect, -) +from cuml.internals.outputs import exit_internal_api, reflect diff --git a/python/cuml/cuml/internals/base.py b/python/cuml/cuml/internals/base.py index 75d50249aa..5fc34be877 100644 --- a/python/cuml/cuml/internals/base.py +++ b/python/cuml/cuml/internals/base.py @@ -18,7 +18,7 @@ from cuml.internals.outputs import check_output_type -class Base(TagsMixin, metaclass=cuml.internals.BaseMetaClass): +class Base(TagsMixin): """ Base class for all the ML algos. It handles some of the common operations across all algos. Every ML algo class exposed at cython level must inherit diff --git a/python/cuml/cuml/internals/base_helpers.py b/python/cuml/cuml/internals/base_helpers.py deleted file mode 100644 index 92d58b9f7c..0000000000 --- a/python/cuml/cuml/internals/base_helpers.py +++ /dev/null @@ -1,94 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 -# - -from cuml.internals.base_return_types import _get_base_return_type -from cuml.internals.constants import CUML_WRAPPED_FLAG -from cuml.internals.outputs import ( - api_base_return_any, - api_base_return_array, - api_return_any, -) - - -def _wrap_attribute(class_name: str, attribute_name: str, attribute, **kwargs): - # Skip items marked with autowrap_ignore - if attribute.__dict__.get(CUML_WRAPPED_FLAG, False): - return attribute - - return_type = _get_base_return_type(class_name, attribute) - - if return_type in ("generic", "array", "sparsearray"): - attribute = api_base_return_array(**kwargs)(attribute) - elif return_type == "base": - attribute = api_base_return_any(**kwargs)(attribute) - elif not attribute_name.startswith("_"): - # Only replace public functions with return any - attribute = api_return_any()(attribute) - - return attribute - - -class BaseMetaClass(type): - """ - Metaclass for all estimators in cuML. - - This metaclass will get called for estimators deriving from `cuml.common.Base` - as well as `cuml.dask.common.BaseEstimator`. It automatically wraps methods and - properties in the API decorators (`cuml.common.Base` only). - """ - - def __new__(cls, classname, bases, namespace): - # Skip wrapping methods in dask estimators - if not namespace["__module__"].startswith("cuml.dask"): - for name, attribute in namespace.items(): - if callable(attribute): - # Wrap method - namespace[name] = _wrap_attribute( - classname, name, attribute - ) - - elif ( - isinstance(attribute, property) - and attribute.fget is not None - ): - # Wrap property getters - namespace[name] = attribute.getter( - _wrap_attribute( - classname, - name, - attribute.fget, - input_arg=None, - ) - ) - - return type.__new__(cls, classname, bases, namespace) - - -class _tags_class_and_instance: - """ - Decorator for Base class to allow for dynamic and static _get_tags. - In general, most methods are either dynamic or static, so this decorator - is only meant to be used in the Base estimator _get_tags. - """ - - def __init__(self, _class, _instance=None): - self._class = _class - self._instance = _instance - - def instance_method(self, _instance): - """ - Factory to create a _tags_class_and_instance instance method with - the existing class associated. - """ - return _tags_class_and_instance(self._class, _instance) - - def __get__(self, _instance, _class): - # if the caller had no instance (i.e. it was a class) or there is no - # instance associated we the method we return the class call - if _instance is None or self._instance is None: - return self._class.__get__(_class, None) - - # otherwise return instance call - return self._instance.__get__(_instance, _class) diff --git a/python/cuml/cuml/internals/constants.py b/python/cuml/cuml/internals/constants.py deleted file mode 100644 index 0a10b89cc3..0000000000 --- a/python/cuml/cuml/internals/constants.py +++ /dev/null @@ -1,5 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 -# -CUML_WRAPPED_FLAG = "__cuml_is_wrapped" diff --git a/python/cuml/cuml/internals/mixins.py b/python/cuml/cuml/internals/mixins.py index 62853f5696..54ca615c00 100644 --- a/python/cuml/cuml/internals/mixins.py +++ b/python/cuml/cuml/internals/mixins.py @@ -8,7 +8,6 @@ from cuml._thirdparty._sklearn_compat import _to_new_tags from cuml.common.doc_utils import generate_docstring -from cuml.internals.base_helpers import _tags_class_and_instance from cuml.internals.outputs import reflect ############################################################################### @@ -45,6 +44,34 @@ } +class _tags_class_and_instance: + """ + Decorator for mixins to allow for dynamic and static _get_tags. + In general, most methods are either dynamic or static, so this decorator + is only meant to be used in the mixins _get_tags. + """ + + def __init__(self, _class, _instance=None): + self._class = _class + self._instance = _instance + + def instance_method(self, _instance): + """ + Factory to create a _tags_class_and_instance instance method with + the existing class associated. + """ + return _tags_class_and_instance(self._class, _instance) + + def __get__(self, _instance, _class): + # if the caller had no instance (i.e. it was a class) or there is no + # instance associated we the method we return the class call + if _instance is None or self._instance is None: + return self._class.__get__(_class, None) + + # otherwise return instance call + return self._instance.__get__(_instance, _class) + + class TagsMixin: @_tags_class_and_instance def _get_tags(cls): diff --git a/python/cuml/cuml/internals/outputs.py b/python/cuml/cuml/internals/outputs.py index 1873e0bb4c..24cb267cf7 100644 --- a/python/cuml/cuml/internals/outputs.py +++ b/python/cuml/cuml/internals/outputs.py @@ -11,7 +11,6 @@ # TODO: Try to resolve circular import that makes this necessary: from cuml.internals import input_utils as iu from cuml.internals.array_sparse import SparseCumlArray -from cuml.internals.constants import CUML_WRAPPED_FLAG from cuml.internals.global_settings import GlobalSettings __all__ = ( @@ -353,9 +352,6 @@ def reflect( skip=skip, ) - # TODO: remove this once auto-decorating is ripped out - setattr(func, CUML_WRAPPED_FLAG, True) - sig = inspect.signature(func, follow_wrapped=True) has_self = "self" in sig.parameters @@ -437,28 +433,3 @@ def inner(*args, **kwargs): return res return inner - - -def api_return_array(input_arg=default, get_output_type=False): - return reflect(array=None if not get_output_type else input_arg) - - -def api_return_any(): - return reflect(array=None, skip=True) - - -def api_base_return_any(): - return reflect(reset=True) - - -def api_base_return_array(input_arg=default): - return reflect(array="self" if input_arg is None else input_arg) - - -def api_base_fit_transform(): - return reflect(reset=True) - - -# TODO: investigate and remove these -api_base_return_any_skipall = api_return_any() -api_base_return_array_skipall = reflect diff --git a/python/cuml/tests/test_cuml_descr_decor.py b/python/cuml/tests/test_cuml_descr_decor.py index 999466334a..a98644da88 100644 --- a/python/cuml/tests/test_cuml_descr_decor.py +++ b/python/cuml/tests/test_cuml_descr_decor.py @@ -45,27 +45,30 @@ class DummyTestEstimator(cuml.Base): def _set_input(self, X): self.input_any_ = X - @cuml.internals.api_base_return_any() + @cuml.internals.reflect(reset=True) def store_input(self, X): self.input_any_ = X - @cuml.internals.api_return_any() + @cuml.internals.reflect(skip=True) def get_input(self): return self.input_any_ - # === Standard Functions === - def fit(self, X, convert_dtype=True) -> "DummyTestEstimator": + @cuml.internals.reflect(reset=True) + def fit(self, X, convert_dtype=True): self._set_output_type(X) self._set_n_features_in(X) return self - def predict(self, X, convert_dtype=True) -> CumlArray: + @cuml.internals.reflect + def predict(self, X, convert_dtype=True): return X - def transform(self, X, convert_dtype=False) -> CumlArray: + @cuml.internals.reflect + def transform(self, X, convert_dtype=False): pass - def fit_transform(self, X, y=None) -> CumlArray: + @cuml.internals.reflect + def fit_transform(self, X, y=None): return self.fit(X).transform(X) @@ -280,7 +283,7 @@ def test_return_array(input_arg: str): X_in = create_input(input_type_X, input_dtype_X, (10, 10), "F") Y_in = create_input(input_type_Y, input_dtype_Y, (10, 10), "F") - @cuml.internals.api_return_array(input_arg=input_arg, get_output_type=True) + @cuml.internals.reflect(array=input_arg) def test_func(X, y): return X From df7799f318f9cd0454e478a569b84bcf6e77917d Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Wed, 26 Nov 2025 09:47:24 -0600 Subject: [PATCH 22/28] Update doc refs --- docs/source/api.rst | 5 +++-- docs/source/cuml_intro.rst | 6 ++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/source/api.rst b/docs/source/api.rst index db74964fbf..de62a52041 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -15,8 +15,9 @@ Module Configuration Output Data Type Configuration ------------------------------ - .. autofunction:: cuml.internals.memory_utils.set_global_output_type - .. autofunction:: cuml.internals.memory_utils.using_output_type +.. autofunction:: cuml.set_global_output_type + +.. autofunction:: cuml.using_output_type .. _verbosity-levels: diff --git a/docs/source/cuml_intro.rst b/docs/source/cuml_intro.rst index e5b1d56142..06c028ca1d 100644 --- a/docs/source/cuml_intro.rst +++ b/docs/source/cuml_intro.rst @@ -77,10 +77,8 @@ fit a model with a NumPy array, the ``model.coef_`` property containing fitted coefficients will also be a NumPy array. If you fit a model using cuDF's GPU-based DataFrame and Series objects, the model's output properties will be cuDF objects. You can always -override this behavior and select a default datatype with the -`memory_utils.set_global_output_type -`_ -function. +override this behavior and select a default datatype with +:func:`cuml.set_global_output_type`. The `RAPIDS Configurable Input and Output Types `_ blog post goes into much From b935d763599ae049dab65d311129f86226c13471 Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Wed, 26 Nov 2025 10:54:42 -0600 Subject: [PATCH 23/28] Improve docstrings --- python/cuml/cuml/internals/__init__.py | 1 + python/cuml/cuml/internals/base.py | 116 +++--------- python/cuml/cuml/internals/outputs.py | 252 ++++++++++++------------- 3 files changed, 156 insertions(+), 213 deletions(-) diff --git a/python/cuml/cuml/internals/__init__.py b/python/cuml/cuml/internals/__init__.py index 33944e598f..b3d971370c 100644 --- a/python/cuml/cuml/internals/__init__.py +++ b/python/cuml/cuml/internals/__init__.py @@ -2,5 +2,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2019-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # +from cuml.internals.base import Base from cuml.internals.internals import GraphBasedDimRedCallback from cuml.internals.outputs import exit_internal_api, reflect diff --git a/python/cuml/cuml/internals/base.py b/python/cuml/cuml/internals/base.py index 5fc34be877..4cd51b7a05 100644 --- a/python/cuml/cuml/internals/base.py +++ b/python/cuml/cuml/internals/base.py @@ -19,68 +19,18 @@ class Base(TagsMixin): - """ - Base class for all the ML algos. It handles some of the common operations - across all algos. Every ML algo class exposed at cython level must inherit - from this class. - - Typical estimator design using Base requires three main things: - - 1. Call the base __init__ method explicitly from inheriting estimators in - their __init__. - - 2. Attributes that users will want to access, and are array-like should - use cuml.internals.array, and have a preceding underscore `_` before - the name the user expects. That way the __getattr__ of Base will - convert it automatically to the appropriate output format for the - user. For example, in DBSCAN the user expects to be able to access - `model.labels_`, so the code actually has an attribute - `model._labels_` that gets converted at the moment the user accesses - `labels_` automatically. No need for extra code in inheriting classes - as long as they follow that naming convention. It is recommended to - create the attributes in the constructor assigned to None, and - add a note for users that might look into the code to see what - attributes the class might have. For example, in KMeans: - - .. code-block:: python + """Base class for cuml estimators. - def __init__(...) - super(KMeans, self).__init__(handle, verbose, output_type) + Subclasses should: - # initialize numeric variables + - Define ``_get_param_names`` to extend the base implementation with + any additional parameter names. - # internal array attributes - self._labels_ = None # accessed via estimator.labels_ - self._cluster_centers_ = None # accessed via estimator.cluster_centers_ # noqa + - Decorate their ``fit`` method with ``cuml.internals.reflect(reset=True)`` + to store their fitted input type. - 3. To appropriately work for outputs mirroring the format of inputs of the - user when appropriate, the code in the inheriting estimator must call - the following methods, with input being the data sent by the user: - - - `self._set_output_type(input)` in `fit` methods that modify internal - structures. This will allow users to receive the correct format when - accessing internal attributes of the class (eg. labels_ in KMeans).: - - .. code-block:: python - - def fit(self, X): - self._set_output_type(X) - # rest of the fit code - - - `out_type = self._get_output_type(input)` in `predict`/`transform` style - methods, that don't modify class attributes. out_type then can be used - to return the correct format to the user. For example, in KMeans: - - .. code-block:: python - - def transform(self, X, convert_dtype=False): - out_type = self._get_output_type(X) - X_m, n_rows, n_cols, dtype = input_to_cuml_array(X ...) - preds = CumlArray.zeros(...) - - # method code and call to C++ and whatever else is needed - - return preds.to_output(out_type) + - Decorate methods that return array likes with ``cuml.internals.reflect`` + to properly coerce outputs to the proper type. Parameters ---------- @@ -106,42 +56,34 @@ def transform(self, X, convert_dtype=False): .. code-block:: python - from cuml import Base + import cupy as cp + from cuml.internals import Base, reflect - # assuming this ML algo has separate 'fit' and 'predict' methods class MyAlgo(Base): - def __init__(self, ...): - super(MyAlgo, self).__init__(...) - # other setup logic - - def fit(self, data, ...): - # check output format - self._check_output_type(data) - # train logic goes here - - def predict(self, data, ...): - # check output format - self._check_output_type(data) - # inference logic goes here + def __init__( + self, + *, + param=123, + handle=None, + verbose=False, + output_type=None, + ): + super().__init__(handle=handle, verbose=verbose, output_type=output_type) + self.param = param @classmethod def _get_param_names(cls): - # return a list of hyperparam names supported by this algo - - # stream and handle example: - - stream = pylibraft.common.Stream() - handle = pylibraft.common.Handle(stream=stream) + return [*super()._get_param_names(), "param"] - algo = MyAlgo(handle=handle) - algo.fit(...) - result = algo.predict(...) + @reflect(reset=True) + def fit(self, X, y): + # Training logic goes here... + return self - # final sync of all gpu-work launched inside this object - # this is same as `pylibraft.common.Stream.sync()` call, but safer in case - # the default stream inside the `raft::handle_t` is being used - base.handle.sync() - del base # optional! + @reflect + def predict(self, X): + # Inference logic goes here... + return cp.ones(len(X), dtype="int32") """ def __init__( diff --git a/python/cuml/cuml/internals/outputs.py b/python/cuml/cuml/internals/outputs.py index 24cb267cf7..c3ebe0cb92 100644 --- a/python/cuml/cuml/internals/outputs.py +++ b/python/cuml/cuml/internals/outputs.py @@ -22,12 +22,6 @@ ) -default = type( - "default", - (), - dict.fromkeys(["__repr__", "__reduce__"], lambda s: "default"), -)() - OUTPUT_TYPES = ( "input", "numpy", @@ -43,6 +37,7 @@ def check_output_type(output_type: str) -> str: + """Validate and normalize an ``output_type`` value""" # normalize as lower, keeping original str reference to appease the sklearn # standard estimator checks as much as possible. if output_type != (temp := output_type.lower()): @@ -58,74 +53,79 @@ def check_output_type(output_type: str) -> str: def set_global_output_type(output_type): - """ - Method to set cuML's single GPU estimators global output type. - It will be used by all estimators unless overridden in their initialization - with their own output_type parameter. Can also be overridden by the context - manager method :func:`using_output_type`. + """Set the global output type. + + This output type will be used by functions and estimator methods. + + Note that instead of setting globally, an output type may be set + contextually using :func:`using_output_type`, or on the estimator itself + with the ``output_type`` parameter. Parameters ---------- - output_type : {'input', 'cudf', 'cupy', 'numpy'} (default = 'input') + output_type : {'input', 'cupy', 'numpy', 'cudf', 'pandas', None} Desired output type of results and attributes of the estimators. - * ``'input'`` will mean that the parameters and methods will mirror the - format of the data sent to the estimators/methods as much as - possible. Specifically: - - +---------------------------------------+--------------------------+ - | Input type | Output type | - +=======================================+==========================+ - | cuDF DataFrame or Series | cuDF DataFrame or Series | - +---------------------------------------+--------------------------+ - | NumPy arrays | NumPy arrays | - +---------------------------------------+--------------------------+ - | Pandas DataFrame or Series | NumPy arrays | - +---------------------------------------+--------------------------+ - | Numba device arrays | Numba device arrays | - +---------------------------------------+--------------------------+ - | CuPy arrays | CuPy arrays | - +---------------------------------------+--------------------------+ - | Other `__cuda_array_interface__` objs | CuPy arrays | - +---------------------------------------+--------------------------+ - - * ``'cudf'`` will return cuDF Series for single dimensional results and - DataFrames for the rest. - - * ``'cupy'`` will return CuPy arrays. - - * ``'numpy'`` will return NumPy arrays. + * ``None``: No globally configured output type. This is the same as + ``'input'``, except in cases where an estimator explicitly sets + an ``output_type``. + + * ``'input'``: returns arrays of the same type as the inputs to the + function or method. Fitted attributes will be of the same array type + as ``X``. + + * ``'cupy'``: returns ``cupy`` arrays. + + * ``'numpy'``: returns ``numpy`` arrays. + + * ``'cudf'``: returns ``cudf.Series`` for single dimensional results + and ``cudf.DataFrame`` otherwise. + + * ``'pandas'``: returns ``pandas.Series`` for single dimensional results + and ``pandas.DataFrame`` otherwise. + + See Also + -------- + cuml.using_output_type + + Notes + ----- + ``cupy`` is the most efficient output type, as it supports flexible memory + layouts and doesn't require device <-> host transfers. + + ``cudf`` has slightly more overhead for single dimensional outputs. For two + dimensional outputs additional copies may be needed due to memory layout + requirements of ``cudf.DataFrame``. + + ``numpy`` and ``pandas`` have a more significant overhead as they require + device <-> host transfers. Whether that overhead matters is of course + application specific. Examples -------- >>> import cuml >>> import cupy as cp - >>> ary = [[1.0, 4.0, 4.0], [2.0, 2.0, 2.0], [5.0, 1.0, 1.0]] - >>> ary = cp.asarray(ary) - >>> prev_output_type = cuml.global_settings.output_type - >>> cuml.set_global_output_type('cudf') - >>> dbscan_float = cuml.DBSCAN(eps=1.0, min_samples=1) - >>> dbscan_float.fit(ary) - DBSCAN() - >>> - >>> # cuML output type - >>> dbscan_float.labels_ - 0 0 - 1 1 - 2 2 - dtype: int32 - >>> type(dbscan_float.labels_) - - >>> cuml.set_global_output_type(prev_output_type) + >>> import cudf + >>> original_output_type = cuml.global_settings.output_type - Notes - ----- - ``'cupy'`` and ``'numba'`` options (as well as ``'input'`` when using Numba - and CuPy ndarrays for input) have the least overhead. cuDF add memory - consumption and processing time needed to build the Series and DataFrames. - ``'numpy'`` has the biggest overhead due to the need to transfer data to - CPU memory. + Fit a model with a cupy array. By default the fitted attributes will be + cupy arrays. + + >>> X = cp.array([[1.0, 4.0, 4.0], [2.0, 2.0, 2.0], [5.0, 1.0, 1.0]]) + >>> model = cuml.DBSCAN(eps=1.0, min_samples=1).fit(X) + >>> isinstance(model.labels_, cp.ndarray) + True + + With a global output type set though, the fitted attributes will match + the configured output type. + >>> cuml.set_global_output_type("cudf") + >>> isinstance(model.labels_, cudf.Series) + True + + Reset the output type back to its original value. + + >>> cuml.set_global_output_type(original_output_type) """ if output_type is not None: output_type = check_output_type(output_type) @@ -133,72 +133,54 @@ def set_global_output_type(output_type): class using_output_type: - """ - Context manager method to set cuML's global output type inside a `with` - statement. It gets reset to the prior value it had once the `with` code - block is executer. + """Configure the output type within a context. Parameters ---------- - output_type : {'input', 'cudf', 'cupy', 'numpy'} (default = 'input') + output_type : {'input', 'cupy', 'numpy', 'cudf', 'pandas', None} Desired output type of results and attributes of the estimators. - * ``'input'`` will mean that the parameters and methods will mirror the - format of the data sent to the estimators/methods as much as - possible. Specifically: - - +---------------------------------------+--------------------------+ - | Input type | Output type | - +=======================================+==========================+ - | cuDF DataFrame or Series | cuDF DataFrame or Series | - +---------------------------------------+--------------------------+ - | NumPy arrays | NumPy arrays | - +---------------------------------------+--------------------------+ - | Pandas DataFrame or Series | NumPy arrays | - +---------------------------------------+--------------------------+ - | Numba device arrays | Numba device arrays | - +---------------------------------------+--------------------------+ - | CuPy arrays | CuPy arrays | - +---------------------------------------+--------------------------+ - | Other `__cuda_array_interface__` objs | CuPy arrays | - +---------------------------------------+--------------------------+ - - * ``'cudf'`` will return cuDF Series for single dimensional results and - DataFrames for the rest. - - * ``'cupy'`` will return CuPy arrays. - - * ``'numpy'`` will return NumPy arrays. + * ``None``: No globally configured output type. This is the same as + ``'input'``, except in cases where an estimator explicitly sets + an ``output_type``. + + * ``'input'``: returns arrays of the same type as the inputs to the + function or method. Fitted attributes will be of the same array type + as ``X``. + + * ``'cupy'``: returns ``cupy`` arrays. + + * ``'numpy'``: returns ``numpy`` arrays. + + * ``'cudf'``: returns ``cudf.Series`` for single dimensional results + and ``cudf.DataFrame`` otherwise. + + * ``'pandas'``: returns ``pandas.Series`` for single dimensional results + and ``pandas.DataFrame`` otherwise. + + See Also + -------- + cuml.set_global_output_type Examples -------- >>> import cuml >>> import cupy as cp - >>> ary = [[1.0, 4.0, 4.0], [2.0, 2.0, 2.0], [5.0, 1.0, 1.0]] - >>> ary = cp.asarray(ary) - >>> with cuml.using_output_type('cudf'): - ... dbscan_float = cuml.DBSCAN(eps=1.0, min_samples=1) - ... dbscan_float.fit(ary) - ... - ... print("cuML output inside 'with' context") - ... print(dbscan_float.labels_) - ... print(type(dbscan_float.labels_)) - ... - DBSCAN() - cuML output inside 'with' context - 0 0 - 1 1 - 2 2 - dtype: int32 - - >>> # use cuml again outside the context manager - >>> dbscan_float2 = cuml.DBSCAN(eps=1.0, min_samples=1) - >>> dbscan_float2.fit(ary) - DBSCAN() - >>> # cuML default output - >>> dbscan_float2.labels_ - array([0, 1, 2], dtype=int32) - >>> isinstance(dbscan_float2.labels_, cp.ndarray) + >>> import cudf + + Fit a model with a cupy array. By default the fitted attributes will be + cupy arrays. + + >>> X = cp.array([[1.0, 4.0, 4.0], [2.0, 2.0, 2.0], [5.0, 1.0, 1.0]]) + >>> model = cuml.DBSCAN(eps=1.0, min_samples=1).fit(X) + >>> isinstance(model.labels_, cp.ndarray) + True + + With a global output type set though, the fitted attributes will match + the configured output type. + + >>> with cuml.using_output_type("cudf"): + ... print(isinstance(model.labels_, cudf.Series)) True """ @@ -309,23 +291,39 @@ def coerce_arrays(res, output_type): def reflect( func=None, *, - array=default, - model=default, + array=..., + model=..., reset=False, skip=False, ): """Mark a function or method as participating in the reflection system. + Functions and methods decorated with this get a few additional behaviors: + + - They are run within an "internal API context". This mainly means that + reflected functions/methods or estimator fitted attributes will be + returned as ``CumlArray`` instances instead of their reflected types. If + this is the only behavior you want, decorate with ``reflect(skip=True)``. + + - Their output type is converted to the proper output type following + standard cuml behavior. The default behavior covers most cases, but when + needed you may want to specify the ``model`` and/or ``array`` parameters + manually. + + - For estimators, fit-like methods will store the required metadata like + ``_input_type`` to support cases like ``output_type="input"``. To enable + this for a method set ``reset=True``. + Parameters ---------- func : callable or None The function to be decorated, or None to curry to be applied later. - model : int, str, or None, default=default + model : int, str, or None, default=... The ``cuml.Base`` parameter to infer the reflected output type from. By default this will be ``'self'`` (if present), and ``None`` otherwise. Provide a parameter position or name to override. May also provide ``None`` to disable this inference entirely. - array : int, str, or None, default=default + array : int, str, or None, default=... The array-like parameter to infer the reflected output type from. By default this will be the first argument to the method or function (excluding ``'self'`` or ``model``), or ``None`` if there are no other @@ -337,8 +335,10 @@ def reflect( Set to True for methods like ``fit`` that reset the reflected type on an estimator. skip : bool, default=False - Set to True to skip output processing for a method. This is mostly - useful if output processing will be handled manually. + Set to True to skip output type inference and processing for a method. + This is mostly useful if this step is unnecessary (but the function + should still run within an internal API context), or if output type + conversion will be handled manually. """ # Local to avoid circular imports import cuml.accel @@ -356,7 +356,7 @@ def reflect( has_self = "self" in sig.parameters # Normalize model to str | None - if model is default: + if model is ...: if skip and not reset: # We're skipping output processing and not resetting an estimator, # there's no need to touch input parameters at all @@ -367,7 +367,7 @@ def reflect( model = _get_param(sig, model) # Normalize array to str | None - if array is default: + if array is ...: if skip and not reset: array = None else: From d58ac8088033a98a19792c9e7445c3d1c6e78ea4 Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Wed, 26 Nov 2025 12:28:30 -0600 Subject: [PATCH 24/28] Rename `test_module_config.py -> test_reflection.py` --- python/cuml/tests/{test_module_config.py => test_reflection.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename python/cuml/tests/{test_module_config.py => test_reflection.py} (100%) diff --git a/python/cuml/tests/test_module_config.py b/python/cuml/tests/test_reflection.py similarity index 100% rename from python/cuml/tests/test_module_config.py rename to python/cuml/tests/test_reflection.py From a44e6df2446915ba696e718463d5e0c53b563e0e Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Wed, 26 Nov 2025 14:35:57 -0600 Subject: [PATCH 25/28] Consolidate and expand reflection tests --- python/cuml/cuml/internals/outputs.py | 12 +- python/cuml/tests/test_cuml_descr_decor.py | 297 --------------- python/cuml/tests/test_reflection.py | 409 +++++++++++++++++---- 3 files changed, 339 insertions(+), 379 deletions(-) delete mode 100644 python/cuml/tests/test_cuml_descr_decor.py diff --git a/python/cuml/cuml/internals/outputs.py b/python/cuml/cuml/internals/outputs.py index c3ebe0cb92..954d920ca7 100644 --- a/python/cuml/cuml/internals/outputs.py +++ b/python/cuml/cuml/internals/outputs.py @@ -46,8 +46,8 @@ def check_output_type(output_type: str) -> str: if output_type != "cuml" and output_type not in OUTPUT_TYPES: valid_output_types = ", ".join(map(repr, OUTPUT_TYPES)) raise ValueError( - f"output_type must be one of {valid_output_types}" - f" or None. Got: {output_type}" + f"`output_type` must be one of {valid_output_types}" + f" or None. Got: {output_type!r}" ) return output_type @@ -285,6 +285,11 @@ def coerce_arrays(res, output_type): # Return CumlArray/SparseCumlArray directly return res + if is_sparse: + # Coerce output_type to supported sparse types. + # Host types -> scipy, cupy otherwise. + output_type = "scipy" if output_type in ["numpy", "pandas"] else "cupy" + return res.to_output(output_type=output_type) @@ -353,7 +358,6 @@ def reflect( ) sig = inspect.signature(func, follow_wrapped=True) - has_self = "self" in sig.parameters # Normalize model to str | None if model is ...: @@ -362,7 +366,7 @@ def reflect( # there's no need to touch input parameters at all model = None else: - model = "self" if has_self else None + model = "self" if ("self" in sig.parameters) else None if model is not None: model = _get_param(sig, model) diff --git a/python/cuml/tests/test_cuml_descr_decor.py b/python/cuml/tests/test_cuml_descr_decor.py deleted file mode 100644 index a98644da88..0000000000 --- a/python/cuml/tests/test_cuml_descr_decor.py +++ /dev/null @@ -1,297 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 -# - -import pickle - -import cupy as cp -import numpy as np -import pytest - -import cuml -import cuml.internals -from cuml.common.array_descriptor import CumlArrayDescriptor -from cuml.internals.array import CumlArray -from cuml.internals.input_utils import ( - determine_array_type, - input_to_cuml_array, -) - -test_input_types = ["numpy", "numba", "cupy", "cudf"] - -test_output_types_str = ["numpy", "numba", "cupy", "cudf"] - -test_dtypes_short = [ - np.uint8, - np.float16, - np.int32, - np.float64, -] - -unsupported_cudf_dtypes = [ - np.uint8, - np.uint16, - np.uint32, - np.uint64, - np.float16, -] - -test_shapes = [10, (10, 1), (10, 5), (1, 10)] - - -class DummyTestEstimator(cuml.Base): - input_any_ = CumlArrayDescriptor() - - def _set_input(self, X): - self.input_any_ = X - - @cuml.internals.reflect(reset=True) - def store_input(self, X): - self.input_any_ = X - - @cuml.internals.reflect(skip=True) - def get_input(self): - return self.input_any_ - - @cuml.internals.reflect(reset=True) - def fit(self, X, convert_dtype=True): - self._set_output_type(X) - self._set_n_features_in(X) - return self - - @cuml.internals.reflect - def predict(self, X, convert_dtype=True): - return X - - @cuml.internals.reflect - def transform(self, X, convert_dtype=False): - pass - - @cuml.internals.reflect - def fit_transform(self, X, y=None): - return self.fit(X).transform(X) - - -def assert_array_identical(a, b): - cupy_a = input_to_cuml_array(a, order="K").array - cupy_b = input_to_cuml_array(b, order="K").array - - if len(a) == 0 and len(b) == 0: - return True - - assert cupy_a.shape == cupy_b.shape - assert cupy_a.dtype == cupy_b.dtype - assert cupy_a.order == cupy_b.order - assert cp.all(cp.asarray(cupy_a) == cp.asarray(cupy_b)).item() - - -def create_input(input_type, input_dtype, input_shape, input_order): - rand_ary = cp.ones(input_shape, dtype=input_dtype, order=input_order) - - cuml_ary = CumlArray(rand_ary) - - return cuml_ary.to_output(input_type) - - -def create_output(X_in, output_type): - cuml_ary_tuple = input_to_cuml_array(X_in, order="K") - - return cuml_ary_tuple.array.to_output(output_type) - - -@pytest.mark.parametrize("input_type", test_input_types) -def test_pickle(input_type): - if input_type == "numba": - pytest.skip("numba arrays cant be picked at this time") - - est = DummyTestEstimator() - - X_in = create_input(input_type, np.float32, (10, 5), "C") - - est.store_input(X_in) - - # Loop and verify we have filled the cache - for out_type in test_output_types_str: - with cuml.using_output_type(out_type): - assert_array_identical( - est.input_any_, create_output(X_in, out_type) - ) - - est_pickled_bytes = pickle.dumps(est) - est_unpickled: DummyTestEstimator = pickle.loads(est_pickled_bytes) - - # Assert that we only resture the input - assert est_unpickled.__dict__["input_any_"].input_type == input_type - assert len(est_unpickled.__dict__["input_any_"].values) == 1 - - assert_array_identical(est.get_input(), est_unpickled.get_input()) - assert_array_identical(est.input_any_, est_unpickled.input_any_) - - # Loop one more time with the picked one to make sure it works right - for out_type in test_output_types_str: - with cuml.using_output_type(out_type): - assert_array_identical( - est.input_any_, create_output(X_in, out_type) - ) - - est_unpickled.output_type = out_type - - assert_array_identical( - est_unpickled.input_any_, create_output(X_in, out_type) - ) - - -@pytest.mark.parametrize("input_type", test_input_types) -@pytest.mark.parametrize("input_dtype", [np.float32, np.int16]) -@pytest.mark.parametrize("input_shape", [10, (10, 5)]) -@pytest.mark.parametrize("output_type", test_output_types_str) -def test_dec_input_output(input_type, input_dtype, input_shape, output_type): - if input_type == "cudf" or output_type == "cudf": - if input_dtype in unsupported_cudf_dtypes: - pytest.skip("Unsupported cudf combination") - - X_in = create_input(input_type, input_dtype, input_shape, "C") - X_out = create_output(X_in, output_type) - - # Test with output_type="input" - est = DummyTestEstimator(output_type="input") - - est.store_input(X_in) - - # Test is was stored internally correctly - assert X_in is est.get_input() - - assert est.__dict__["input_any_"].input_type == input_type - - # Check the current type matches input type - assert determine_array_type(est.input_any_) == input_type - - assert_array_identical(est.input_any_, X_in) - - # Switch output type and check type and equality - with cuml.using_output_type(output_type): - assert determine_array_type(est.input_any_) == output_type - - assert_array_identical(est.input_any_, X_out) - - # Now Test with output_type=output_type - est = DummyTestEstimator(output_type=output_type) - - est.store_input(X_in) - - # Check the current type matches output type - assert determine_array_type(est.input_any_) == output_type - - assert_array_identical(est.input_any_, X_out) - - with cuml.using_output_type("input"): - assert determine_array_type(est.input_any_) == input_type - - assert_array_identical(est.input_any_, X_in) - - -@pytest.mark.parametrize("input_type", test_input_types) -@pytest.mark.parametrize("input_dtype", [np.float32, np.int16]) -@pytest.mark.parametrize("input_shape", test_shapes) -def test_auto_fit(input_type, input_dtype, input_shape): - """ - Test autowrapping on fit that will set output_type, and n_features - """ - X_in = create_input(input_type, input_dtype, input_shape, "C") - - # Test with output_type="input" - est = DummyTestEstimator() - - est.fit(X_in) - - def calc_n_features(shape): - if isinstance(shape, tuple) and len(shape) >= 1: - # When cudf and shape[1] is used, a series is created which will - # remove the last shape - if input_type == "cudf" and shape[1] == 1: - return 1 - - return shape[1] - - return 1 - - assert est._input_type == input_type - assert est.n_features_in_ == calc_n_features(input_shape) - - -@pytest.mark.parametrize("input_type", test_input_types) -@pytest.mark.parametrize("base_output_type", test_input_types) -@pytest.mark.parametrize( - "global_output_type", test_output_types_str + ["input", None] -) -def test_auto_predict(input_type, base_output_type, global_output_type): - """ - Test autowrapping on predict that will set target_type - """ - X_in = create_input(input_type, np.float32, (10, 10), "F") - - # Test with output_type="input" - est = DummyTestEstimator() - - # With cuml.global_settings.output_type == None, this should return the - # input type - X_out = est.predict(X_in) - - assert determine_array_type(X_out) == input_type - - assert_array_identical(X_in, X_out) - - # Test with output_type=base_output_type - est = DummyTestEstimator(output_type=base_output_type) - - # With cuml.global_settings.output_type == None, this should return the - # base_output_type - X_out = est.predict(X_in) - - assert determine_array_type(X_out) == base_output_type - - assert_array_identical(X_in, X_out) - - # Test with global_output_type, should return global_output_type - with cuml.using_output_type(global_output_type): - X_out = est.predict(X_in) - - target_output_type = global_output_type - - if target_output_type is None or target_output_type == "input": - target_output_type = base_output_type - - if target_output_type == "input": - target_output_type = input_type - - assert determine_array_type(X_out) == target_output_type - - assert_array_identical(X_in, X_out) - - -@pytest.mark.parametrize("input_arg", ["X", "y", 0]) -def test_return_array(input_arg: str): - """ - Test autowrapping on predict that will set target_type - """ - input_type_X = "numpy" - input_dtype_X = np.float64 - - input_type_Y = "cupy" - input_dtype_Y = np.int32 - - X_in = create_input(input_type_X, input_dtype_X, (10, 10), "F") - Y_in = create_input(input_type_Y, input_dtype_Y, (10, 10), "F") - - @cuml.internals.reflect(array=input_arg) - def test_func(X, y): - return X - - X_out = test_func(X=X_in, y=Y_in) - - if input_arg == "y": - target_type = input_type_Y - else: - target_type = input_type_X - - assert determine_array_type(X_out) == target_type diff --git a/python/cuml/tests/test_reflection.py b/python/cuml/tests/test_reflection.py index 040e443b4f..e428af532d 100644 --- a/python/cuml/tests/test_reflection.py +++ b/python/cuml/tests/test_reflection.py @@ -1,130 +1,383 @@ -# # SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 -# +import pickle import cudf +import cudf.pandas import cupy as cp +import cupyx.scipy.sparse import numpy as np import pandas as pd import pytest -from cudf.pandas import LOADED as cudf_pandas_active +import scipy.sparse from numba.cuda import as_cuda_array, is_cuda_array import cuml +from cuml.common.array_descriptor import CumlArrayDescriptor +from cuml.internals import reflect +from cuml.internals.array import CumlArray +from cuml.internals.array_sparse import SparseCumlArray +from cuml.internals.base import Base +from cuml.internals.global_settings import GlobalSettings -############################################################################### -# Parameters # -############################################################################### +OUTPUT_TYPES = ["numpy", "numba", "cupy", "cudf", "pandas"] -global_input_configs = ["numpy", "numba", "cupy", "cudf"] -global_input_types = ["numpy", "numba", "cupy", "cudf", "pandas"] +@pytest.fixture(autouse=True) +def reset_global_output_type(): + yield + # Ensure we reset the type at the end of the test + cuml.set_global_output_type(None) + + +def assert_output_type(arr, output_type): + if output_type == "numba": + assert is_cuda_array(arr) + else: + cls = { + "numpy": np.ndarray, + "cupy": cp.ndarray, + "cudf": (cudf.Series, cudf.DataFrame), + "pandas": (pd.Series, pd.DataFrame), + }[output_type] + assert isinstance(arr, cls) + + +def rand_array(output_type, *, shape=(8, 4), seed=42): + X = cp.random.default_rng(seed).uniform( + low=0.0, high=10.0, size=shape, dtype="float32" + ) + if output_type == "numba": + return as_cuda_array(X) + elif output_type == "cupy": + return X + elif output_type == "numpy": + return cp.asnumpy(X) + elif output_type == "pandas": + return pd.DataFrame(X.get()) + else: + assert output_type == "cudf" + return cudf.DataFrame(X) -test_output_types = { - "numpy": np.ndarray, - "cupy": cp.ndarray, - "cudf": cudf.Series, - "pandas": pd.Series, -} +class DummyEstimator(Base): + X_ = CumlArrayDescriptor() -@pytest.fixture(scope="function", params=global_input_configs) -def global_output_type(request): - output_type = request.param + @reflect(reset=True) + def fit(self, X, y=None): + self.X_ = CumlArray.from_input(X) + return self - yield output_type + @reflect + def example(self, X): + return cp.zeros(3) + + @reflect + def example_no_args(self): + return cp.zeros(3) + + @reflect(skip=True) + def check_descriptor(self): + # When run in an internal context, a descriptor returns its original + # internal value. + assert isinstance(self.X_, CumlArray) + + with cuml.using_output_type("cupy"): + # Can override with using_output_type + assert_output_type(self.X_, "cupy") + + +@reflect +def reflects_input(X): + return X + + +@reflect +def returns_array_no_args(): + return cp.ones(3) + + +@reflect(array=None) +def returns_array_one_arg(n): + return cp.ones(n) + + +def test_set_global_output_type(): + gs = GlobalSettings() + assert gs.output_type is None + + cuml.set_global_output_type("cupy") + assert gs.output_type == "cupy" - # Ensure we reset the type at the end of the test cuml.set_global_output_type(None) + assert gs.output_type is None + with pytest.raises(ValueError, match="`output_type` must be one of"): + cuml.set_global_output_type("bad") -############################################################################### -# Tests # -############################################################################### +def test_using_output_type(): + gs = GlobalSettings() + assert gs.output_type is None -@pytest.mark.parametrize("input_type", global_input_types) + cuml.set_global_output_type("cupy") + assert gs.output_type == "cupy" + + with cuml.using_output_type("cudf"): + assert gs.output_type == "cudf" + assert gs.output_type == "cupy" + + with cuml.using_output_type(None): + assert gs.output_type is None + assert gs.output_type == "cupy" + + with pytest.raises(ValueError, match="`output_type` must be one of"): + with cuml.using_output_type("bad"): + pass + + +@pytest.mark.parametrize("input_type", OUTPUT_TYPES) @pytest.mark.filterwarnings("ignore::UserWarning") -def test_default_global_output_type(input_type): - dataset = get_small_dataset(input_type) +def test_default_output_type(input_type): + X = rand_array(input_type) + model = cuml.DBSCAN(eps=1.0, min_samples=1) + labels = model.fit_predict(X) + assert_output_type(labels, input_type) + assert_output_type(model.components_, input_type) + + +@pytest.mark.parametrize("input_type", OUTPUT_TYPES) +@pytest.mark.parametrize("output_type", OUTPUT_TYPES) +def test_estimator_output_type(input_type, output_type): + X = rand_array(input_type) + model = cuml.DBSCAN(eps=1.0, min_samples=1, output_type=output_type) + labels = model.fit_predict(X) + assert_output_type(labels, output_type) + assert_output_type(model.components_, output_type) + + +@pytest.mark.parametrize("input_type", OUTPUT_TYPES) +@pytest.mark.parametrize("output_type", OUTPUT_TYPES) +def test_global_output_type(input_type, output_type): + cuml.set_global_output_type(output_type) + + X = rand_array(input_type) + model = cuml.DBSCAN(eps=1.0, min_samples=1) + labels = model.fit_predict(X) + assert_output_type(labels, output_type) + assert_output_type(model.components_, output_type) + + +def test_global_overrides_estimator_output_type(): + cuml.set_global_output_type("numpy") + X = rand_array("pandas") + model = cuml.DBSCAN(eps=1.0, min_samples=1, output_type="cupy") + labels = model.fit_predict(X) + assert_output_type(labels, "numpy") + assert_output_type(model.components_, "numpy") + + +def test_global_input_with_estimator_output_type(): + cuml.set_global_output_type("input") + X = rand_array("pandas") + model = cuml.DBSCAN(eps=1.0, min_samples=1, output_type="cupy") + labels = model.fit_predict(X) + # The difference here is probably a bug, but it's been the behavior for a + # long time. Methods respect `estimator.output_type` if the global + # `output_type` is 'input', while attributes respect the global + # `output_type`. + assert_output_type(labels, "cupy") + assert_output_type(model.components_, "pandas") + + +@pytest.mark.parametrize( + "construct", + [ + pytest.param(lambda x, y: [x, 1, y], id="list"), + pytest.param(lambda x, y: (x, 1, y), id="tuple"), + pytest.param(lambda x, y: {"x": x, "y": y, "z": 1}, id="dict"), + pytest.param(lambda x, y: {"a": [(x, 1), (y, 2)]}, id="nested"), + ], +) +@pytest.mark.parametrize("output_type", ["input", "cupy", "cudf"]) +def test_convert_nested_outputs(construct, output_type): + cuml.set_global_output_type(output_type) + x = rand_array("numpy") + + @reflect(array="x") + def apply(func, x): + return func(x, x + 1) + + res = apply(construct, x) + sol = construct("x", "y") + + expected_type = "numpy" if output_type == "input" else output_type + + def check_nested_types(res, sol): + """Check types match, using `x` and `y` as placeholders for arrays""" + if sol in ("x", "y"): + assert_output_type(res, expected_type) + else: + assert type(res) is type(sol) + if isinstance(res, dict): + assert set(res) == set(sol) + for k in res: + check_nested_types(res[k], sol[k]) + elif isinstance(res, (tuple, list)): + assert len(res) == len(sol) + for r, s in zip(res, sol): + check_nested_types(r, s) + + check_nested_types(res, sol) + + +@pytest.mark.parametrize("sparse_type", ["cupy", "numpy", "cuml"]) +@pytest.mark.parametrize("output_type", [None, *OUTPUT_TYPES]) +def test_convert_sparse_outputs(sparse_type, output_type): + @reflect + def make_sparse(): + arr = cupyx.scipy.sparse.random(5, 5, random_state=42) + if sparse_type == "cupy": + return arr + elif sparse_type == "numpy": + return arr.get() + else: + return SparseCumlArray(arr) - dbscan_float = cuml.DBSCAN(eps=1.0, min_samples=1) - dbscan_float.fit(dataset) + cuml.set_global_output_type(output_type) + res = make_sparse() - res = dbscan_float.labels_ + if output_type == "cuml": + assert isinstance(res, SparseCumlArray) + elif output_type in [None, "input", "cupy", "cudf", "numba"]: + assert cupyx.scipy.sparse.issparse(res) + else: + assert scipy.sparse.issparse(res) - if input_type == "numba": - assert is_cuda_array(res) - elif not (input_type == "pandas" and cudf_pandas_active): - assert isinstance(res, test_output_types[input_type]) +@pytest.mark.parametrize("output_type", [None, *OUTPUT_TYPES]) +def test_functions(output_type): + cuml.set_global_output_type(output_type) + X = rand_array("numpy") -@pytest.mark.parametrize("input_type", global_input_types) -def test_global_output_type(global_output_type, input_type): - dataset = get_small_dataset(input_type) + # Reflected functions treat None/"input" the same + assert_output_type( + reflects_input(X), + "numpy" if output_type in (None, "input") else output_type, + ) - cuml.set_global_output_type(global_output_type) + # With no array argument functions default to 'cupy' unless + # a concrete type is configured + expected = "cupy" if output_type in (None, "input") else output_type + assert_output_type(returns_array_no_args(), expected) + assert_output_type(returns_array_one_arg(3), expected) - dbscan_float = cuml.DBSCAN(eps=1.0, min_samples=1) - dbscan_float.fit(dataset) - res = dbscan_float.labels_ +@pytest.mark.parametrize("output_type", [None, "input", "numpy"]) +def test_internal_calls(output_type): + @reflect(array="X") + def apply(func, X): + result = func(X) + # Internal calls return internal types by default + assert isinstance(result, CumlArray) - if global_output_type == "numba": - assert is_cuda_array(res) - else: - assert isinstance(res, test_output_types[global_output_type]) + with cuml.using_output_type("cupy"): + temp = func(X) + # Internal calls can configure output type to get + # something specific when needed + assert isinstance(temp, cp.ndarray) -@pytest.mark.parametrize("context_type", global_input_configs) -def test_output_type_context_mgr(global_output_type, context_type): - dataset = get_small_dataset("numba") + return result - test_type = "cupy" if global_output_type != "cupy" else "numpy" - cuml.set_global_output_type(test_type) + cuml.set_global_output_type(output_type) + X = rand_array("pandas") + res = apply(reflects_input, X) + expected = "pandas" if output_type in (None, "input") else output_type + assert_output_type(res, expected) - # use cuml context manager - with cuml.using_output_type(context_type): - dbscan_float = cuml.DBSCAN(eps=1.0, min_samples=1) - dbscan_float.fit(dataset) - res = dbscan_float.labels_ +def test_skip_true(): + """skip=True applies internal context, but skips output processing""" - if context_type == "numba": - assert is_cuda_array(res) - else: - assert isinstance(res, test_output_types[context_type]) + @reflect(skip=True) + def always_returns_numpy(func, X): + result = func(X) + # Internal calls return internal types by default + assert isinstance(result, CumlArray) - # use cuml again outside the context manager + return result.to_output("numpy") - dbscan_float = cuml.DBSCAN(eps=1.0, min_samples=1) - dbscan_float.fit(dataset) + cuml.set_global_output_type("cudf") + X = rand_array("pandas") + res = always_returns_numpy(reflects_input, X) + assert_output_type(res, "numpy") - res = dbscan_float.labels_ - assert isinstance(res, test_output_types[test_type]) +def test_reset_true(): + X = rand_array("numpy", shape=(10, 5)) + model = DummyEstimator().fit(X) + assert model.n_features_in_ == 5 + assert model._input_type == "numpy" -############################################################################### -# Utility Functions # -############################################################################### +def test_estimator_method_with_array_input(): + X = rand_array("numpy", shape=(10, 5)) + X2 = rand_array("cudf", shape=(10, 5)) + model = DummyEstimator().fit(X) -def get_small_dataset(output_type): - ary = [[1.0, 4.0, 4.0], [2.0, 2.0, 2.0], [5.0, 1.0, 1.0]] - ary = cp.asarray(ary) + # Reflects method input by default + assert_output_type(model.example(X2), "cudf") - if output_type == "numba": - return as_cuda_array(ary) + # Estimator output_type can override + model.output_type = "pandas" + assert_output_type(model.example(X2), "pandas") - elif output_type == "cupy": - return ary + # Global output type overrides + with cuml.using_output_type("cupy"): + assert_output_type(model.example(X2), "cupy") - elif output_type == "numpy": - return cp.asnumpy(ary) - elif output_type == "pandas": - return cudf.DataFrame(ary).to_pandas() +def test_estimator_method_with_no_array_input(): + X = rand_array("numpy", shape=(10, 5)) + model = DummyEstimator().fit(X) - else: - return cudf.DataFrame(ary) + # Reflects fit input by default + assert_output_type(model.example_no_args(), "numpy") + + # Estimator output_type can override + model.output_type = "cupy" + assert_output_type(model.example_no_args(), "cupy") + + # Global output type overrides + with cuml.using_output_type("pandas"): + assert_output_type(model.example_no_args(), "pandas") + + +def test_cuml_array_descriptor_type_in_internal_api(): + X = rand_array("numpy") + model = DummyEstimator().fit(X) + assert_output_type(model.X_, "numpy") + model.check_descriptor() + + +def test_array_descriptor_cache_behavior(): + X = rand_array("cupy") + model = DummyEstimator().fit(X) + assert_output_type(model.X_, "cupy") + # Instance is cached + assert model.X_ is model.X_ + assert len(model.__dict__["X_"].values) == 2 # cuml + cupy + + with cuml.using_output_type("pandas"): + assert_output_type(model.X_, "pandas") + # Instance is cached, but original cache isn't wiped + assert model.X_ is model.X_ + assert len(model.__dict__["X_"].values) == 3 # cuml + cupy + pandas + + msg = pickle.dumps(model) + model2 = pickle.loads(msg) + # Only one array is serialized when pickled, and the cache is reset + assert b"pandas" not in msg + assert_output_type(model2.X_, "cupy") + assert len(model2.__dict__["X_"].values) == 2 # cuml + cupy From fd6bddddfe73c9d3f52b300aa31fe90a008a7717 Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Wed, 26 Nov 2025 16:45:40 -0600 Subject: [PATCH 26/28] Fixup for SpectralClustering change that snuck in on rebase --- .../cuml/cuml/cluster/spectral_clustering.pyx | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/python/cuml/cuml/cluster/spectral_clustering.pyx b/python/cuml/cuml/cluster/spectral_clustering.pyx index 5658a0d809..46ad90391a 100644 --- a/python/cuml/cuml/cluster/spectral_clustering.pyx +++ b/python/cuml/cuml/cluster/spectral_clustering.pyx @@ -2,7 +2,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # - import warnings import cupy as cp @@ -56,7 +55,7 @@ cdef extern from "cuml/cluster/spectral_clustering.hpp" \ device_vector_view[int, int] labels) except + -@cuml.internals.api_return_array(get_output_type=True) +@cuml.internals.reflect def spectral_clustering(X, *, int n_clusters=8, @@ -394,6 +393,7 @@ class SpectralClustering(Base): "affinity", ] + @cuml.internals.reflect def fit_predict(self, X, y=None) -> CumlArray: """Perform spectral clustering on ``X`` and return cluster labels. @@ -414,19 +414,10 @@ class SpectralClustering(Base): labels : cupy.ndarray of shape (n_samples,) Cluster labels. """ - self.labels_ = spectral_clustering( - X, - n_clusters=self.n_clusters, - n_components=self.n_components, - random_state=self.random_state, - n_neighbors=self.n_neighbors, - n_init=self.n_init, - eigen_tol=self.eigen_tol, - affinity=self.affinity, - handle=self.handle - ) + self.fit(X, y=y) return self.labels_ + @cuml.internals.reflect(reset=True) def fit(self, X, y=None) -> "SpectralClustering": """Perform spectral clustering on ``X``. @@ -447,5 +438,15 @@ class SpectralClustering(Base): self : object Returns the instance itself. """ - self.fit_predict(X, y) + self.labels_ = spectral_clustering( + X, + n_clusters=self.n_clusters, + n_components=self.n_components, + random_state=self.random_state, + n_neighbors=self.n_neighbors, + n_init=self.n_init, + eigen_tol=self.eigen_tol, + affinity=self.affinity, + handle=self.handle + ) return self From 6e7837071823e151ef38a23f16beca43bd706bbd Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Mon, 1 Dec 2025 16:37:18 -0600 Subject: [PATCH 27/28] Remove `type_utils` This was now dead code. --- .../cuml/feature_extraction/_vectorizers.py | 3 +- python/cuml/cuml/internals/type_utils.py | 28 ------------------- 2 files changed, 2 insertions(+), 29 deletions(-) delete mode 100644 python/cuml/cuml/internals/type_utils.py diff --git a/python/cuml/cuml/feature_extraction/_vectorizers.py b/python/cuml/cuml/feature_extraction/_vectorizers.py index 84bc3f290a..0f9a1b64bb 100644 --- a/python/cuml/cuml/feature_extraction/_vectorizers.py +++ b/python/cuml/cuml/feature_extraction/_vectorizers.py @@ -18,7 +18,8 @@ csr_row_normalize_l2, ) from cuml.feature_extraction._stop_words import ENGLISH_STOP_WORDS -from cuml.internals.type_utils import CUPY_SPARSE_DTYPES + +CUPY_SPARSE_DTYPES = [cp.float32, cp.float64, cp.complex64, cp.complex128] def min_signed_type(n): diff --git a/python/cuml/cuml/internals/type_utils.py b/python/cuml/cuml/internals/type_utils.py deleted file mode 100644 index 0b94fbddf7..0000000000 --- a/python/cuml/cuml/internals/type_utils.py +++ /dev/null @@ -1,28 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 2020-2025, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 -# -import functools -import typing - -import cupy as cp - -CUPY_SPARSE_DTYPES = [cp.float32, cp.float64, cp.complex64, cp.complex128] - -# Use _DecoratorType as a type variable for decorators. See: -# https://github.com/python/mypy/pull/8336/files#diff-eb668b35b7c0c4f88822160f3ca4c111f444c88a38a3b9df9bb8427131538f9cR260 -_DecoratorType = typing.TypeVar( - "_DecoratorType", bound=typing.Callable[..., typing.Any] -) - - -def wraps_typed( - wrapped: _DecoratorType, - assigned=("__doc__", "__annotations__"), - updated=functools.WRAPPER_UPDATES, -) -> typing.Callable[[_DecoratorType], _DecoratorType]: - """ - Typed version of `functools.wraps`. Allows decorators to retain their - return type. - """ - return functools.wraps(wrapped=wrapped, assigned=assigned, updated=updated) From 2e13e1d7e588d48b721bf0cdc3c1514555c17ecd Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Mon, 1 Dec 2025 16:39:14 -0600 Subject: [PATCH 28/28] Remove `base_return_types` This was fully dead code --- .../cuml/cuml/internals/base_return_types.py | 90 ------------------- 1 file changed, 90 deletions(-) delete mode 100644 python/cuml/cuml/internals/base_return_types.py diff --git a/python/cuml/cuml/internals/base_return_types.py b/python/cuml/cuml/internals/base_return_types.py deleted file mode 100644 index 7b53f559e7..0000000000 --- a/python/cuml/cuml/internals/base_return_types.py +++ /dev/null @@ -1,90 +0,0 @@ -# -# SPDX-FileCopyrightText: Copyright (c) 2022-2025, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 -# - -import typing - -import cuml.internals -from cuml.internals.array import CumlArray -from cuml.internals.array_sparse import SparseCumlArray - - -def _process_generic(gen_type): - # Check if the type is not a generic. If not, must return "generic" if - # subtype is CumlArray otherwise None - if not isinstance(gen_type, typing._GenericAlias): - if issubclass(gen_type, CumlArray): - return "generic" - - # We don't handle SparseCumlArray at this time - if issubclass(gen_type, SparseCumlArray): - raise NotImplementedError( - "Generic return types with SparseCumlArray are not supported " - "at this time" - ) - - # Otherwise None (keep processing) - return None - - # Its a generic type by this point. Support Union, Tuple, Dict and List - supported_gen_types = [ - tuple, - dict, - list, - typing.Union, - ] - - if gen_type.__origin__ in supported_gen_types: - # Check for a CumlArray type in the args - for arg in gen_type.__args__: - inner_type = _process_generic(arg) - - if inner_type is not None: - return inner_type - else: - raise NotImplementedError("Unknow generic type: {}".format(gen_type)) - - return None - - -def _get_base_return_type(class_name, attr): - if ( - not hasattr(attr, "__annotations__") - or "return" not in attr.__annotations__ - ): - return None - - try: - type_hints = typing.get_type_hints(attr) - - if "return" in type_hints: - ret_type = type_hints["return"] - - is_generic = isinstance(ret_type, typing._GenericAlias) - - if is_generic: - return _process_generic(ret_type) - elif issubclass(ret_type, CumlArray): - return "array" - elif issubclass(ret_type, SparseCumlArray): - return "sparsearray" - elif issubclass(ret_type, cuml.internals.base.Base): - return "base" - else: - return None - except NameError: - # A NameError is raised if the return type is the same as the - # type being defined (which is incomplete). Check that here and - # return base if the name matches - # Cython 3 changed to preferring types rather than strings for - # annotations. Strings end up wrapped in an extra layer of quotes, - # which we have to replace here. - if attr.__annotations__["return"].replace("'", "") == class_name: - return "base" - except Exception: - raise AssertionError( - f"Failed to determine return type for {attr} (class = '${class_name}'). This is a bug in cuML, please report it." - ) - - return None