fix: fix failing action in Merlin CI Workflow

.github/workflows/merlin.yml

@@ -43,16 +43,19 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
     env:
       PIPENV_DEFAULT_PYTHON_VERSION: ${{ matrix.python-version }}
+      PIPENV_VENV_IN_PROJECT: true
     steps:
       - uses: actions/checkout@v4
       - name: Set up Google Cloud SDK
         uses: google-github-actions/setup-gcloud@v2
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@stable
       - uses: actions/cache@v4
         with:
           path: ~/.cache/pip
@@ -77,9 +80,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
     env:
       PIPENV_DEFAULT_PYTHON_VERSION: ${{ matrix.python-version }}
+      PIPENV_VENV_IN_PROJECT: true
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -109,7 +113,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
     env:
       PIPENV_DEFAULT_PYTHON_VERSION: ${{ matrix.python-version }}
     steps:

.github/workflows/release.yml

@@ -12,6 +12,7 @@ on:
         required: true
       pypi_password:
         required: true
+  workflow_dispatch:
 
 env:
   DOCKER_REGISTRY: ghcr.io
@@ -101,7 +102,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
     needs:
       - publish-python-sdk
     steps:

python/batch-predictor/Pipfile

@@ -7,4 +7,5 @@ verify_ssl = true
 merlin-batch-predictor = {editable = true,extras = ["test"],path = "."}
 
 [packages]
+cython = ">=3.0"
 merlin-batch-predictor = {extras = ["test"], file = ".", editable = true}

python/batch-predictor/requirements.txt

@@ -2,6 +2,6 @@ cloudpickle==2.0.0
 findspark==2.0.1
 merlin-sdk==0.0.0
 mlflow==1.26.1
-pyarrow>=0.14.1,<=17.0.0
+pyarrow==19.0.0
 pyspark==3.1.3
 setuptools<75

python/batch-predictor/requirements_test.txt

@@ -1,8 +1,8 @@
-pytest
+pytest>=8.1
 pytest-cov
 mypy
 google-cloud-bigquery
-scikit-learn>=1.1.2
-joblib>=0.13.0,<1.2.0  # >=1.2.0 upon upgrade of kserve's version
+scikit-learn>=1.3.1
+joblib>=1.2.0
 mypy-protobuf>=1.19
 types-PyYAML

python/batch-predictor/setup.py

@@ -12,14 +12,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import imp
+import importlib.util
 import os
 
 from setuptools import find_packages, setup
 
-version = imp.load_source(
-    "merlinpyspark.version", os.path.join("merlinpyspark", "version.py")
-).VERSION
+# get version from version.py
+spec = importlib.util.spec_from_file_location(
+    "merlinpyspark.version", os.path.join("merlinpyspark","version.py")
+)
+
+v_module = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(v_module)
+
+version = v_module.VERSION
 
 with open("requirements.txt") as f:
     REQUIRE = f.read().splitlines()
@@ -34,7 +40,7 @@
     description="Base PySpark application for running Merlin prediction batch job",
     long_description=open("README.md").read(),
     long_description_content_type="text/markdown",
-    python_requires=">=3.8,<3.11",
+    python_requires=">=3.9,<3.14",
     packages=find_packages(exclude=["test"]),
     install_requires=REQUIRE,
     tests_require=TESTS_REQUIRE,

python/observation-publisher/requirements.txt

@@ -30,7 +30,7 @@ botocore==1.35.39
     #   s3transfer
 cachetools==5.3.2
     # via google-auth
-caraml-auth-google==0.0.0.post7
+caraml-auth-google==0.0.0.post16.dev0
     # via merlin-sdk
 caraml-upi-protos==1.0.0
     # via
@@ -150,7 +150,7 @@ mlflow==1.26.1
     # via merlin-sdk
 mypy-extensions==1.0.0
     # via typing-inspect
-numpy==1.23.5
+numpy>=1.26.4,<2.0
     # via
     #   merlin-sdk
     #   mlflow

python/pyfunc-server/Pipfile

@@ -5,6 +5,7 @@ verify_ssl = true
 
 [packages]
 pyfuncserver = {editable = true, extras = ["test"], path = "."}
+cython = ">=3.0"
 merlin-pyfunc-server = {extras = ["test"], file = ".", editable = true}
 
 [dev-packages]

python/pyfunc-server/requirements.txt

@@ -1,7 +1,7 @@
 argparse>=1.4.0
 caraml-upi-protos>=0.3.4
 cloudpickle==2.0.0
-confluent-kafka==2.3.0
+confluent-kafka>=2.6.0
 grpcio-health-checking
 grpcio-reflection
 merlin-sdk==0.0.0

python/pyfunc-server/setup.py

@@ -12,29 +12,35 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import imp
+import importlib.util
 import os
 
 from setuptools import find_packages, setup
 
-version = imp.load_source(
-    "pyfuncserver.version", os.path.join("pyfuncserver", "version.py")
-).VERSION
+# get version from version.py
+spec = importlib.util.spec_from_file_location(
+    "pyfuncserver.version", os.path.join("pyfuncserver","version.py")
+)
+
+v_module = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(v_module)
+
+version = v_module.VERSION
 
 with open("requirements.txt") as f:
     REQUIRE = f.read().splitlines()
 
 TESTS_REQUIRE = [
-    "joblib>=0.13.0,<1.2.0",  # >=1.2.0 upon upgrade of kserve's version
-    "mypy",
-    "pytest-benchmark",
+    "joblib>=1.2.0",
+    "mypy>=1.5.4",
+    "pytest>=8.1",
+    "pytest-benchmark>=5.1.0",
     "pytest-tornasync",
-    "pytest",
-    "requests",
-    "scikit-learn>=1.1.2",
+    "requests>=2.31.0",
+    "scikit-learn>=1.3.1",
     "types-protobuf",
     "types-requests",
-    "xgboost==1.6.2",
+    "xgboost>=1.7.6",
 ]
 
 setup(
@@ -44,7 +50,7 @@
     description="Model Server implementation for Merlin PyFunc model",
     long_description=open("README.md").read(),
     long_description_content_type="text/markdown",
-    python_requires=">=3.8,<3.11",
+    python_requires=">=3.8,<3.14",
     packages=find_packages(exclude=["test"]),
     install_requires=REQUIRE,
     tests_require=TESTS_REQUIRE,

python/sdk/requirements.txt

@@ -8,13 +8,15 @@ dataclasses-json>=0.5.2  # allow Flyte version 1.2.0 or above to import Merlin S
 docker<=6.1.3
 GitPython>=3.1.40
 google-cloud-storage>=1.19.0
-protobuf>=3.12.0,<5.0.0  # Determined by the mlflow dependency
+protobuf>=4.21.6,<5
 mlflow==1.26.1
 PyPrind>=2.11.2
 python_dateutil>=2.5.3
 PyYAML>=5.4
 six>=1.10
 urllib3>=1.26
-numpy<=1.23.5  # Temporary pin numpy due to https://numpy.org/doc/stable/release/1.20.0-notes.html#numpy-1-20-0-release-notes
-caraml-auth-google==0.0.0.post7
-pydantic==2.5.3
+numpy>=1.26.4
+caraml-auth-google==0.0.0.post16.dev0
+pydantic==2.5.3
+grpcio-tools>=1.50.0,<1.63
+grpcio>=1.60.1

python/sdk/requirements_test.txt

@@ -13,6 +13,6 @@ types-python-dateutil
 types-PyYAML
 types-six
 types-protobuf
-urllib3-mock>=0.3.3
 xarray
-xgboost==1.6.2
+xgboost==1.6.2
+urllib3-mock>=0.3.3

python/sdk/setup.py

@@ -13,14 +13,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import imp
+import importlib.util
 import os
 
 from setuptools import find_packages, setup
 
-version = imp.load_source(
-    "merlin.version", os.path.join("merlin", "version.py")
-).VERSION
+# get version from version.py
+spec = importlib.util.spec_from_file_location(
+    "sdk.version", os.path.join("merlin/version.py")
+)
+
+v_module = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(v_module)
+
+version = v_module.VERSION
 
 with open("requirements.txt") as f:
     REQUIRE = f.read().splitlines()
@@ -41,7 +47,7 @@
     setup_requires=["setuptools_scm"],
     tests_require=TESTS_REQUIRE,
     extras_require={"test": TESTS_REQUIRE},
-    python_requires=">=3.8,<3.11",
+    python_requires=">=3.9,<3.14",
     long_description=open("README.md").read(),
     long_description_content_type="text/markdown",
     entry_points="""

python/sdk/test/client_test.py

@@ -427,4 +427,4 @@ def test_get_default_environment(mock_url, api_client, mock_oauth, use_google_oa
     )
     env = client.get_default_environment()
 
-    assert env is None
+    assert env is None

python/sdk/test/conftest.py

@@ -258,4 +258,4 @@ def mock_oauth():
     """,
         status=200,
         content_type="application/json",
-    )
+    )

python/sdk/test/merlin_test.py

@@ -263,4 +263,4 @@ def _mock_list_environment_call():
         body=json.dumps([env_1.to_dict(), env_2.to_dict()]),
         status=200,
         content_type="application/json",
-    )
+    )

python/sdk/test/model_test.py

@@ -1847,4 +1847,4 @@ def test_model_deletion(self, model):
         )
         response = model.delete_model()
 
-        assert response == 1
+        assert response == 1