Fix KFTO-SDK MNIST test for disconnected

Author: abhijeet-dhumal

tests/kfto/resources/kfto_sdk_mnist.py

@@ -9,7 +9,7 @@ def train_func():
     from minio import Minio
     import shutil
     import gzip
-
+    from urllib.parse import urlparse
 
     # [1] Setup PyTorch DDP. Distributed environment will be set automatically by Training Operator.
     dist.init_process_group(backend="nccl" if torch.cuda.is_available() else "gloo")
@@ -57,13 +57,20 @@ def forward(self, x):
     secret_key = "{{.StorageBucketSecretKey}}"
     bucket_name = "{{.StorageBucketName}}"
     prefix = "{{.StorageBucketMnistDir}}"
-    if with_aws != "true":
+
+    # Sanitize endpoint to remove any scheme or path.
+    parsed = urlparse(endpoint)
+    # If the endpoint URL contains a scheme, netloc contains the host and optional port.
+    endpoint = parsed.netloc if parsed.netloc else parsed.path
+    secure = parsed.scheme == "https"
+
+    if with_aws == "true":
         client = Minio(
             endpoint,
             access_key=access_key,
             secret_key=secret_key,
             cert_check=False,
-            secure=False,  #TODO
+            secure=secure,
         )
 
         if not os.path.exists(dataset_dir):

tests/kfto/resources/mnist_kfto.ipynb

@@ -61,10 +61,11 @@
     "   resources_per_worker={\"gpu\": num_gpus},\n",
     "   base_image=training_image,\n",
     "   packages_to_install=[\"torchvision==0.19.0\",\"minio==7.2.13\"],\n",
+    "   pip_index_url= os.environ.get(\"PIP_INDEX_URL\"),\n",
     "   env_vars={\n",
     "      \"NCCL_DEBUG\": \"INFO\", \n",
     "      \"TORCH_DISTRIBUTED_DEBUG\": \"DETAIL\", \n",
-    "      \"PIP_INDEX_URL\": os.environ.get(\"PIP_INDEX_URL\"),\n",
+    "      \"DEFAULT_PIP_INDEX_URL\": os.environ.get(\"PIP_INDEX_URL\"),\n",
     "      \"PIP_TRUSTED_HOST\": os.environ.get(\"PIP_TRUSTED_HOST\")\n",
     "   }\n",
     ")"
@@ -78,7 +79,8 @@
    "outputs": [],
    "source": [
     "while not tc.is_job_succeeded(name=\"pytorch-ddp\", namespace=namespace): \n",
-    "    time.sleep(1)"
+    "    time.sleep(1)\n",
+    "print(\"PytorchJob Succeeded!\")"
    ]
   },
   {