fixup: Avoid deploying to non-kind cluster without base accessible uri

Author: maru-ava

.github/workflows/ci.yml

@@ -255,6 +255,7 @@ jobs:
       - uses: ./.github/actions/run-monitored-tmpnet-cmd
         with:
           run: ./scripts/run_task.sh test-load-kube
+          runtime: kube
           artifact_prefix: load-kube
           prometheus_username: ${{ secrets.PROMETHEUS_ID || '' }}
           prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }}

scripts/tests.e2e.kube.sh

@@ -23,4 +23,17 @@ else
   XSVM_IMAGE="${XSVM_IMAGE}" AVALANCHEGO_IMAGE="${AVALANCHEGO_IMAGE}" bash -x ./scripts/build_xsvm_image.sh
 fi
 
-bash -x ./scripts/tests.e2e.sh --runtime=kube --kube-image="${XSVM_IMAGE}" "$@"
+# Determine kubeconfig context to use
+KUBECONFIG_CONTEXT=""
+
+# Check if --kubeconfig-context is already provided in arguments
+if [[ "$*" =~ --kubeconfig-context ]]; then
+    # User provided a context, use it as-is
+    echo "Using provided kubeconfig context from arguments"
+else
+    # Default to the RBAC context
+    KUBECONFIG_CONTEXT="--kubeconfig-context=kind-kind-tmpnet"
+    echo "Defaulting to limited-permission context 'kind-kind-tmpnet' to test RBAC Role permissions"
+fi
+
+bash -x ./scripts/tests.e2e.sh --runtime=kube --kube-image="${XSVM_IMAGE}" "$KUBECONFIG_CONTEXT" "$@"

tests/fixture/tmpnet/flags/kube_runtime.go

@@ -92,7 +92,7 @@ func (v *kubeRuntimeVars) register(stringVar varFunc[string], uintVar varFunc[ui
 	stringVar(
 		&v.baseAccessibleURI,
 		"kube-base-accessible-uri",
-		"http://localhost:30791",
+		"",
 		kubeDocPrefix+"The base URI for constructing node URIs when running outside of the cluster hosting nodes",
 	)
 }
@@ -110,7 +110,15 @@ func (v *kubeRuntimeVars) getKubeRuntimeConfig() (*tmpnet.KubeRuntimeConfig, err
 	if v.useExclusiveScheduling && (len(v.schedulingLabelKey) == 0 || len(v.schedulingLabelValue) == 0) {
 		return nil, errKubeSchedulingLabelRequired
 	}
-	if !tmpnet.IsRunningInCluster() && len(v.baseAccessibleURI) == 0 {
+	baseAccessibleURI := v.baseAccessibleURI
+	if strings.HasPrefix(v.config.Context, "kind-kind") && len(baseAccessibleURI) == 0 {
+		// Use the base uri expected for the kind cluster deployed by tmpnet. Not supplying this as a default
+		// ensures that an explicit value is required for non-kind clusters.
+		//
+		// TODO(marun) Log why this value is being used. This will require passing a log through the call chain.
+		baseAccessibleURI = "http://localhost:30791"
+	}
+	if !tmpnet.IsRunningInCluster() && len(baseAccessibleURI) == 0 {
 		return nil, errKubeBaseAccessibleURIRequired
 	}
 	return &tmpnet.KubeRuntimeConfig{

tests/fixture/tmpnet/kube_runtime.go

@@ -1020,14 +1020,14 @@ func (p *KubeRuntime) waitForIngressReadiness(ctx context.Context, serviceName s
 		return err
 	}
 
-	// Wait for the ingress to exist and service endpoints to be available
+	// Wait for the ingress to exist, be processed by the controller, and service endpoints to be available
 	err = wait.PollUntilContextCancel(
 		ctx,
 		statusCheckInterval,
 		true, // immediate
 		func(ctx context.Context) (bool, error) {
-			// Check if ingress exists
-			_, err := clientset.NetworkingV1().Ingresses(namespace).Get(ctx, serviceName, metav1.GetOptions{})
+			// Check if ingress exists and is processed by the controller
+			ingress, err := clientset.NetworkingV1().Ingresses(namespace).Get(ctx, serviceName, metav1.GetOptions{})
 			if apierrors.IsNotFound(err) {
 				log.Verbo("waiting for Ingress to be created",
 					zap.String("nodeID", nodeID),
@@ -1046,6 +1046,37 @@ func (p *KubeRuntime) waitForIngressReadiness(ctx context.Context, serviceName s
 				return false, nil
 			}
 
+			// Check if ingress controller has processed the ingress
+			// The ingress controller should populate the Status.LoadBalancer.Ingress field
+			// when it has successfully processed and exposed the ingress
+			hasIngressIP := len(ingress.Status.LoadBalancer.Ingress) > 0
+			if !hasIngressIP {
+				log.Verbo("waiting for Ingress controller to process and expose the Ingress",
+					zap.String("nodeID", nodeID),
+					zap.String("namespace", namespace),
+					zap.String("ingress", serviceName),
+				)
+				return false, nil
+			}
+
+			// Validate that at least one ingress has an IP or hostname
+			hasValidIngress := false
+			for _, ing := range ingress.Status.LoadBalancer.Ingress {
+				if ing.IP != "" || ing.Hostname != "" {
+					hasValidIngress = true
+					break
+				}
+			}
+
+			if !hasValidIngress {
+				log.Verbo("waiting for Ingress controller to assign IP or hostname",
+					zap.String("nodeID", nodeID),
+					zap.String("namespace", namespace),
+					zap.String("ingress", serviceName),
+				)
+				return false, nil
+			}
+
 			// Check if service endpoints are available
 			endpoints, err := clientset.CoreV1().Endpoints(namespace).Get(ctx, serviceName, metav1.GetOptions{})
 			if apierrors.IsNotFound(err) {
@@ -1084,7 +1115,7 @@ func (p *KubeRuntime) waitForIngressReadiness(ctx context.Context, serviceName s
 				return false, nil
 			}
 
-			log.Debug("Ingress and Service endpoints are ready",
+			log.Debug("Ingress is exposed by controller and Service endpoints are ready",
 				zap.String("nodeID", nodeID),
 				zap.String("namespace", namespace),
 				zap.String("ingress", serviceName),

tests/log.go

@@ -25,5 +25,6 @@ func LoggerForFormat(prefix string, rawLogFormat string) (logging.Logger, error)
 	if err != nil {
 		return nil, err
 	}
-	return logging.NewLogger(prefix, logging.NewWrappedCore(logging.Verbo, writeCloser, logFormat.ConsoleEncoder())), nil
+	// TODO(marun) Make the log level configurable
+	return logging.NewLogger(prefix, logging.NewWrappedCore(logging.Debug, writeCloser, logFormat.ConsoleEncoder())), nil
 }