[tmpnet] Enable externally accessible URIs for kube-hosted nodes

.github/workflows/ci.yml

@@ -255,6 +255,7 @@ jobs:
       - uses: ./.github/actions/run-monitored-tmpnet-cmd
         with:
           run: ./scripts/run_task.sh test-load-kube-kind
+          runtime: kube
           artifact_prefix: load-kube
           prometheus_username: ${{ secrets.PROMETHEUS_ID || '' }}
           prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }}

flake.nix

@@ -48,7 +48,6 @@
             k9s                                        # Kubernetes TUI
             kind                                       # Kubernetes-in-Docker
             kubernetes-helm                            # Helm CLI (Kubernetes package manager)
-            self.packages.${system}.kind-with-registry # Script installing kind configured with a local registry
 
             # Linters
             shellcheck
@@ -65,32 +64,11 @@
             # macOS-specific frameworks
             darwin.apple_sdk.frameworks.Security
           ];
-        };
-      });
-
-      # Package to install the kind-with-registry script
-      packages = forAllSystems ({ pkgs }: {
-        kind-with-registry = pkgs.stdenv.mkDerivation {
-          pname = "kind-with-registry";
-          version = "1.0.0";
 
-          src = pkgs.fetchurl {
-            url = "https://raw.githubusercontent.com/kubernetes-sigs/kind/7cb9e6be25b48a0e248097eef29d496ab1a044d0/site/static/examples/kind-with-registry.sh";
-            sha256 = "0gri0x0ygcwmz8l4h6zzsvydw8rsh7qa8p5218d4hncm363i81hv";
-          };
-
-          phases = [ "installPhase" ];
-
-          installPhase = ''
-            mkdir -p $out/bin
-            install -m755 $src $out/bin/kind-with-registry.sh
+          # Add scripts/ directory to PATH so kind-with-registry.sh is accessible
+          shellHook = ''
+            export PATH="$PWD/scripts:$PATH"
           '';
-
-          meta = with pkgs.lib; {
-            description = "Script to set up kind with a local registry";
-            license = licenses.mit;
-            maintainers = with maintainers; [ "maru-ava" ];
-          };
         };
       });
     };

scripts/kind-with-registry.sh

@@ -0,0 +1,90 @@
+#!/bin/sh
+# Based on https://raw.githubusercontent.com/kubernetes-sigs/kind/7cb9e6be25b48a0e248097eef29d496ab1a044d0/site/static/examples/kind-with-registry.sh
+# Original work Copyright 2019 The Kubernetes Authors
+# Modifications Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved.
+# See the file LICENSE for licensing terms.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# TODO(marun) Migrate this script to golang
+set -o errexit
+
+# 1. Create registry container unless it already exists
+reg_name='kind-registry'
+reg_port='5001'
+if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then
+  docker run \
+    -d --restart=always -p "127.0.0.1:${reg_port}:5000" --network bridge --name "${reg_name}" \
+    registry:2
+fi
+
+# 2. Create kind cluster with containerd registry config dir enabled
+# TODO: kind will eventually enable this by default and this patch will
+# be unnecessary.
+#
+# See:
+# https://github.com/kubernetes-sigs/kind/issues/2875
+# https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration
+# See: https://github.com/containerd/containerd/blob/main/docs/hosts.md
+cat <<EOF | kind create cluster --config=-
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+containerdConfigPatches:
+- |-
+  [plugins."io.containerd.grpc.v1.cri".registry]
+    config_path = "/etc/containerd/certs.d"
+nodes:
+- role: control-plane
+  extraPortMappings:
+  # Exposing a nodeport for nginx ingress is the reason this script needed to be copied and customized
+  # This port must match the value used to deploy the nginx controller by tests/fixture/tmpnet/start-kind-cluster.go
+  - containerPort: 30791
+    hostPort: 30791
+    protocol: TCP
+EOF
+
+# 3. Add the registry config to the nodes
+#
+# This is necessary because localhost resolves to loopback addresses that are
+# network-namespace local.
+# In other words: localhost in the container is not localhost on the host.
+#
+# We want a consistent name that works from both ends, so we tell containerd to
+# alias localhost:${reg_port} to the registry container when pulling images
+REGISTRY_DIR="/etc/containerd/certs.d/localhost:${reg_port}"
+for node in $(kind get nodes); do
+  docker exec "${node}" mkdir -p "${REGISTRY_DIR}"
+  cat <<EOF | docker exec -i "${node}" cp /dev/stdin "${REGISTRY_DIR}/hosts.toml"
+[host."http://${reg_name}:5000"]
+EOF
+done
+
+# 4. Connect the registry to the cluster network if not already connected
+# This allows kind to bootstrap the network but ensures they're on the same network
+if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name}")" = 'null' ]; then
+  docker network connect "kind" "${reg_name}"
+fi
+
+# 5. Document the local registry
+# https://github.com/kubernetes/enhancements/tree/master/keps/sig-cluster-lifecycle/generic/1755-communicating-a-local-registry
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: local-registry-hosting
+  namespace: kube-public
+data:
+  localRegistryHosting.v1: |
+    host: "localhost:${reg_port}"
+    help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
+EOF

scripts/tests.e2e.kube.sh

@@ -23,4 +23,17 @@ else
   XSVM_IMAGE="${XSVM_IMAGE}" AVALANCHEGO_IMAGE="${AVALANCHEGO_IMAGE}" bash -x ./scripts/build_xsvm_image.sh
 fi
 
-bash -x ./scripts/tests.e2e.sh --runtime=kube --kube-image="${XSVM_IMAGE}" "$@"
+# Determine kubeconfig context to use
+KUBECONFIG_CONTEXT=""
+
+# Check if --kubeconfig-context is already provided in arguments
+if [[ "$*" =~ --kubeconfig-context ]]; then
+    # User provided a context, use it as-is
+    echo "Using provided kubeconfig context from arguments"
+else
+    # Default to the RBAC context
+    KUBECONFIG_CONTEXT="--kubeconfig-context=kind-kind-tmpnet"
+    echo "Defaulting to limited-permission context 'kind-kind-tmpnet' to test RBAC Role permissions"
+fi
+
+bash -x ./scripts/tests.e2e.sh --runtime=kube --kube-image="${XSVM_IMAGE}" "$KUBECONFIG_CONTEXT" "$@"

tests/antithesis/config.go

@@ -102,12 +102,12 @@ func configForNewNetwork(
 	c := &Config{
 		Duration: duration,
 	}
-	localURIs := testEnv.GetNodeURIs()
-	c.URIs = make(CSV, len(localURIs))
-	for i, nodeURI := range localURIs {
+	network := testEnv.GetNetwork()
+	uris := network.GetNodeURIs()
+	c.URIs = make(CSV, len(uris))
+	for i, nodeURI := range uris {
 		c.URIs[i] = nodeURI.URI
 	}
-	network := testEnv.GetNetwork()
 	c.ChainIDs = make(CSV, len(network.Subnets))
 	for i, subnet := range network.Subnets {
 		c.ChainIDs[i] = subnet.Chains[0].ChainID.String()

tests/e2e/c/dynamic_fees.go

@@ -82,7 +82,7 @@ var _ = e2e.DescribeCChain("[Dynamic Fees]", func() {
 		node := privateNetwork.Nodes[0]
 		nodeURI := tmpnet.NodeURI{
 			NodeID: node.NodeID,
-			URI:    e2e.GetLocalURI(tc, node),
+			URI:    node.GetAccessibleURI(),
 		}
 		ethClient := e2e.NewEthClient(tc, nodeURI)
 

tests/e2e/p/interchain_workflow.go

@@ -110,7 +110,7 @@ var _ = e2e.DescribePChain("[Interchain Workflow]", ginkgo.Label(e2e.UsesCChainL
 		e2e.WaitForHealthy(tc, node)
 
 		tc.By("retrieving new node's id and pop")
-		uri := e2e.GetLocalURI(tc, node)
+		uri := node.GetAccessibleURI()
 		infoClient := info.NewClient(uri)
 		nodeID, nodePOP, err := infoClient.GetNodeID(tc.DefaultContext())
 		require.NoError(err)

tests/e2e/p/l1.go

@@ -185,8 +185,10 @@ var _ = e2e.DescribePChain("[L1]", func() {
 		var (
 			networkID           = env.GetNetwork().GetNetworkID()
 			genesisPeerMessages = buffer.NewUnboundedBlockingDeque[p2pmessage.InboundMessage](1)
-			stakingAddress      = e2e.GetLocalStakingAddress(tc, subnetGenesisNode)
 		)
+		stakingAddress, cancel, err := subnetGenesisNode.GetAccessibleStakingAddress(tc.DefaultContext())
+		require.NoError(err)
+		tc.DeferCleanup(cancel)
 		genesisPeer, err := peer.StartTestPeer(
 			tc.DefaultContext(),
 			stakingAddress,
@@ -202,7 +204,7 @@ var _ = e2e.DescribePChain("[L1]", func() {
 		)
 		require.NoError(err)
 
-		subnetGenesisNodeURI := e2e.GetLocalURI(tc, subnetGenesisNode)
+		subnetGenesisNodeURI := subnetGenesisNode.GetAccessibleURI()
 
 		address := []byte{}
 		tc.By("issuing a ConvertSubnetToL1Tx", func() {

tests/e2e/p/staking_rewards.go

@@ -65,13 +65,13 @@ var _ = ginkgo.Describe("[Staking Rewards]", func() {
 		e2e.WaitForHealthy(tc, betaNode)
 
 		tc.By("retrieving alpha node id and pop")
-		alphaNodeURI := e2e.GetLocalURI(tc, alphaNode)
+		alphaNodeURI := alphaNode.GetAccessibleURI()
 		alphaInfoClient := info.NewClient(alphaNodeURI)
 		alphaNodeID, alphaPOP, err := alphaInfoClient.GetNodeID(tc.DefaultContext())
 		require.NoError(err)
 
 		tc.By("retrieving beta node id and pop")
-		betaNodeURI := e2e.GetLocalURI(tc, betaNode)
+		betaNodeURI := betaNode.GetAccessibleURI()
 		betaInfoClient := info.NewClient(betaNodeURI)
 		betaNodeID, betaPOP, err := betaInfoClient.GetNodeID(tc.DefaultContext())
 		require.NoError(err)

tests/e2e/p/validator_sets.go

@@ -81,9 +81,9 @@ var _ = e2e.DescribePChain("[Validator Sets]", func() {
 		require.NoError(err)
 
 		tc.By("checking that validator sets are equal across all heights for all nodes", func() {
-			localURIs := env.GetNodeURIs()
-			pvmClients := make([]*platformvm.Client, len(localURIs))
-			for i, nodeURI := range localURIs {
+			nodeURIs := network.GetNodeURIs()
+			pvmClients := make([]*platformvm.Client, len(nodeURIs))
+			for i, nodeURI := range nodeURIs {
 				pvmClients[i] = platformvm.NewClient(nodeURI.URI)
 				// Ensure that the height of the target node is at least the expected height
 				tc.Eventually(

tests/e2e/vms/xsvm.go

@@ -77,7 +77,7 @@ var _ = ginkgo.Describe("[XSVM]", ginkgo.Label("xsvm"), func() {
 		sourceValidators := getNodesForIDs(network.Nodes, sourceSubnet.ValidatorIDs)
 		require.NotEmpty(sourceValidators)
 		sourceAPINode := sourceValidators[0]
-		sourceAPINodeURI := e2e.GetLocalURI(tc, sourceAPINode)
+		sourceAPINodeURI := sourceAPINode.GetAccessibleURI()
 		tc.Log().Info("issuing transactions for source subnet",
 			zap.String("subnetName", subnetAName),
 			zap.Stringer("nodeID", sourceAPINode.NodeID),
@@ -87,7 +87,7 @@ var _ = ginkgo.Describe("[XSVM]", ginkgo.Label("xsvm"), func() {
 		destinationValidators := getNodesForIDs(network.Nodes, destinationSubnet.ValidatorIDs)
 		require.NotEmpty(destinationValidators)
 		destinationAPINode := destinationValidators[0]
-		destinationAPINodeURI := e2e.GetLocalURI(tc, destinationAPINode)
+		destinationAPINodeURI := destinationAPINode.GetAccessibleURI()
 		tc.Log().Info("issuing transactions for destination subnet",
 			zap.String("subnetName", subnetBName),
 			zap.Stringer("nodeID", destinationAPINode.NodeID),
@@ -125,7 +125,7 @@ var _ = ginkgo.Describe("[XSVM]", ginkgo.Label("xsvm"), func() {
 
 		tc.By("checking that the export transaction has been accepted on all nodes")
 		for _, node := range sourceValidators[1:] {
-			uri := e2e.GetLocalURI(tc, node)
+			uri := node.GetAccessibleURI()
 			require.NoError(api.AwaitTxAccepted(
 				tc.DefaultContext(),
 				api.NewClient(uri, sourceChain.ChainID.String()),
@@ -157,7 +157,7 @@ var _ = ginkgo.Describe("[XSVM]", ginkgo.Label("xsvm"), func() {
 		tc.By(fmt.Sprintf("importing to blockchain %s on subnet %s", destinationChain.ChainID, destinationSubnet.SubnetID))
 		sourceURIs := make([]string, len(sourceValidators))
 		for i, node := range sourceValidators {
-			sourceURIs[i] = e2e.GetLocalURI(tc, node)
+			sourceURIs[i] = node.GetAccessibleURI()
 		}
 		importTxStatus, err := importtx.Import(
 			tc.DefaultContext(),

tests/e2e/x/transfer/virtuous.go

@@ -48,11 +48,12 @@ var _ = e2e.DescribeXChainSerial("[Virtuous Transfer Tx AVAX]", func() {
 	ginkgo.It("can issue a virtuous transfer tx for AVAX asset",
 		func() {
 			var (
-				env       = e2e.GetEnv(tc)
-				localURIs = env.GetNodeURIs()
-				rpcEps    = make([]string, len(localURIs))
+				env     = e2e.GetEnv(tc)
+				network = env.GetNetwork()
+				uris    = network.GetNodeURIs()
+				rpcEps  = make([]string, len(uris))
 			)
-			for i, nodeURI := range localURIs {
+			for i, nodeURI := range uris {
 				rpcEps[i] = nodeURI.URI
 			}
 
@@ -304,6 +305,6 @@ var _ = e2e.DescribeXChainSerial("[Virtuous Transfer Tx AVAX]", func() {
 				runFunc(i)
 			}
 
-			_ = e2e.CheckBootstrapIsPossible(tc, env.GetNetwork())
+			_ = e2e.CheckBootstrapIsPossible(tc, network)
 		})
 })

tests/fixture/e2e/env.go

@@ -222,40 +222,15 @@ func NewTestEnvironment(tc tests.TestContext, flagVars *FlagVars, desiredNetwork
 		testContext:                 tc,
 	}
 
-	if network.DefaultRuntimeConfig.Process != nil {
-		// Display node IDs and URIs for process-based networks since the nodes are guaranteed to be network accessible
-		uris := env.GetNodeURIs()
-		require.NotEmpty(uris, "network contains no nodes")
-		tc.Log().Info("network nodes are available",
-			zap.Any("uris", uris),
-		)
-	} else {
-		// Only display node IDs for kube-based networks since the nodes may not be network accessible and
-		// port-forwarded URIs are ephemeral
-		nodeIDs := network.GetAvailableNodeIDs()
-		require.NotEmpty(nodeIDs, "network contains no nodes")
-		tc.Log().Info("network nodes are available. Not showing node URIs since kube nodes may be running remotely.",
-			zap.Strings("nodeIDs", nodeIDs),
-		)
-	}
+	uris := network.GetNodeURIs()
+	require.NotEmpty(uris, "network contains no nodes")
+	tc.Log().Info("network nodes are available",
+		zap.Any("uris", uris),
+	)
 
 	return env
 }
 
-// Retrieve URIs for validator nodes of the shared network. The URIs
-// are only guaranteed to be accessible until the environment test
-// context is torn down (usually the duration of execution of a single
-// test).
-func (te *TestEnvironment) GetNodeURIs() []tmpnet.NodeURI {
-	var (
-		tc      = te.testContext
-		network = te.GetNetwork()
-	)
-	uris, err := network.GetNodeURIs(tc.DefaultContext(), tc.DeferCleanup)
-	require.NoError(tc, err)
-	return uris
-}
-
 // Retrieve a random URI to naively attempt to spread API load across nodes.
 func (te *TestEnvironment) GetRandomNodeURI() tmpnet.NodeURI {
 	var (
@@ -279,15 +254,10 @@ func (te *TestEnvironment) GetRandomNodeURI() tmpnet.NodeURI {
 
 	require.NotEmpty(tc, availableNodes, "no available nodes to target")
 
-	// Use a local URI for the node to ensure compatibility with kube
 	randomNode := availableNodes[r.Intn(len(availableNodes))]
-	uri, cancel, err := randomNode.GetLocalURI(tc.DefaultContext())
-	require.NoError(tc, err)
-	tc.DeferCleanup(cancel)
-
 	nodeURI := tmpnet.NodeURI{
 		NodeID: randomNode.NodeID,
-		URI:    uri,
+		URI:    randomNode.GetAccessibleURI(),
 	}
 	tc.Log().Info("targeting random node",
 		zap.Stringer("nodeID", nodeURI.NodeID),