Skip to content

Commit 3b5ba03

Browse files
maru-avamaru-ava
committed
fixup: Source ingress configuration from the cluster
1 parent 81e8f3e commit 3b5ba03

File tree

5 files changed

+162
-78
lines changed

5 files changed

+162
-78
lines changed

‎tests/fixture/tmpnet/flags/kube_runtime.go

Lines changed: 3 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ import (
77
"errors"
88
"flag"
99
"fmt"
10-
"strings"
1110

1211
"github.com/spf13/pflag"
1312

@@ -21,10 +20,9 @@ const (
2120
)
2221

2322
var (
24-
errKubeNamespaceRequired = errors.New("--kube-namespace is required")
25-
errKubeImageRequired = errors.New("--kube-image is required")
26-
errKubeMinVolumeSizeRequired = fmt.Errorf("--kube-volume-size must be >= %d", tmpnet.MinimumVolumeSizeGB)
27-
errKubeBaseAccessibleURIRequired = errors.New("--kube-base-accessible-uri is required when running outside of cluster")
23+
errKubeNamespaceRequired = errors.New("--kube-namespace is required")
24+
errKubeImageRequired = errors.New("--kube-image is required")
25+
errKubeMinVolumeSizeRequired = fmt.Errorf("--kube-volume-size must be >= %d", tmpnet.MinimumVolumeSizeGB)
2826
)
2927

3028
type kubeRuntimeVars struct {
@@ -34,7 +32,6 @@ type kubeRuntimeVars struct {
3432
useExclusiveScheduling bool
3533
schedulingLabelKey string
3634
schedulingLabelValue string
37-
baseAccessibleURI string
3835
config *KubeconfigVars
3936
}
4037

@@ -88,12 +85,6 @@ func (v *kubeRuntimeVars) register(stringVar varFunc[string], uintVar varFunc[ui
8885
"",
8986
kubeDocPrefix+"The label value to use for exclusive scheduling for node selection and toleration",
9087
)
91-
stringVar(
92-
&v.baseAccessibleURI,
93-
"kube-base-accessible-uri",
94-
"",
95-
kubeDocPrefix+"The base URI for constructing node URIs when running outside of the cluster hosting nodes",
96-
)
9788
}
9889

9990
func (v *kubeRuntimeVars) getKubeRuntimeConfig() (*tmpnet.KubeRuntimeConfig, error) {
@@ -106,17 +97,6 @@ func (v *kubeRuntimeVars) getKubeRuntimeConfig() (*tmpnet.KubeRuntimeConfig, err
10697
if v.volumeSizeGB < tmpnet.MinimumVolumeSizeGB {
10798
return nil, errKubeMinVolumeSizeRequired
10899
}
109-
baseAccessibleURI := v.baseAccessibleURI
110-
if strings.HasPrefix(v.config.Context, "kind-kind") && len(baseAccessibleURI) == 0 {
111-
// Use the base uri expected for the kind cluster deployed by tmpnet. Not supplying this as a default
112-
// ensures that an explicit value is required for non-kind clusters.
113-
//
114-
// TODO(marun) Log why this value is being used. This will require passing a log through the call chain.
115-
baseAccessibleURI = "http://localhost:30791"
116-
}
117-
if !tmpnet.IsRunningInCluster() && len(baseAccessibleURI) == 0 {
118-
return nil, errKubeBaseAccessibleURIRequired
119-
}
120100
return &tmpnet.KubeRuntimeConfig{
121101
ConfigPath: v.config.Path,
122102
ConfigContext: v.config.Context,
@@ -126,7 +106,5 @@ func (v *kubeRuntimeVars) getKubeRuntimeConfig() (*tmpnet.KubeRuntimeConfig, err
126106
UseExclusiveScheduling: v.useExclusiveScheduling,
127107
SchedulingLabelKey: v.schedulingLabelKey,
128108
SchedulingLabelValue: v.schedulingLabelValue,
129-
// Strip trailing slashes to simplify path composition
130-
BaseAccessibleURI: strings.TrimRight(baseAccessibleURI, "/"),
131109
}, nil
132110
}

‎tests/fixture/tmpnet/kube_runtime.go

Lines changed: 106 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -54,10 +54,14 @@ const (
5454
antiAffinityLabelValue = "exclusive"
5555

5656
// Name of config map containing tmpnet defaults
57-
kubeRuntimeConfigMapName = "tmpnet"
57+
defaultsConfigMapName = "tmpnet-defaults"
58+
ingressHostKey = "ingressHost"
5859
)
5960

60-
var errMissingSchedulingLabels = errors.New("--kube-scheduling-label-key and --kube-scheduling-label-value are required when exclusive scheduling is enabled")
61+
var (
62+
errMissingSchedulingLabels = errors.New("--kube-scheduling-label-key and --kube-scheduling-label-value are required when exclusive scheduling is enabled")
63+
errMissingIngressHost = errors.New("IngressHost is a required value. Ensure the " + defaultsConfigMapName + " ConfigMap contains an entry for " + ingressHostKey)
64+
)
6165

6266
type KubeRuntimeConfig struct {
6367
// Path to the kubeconfig file identifying the target cluster
@@ -78,14 +82,18 @@ type KubeRuntimeConfig struct {
7882
SchedulingLabelKey string `json:"schedulingLabelKey,omitempty"`
7983
// Label value to use for exclusive scheduling for node selection and toleration
8084
SchedulingLabelValue string `json:"schedulingLabelValue,omitempty"`
81-
// Base URI for constructing node URIs when running outside of the cluster hosting nodes (e.g., "http://localhost:30791")
82-
BaseAccessibleURI string `json:"baseAccessibleURI,omitempty"`
85+
// Host for ingress rules (e.g., "localhost:30791" for kind, "tmpnet.example.com" for EKS)
86+
IngressHost string `json:"ingressHost,omitempty"`
87+
// TLS secret name for ingress (empty for HTTP, populated for HTTPS)
88+
IngressSecret string `json:"ingressSecret,omitempty"`
8389
}
8490

8591
// ensureDefaults sets cluster-specific defaults for fields not already set by flags.
8692
func (c *KubeRuntimeConfig) ensureDefaults(ctx context.Context, log logging.Logger) error {
93+
// Only read defaults if necessary
8794
requireSchedulingDefaults := c.UseExclusiveScheduling && (len(c.SchedulingLabelKey) == 0 || len(c.SchedulingLabelValue) == 0)
88-
if !requireSchedulingDefaults {
95+
requireIngressDefaults := !IsRunningInCluster() && len(c.IngressHost) == 0
96+
if !requireSchedulingDefaults && !requireIngressDefaults {
8997
return nil
9098
}
9199

@@ -96,34 +104,55 @@ func (c *KubeRuntimeConfig) ensureDefaults(ctx context.Context, log logging.Logg
96104

97105
log.Info("attempting to retrieve configmap containing tmpnet defaults",
98106
zap.String("namespace", c.Namespace),
99-
zap.String("configMap", kubeRuntimeConfigMapName),
107+
zap.String("configMap", defaultsConfigMapName),
100108
)
101109

102-
configMap, err := clientset.CoreV1().ConfigMaps(c.Namespace).Get(ctx, kubeRuntimeConfigMapName, metav1.GetOptions{})
110+
configMap, err := clientset.CoreV1().ConfigMaps(c.Namespace).Get(ctx, defaultsConfigMapName, metav1.GetOptions{})
103111
if err != nil {
104112
return fmt.Errorf("failed to get ConfigMap: %w", err)
105113
}
106114

107-
var (
108-
schedulingLabelKey = configMap.Data["schedulingLabelKey"]
109-
schedulingLabelValue = configMap.Data["schedulingLabelValue"]
110-
)
111-
if len(c.SchedulingLabelKey) == 0 && len(schedulingLabelKey) > 0 {
112-
log.Info("setting default value for SchedulingLabelKey",
113-
zap.String("schedulingLabelKey", schedulingLabelKey),
115+
if requireSchedulingDefaults {
116+
var (
117+
schedulingLabelKey = configMap.Data["schedulingLabelKey"]
118+
schedulingLabelValue = configMap.Data["schedulingLabelValue"]
114119
)
115-
c.SchedulingLabelKey = schedulingLabelKey
120+
if len(c.SchedulingLabelKey) == 0 && len(schedulingLabelKey) > 0 {
121+
log.Info("setting default value for SchedulingLabelKey",
122+
zap.String("schedulingLabelKey", schedulingLabelKey),
123+
)
124+
c.SchedulingLabelKey = schedulingLabelKey
125+
}
126+
if len(c.SchedulingLabelValue) == 0 && len(schedulingLabelValue) > 0 {
127+
log.Info("setting default value for SchedulingLabelValue",
128+
zap.String("schedulingLabelValue", schedulingLabelValue),
129+
)
130+
c.SchedulingLabelValue = schedulingLabelValue
131+
}
132+
if len(c.SchedulingLabelKey) == 0 || len(c.SchedulingLabelValue) == 0 {
133+
return errMissingSchedulingLabels
134+
}
116135
}
117-
if len(c.SchedulingLabelValue) == 0 && len(schedulingLabelValue) > 0 {
118-
log.Info("setting default value for SchedulingLabelValue",
119-
zap.String("schedulingLabelValue", schedulingLabelValue),
136+
if requireIngressDefaults {
137+
var (
138+
ingressHost = configMap.Data[ingressHostKey]
139+
ingressSecret = configMap.Data["ingressSecret"]
120140
)
121-
c.SchedulingLabelValue = schedulingLabelValue
122-
}
123-
124-
// Validate that the scheduling labels are now set
125-
if len(c.SchedulingLabelKey) == 0 || len(c.SchedulingLabelValue) == 0 {
126-
return errMissingSchedulingLabels
141+
if len(c.IngressHost) == 0 && len(ingressHost) > 0 {
142+
log.Info("setting default value for IngressHost",
143+
zap.String("ingressHost", ingressHost),
144+
)
145+
c.IngressHost = ingressHost
146+
}
147+
if len(c.IngressSecret) == 0 && len(ingressSecret) > 0 {
148+
log.Info("setting default value for IngressSecret",
149+
zap.String("ingressSecret", ingressSecret),
150+
)
151+
c.IngressSecret = ingressSecret
152+
}
153+
if len(c.IngressHost) == 0 {
154+
return errMissingIngressHost
155+
}
127156
}
128157

129158
return nil
@@ -150,8 +179,8 @@ func (p *KubeRuntime) readState(ctx context.Context) error {
150179
)
151180

152181
// Validate that it will be possible to construct accessible URIs when running external to the kube cluster
153-
if !IsRunningInCluster() && len(runtimeConfig.BaseAccessibleURI) == 0 {
154-
return errors.New("BaseAccessibleURI must be set when running outside of the kubernetes cluster")
182+
if !IsRunningInCluster() && len(runtimeConfig.IngressHost) == 0 {
183+
return errors.New("IngressHost must be set when running outside of the kubernetes cluster")
155184
}
156185

157186
clientset, err := p.getClientset()
@@ -207,11 +236,18 @@ func (p *KubeRuntime) GetAccessibleURI() string {
207236
return p.node.URI
208237
}
209238

210-
baseURI := p.runtimeConfig().BaseAccessibleURI
211-
nodeID := p.node.NodeID.String()
212-
networkUUID := p.node.network.UUID
239+
var (
240+
protocol = "http"
241+
nodeID = p.node.NodeID.String()
242+
networkUUID = p.node.network.UUID
243+
runtimeConfig = p.runtimeConfig()
244+
)
245+
// Assume tls is configured for an ingress secret
246+
if len(runtimeConfig.IngressSecret) > 0 {
247+
protocol = "https"
248+
}
213249

214-
return fmt.Sprintf("%s/networks/%s/%s", baseURI, networkUUID, nodeID)
250+
return fmt.Sprintf("%s://%s/networks/%s/%s", protocol, runtimeConfig.IngressHost, networkUUID, nodeID)
215251
}
216252

217253
// GetAccessibleStakingAddress retrieves a StakingAddress for the node intended to be
@@ -993,6 +1029,35 @@ func (p *KubeRuntime) createNodeIngress(ctx context.Context, serviceName string)
9931029
pathType = networkingv1.PathTypeImplementationSpecific
9941030
)
9951031

1032+
// Build the ingress rules
1033+
ingressRules := []networkingv1.IngressRule{
1034+
{
1035+
IngressRuleValue: networkingv1.IngressRuleValue{
1036+
HTTP: &networkingv1.HTTPIngressRuleValue{
1037+
Paths: []networkingv1.HTTPIngressPath{
1038+
{
1039+
Path: pathPattern,
1040+
PathType: &pathType,
1041+
Backend: networkingv1.IngressBackend{
1042+
Service: &networkingv1.IngressServiceBackend{
1043+
Name: serviceName,
1044+
Port: networkingv1.ServiceBackendPort{
1045+
Number: config.DefaultHTTPPort,
1046+
},
1047+
},
1048+
},
1049+
},
1050+
},
1051+
},
1052+
},
1053+
},
1054+
}
1055+
1056+
// Add host if not localhost
1057+
if !strings.HasPrefix(runtimeConfig.IngressHost, "localhost") {
1058+
ingressRules[0].Host = runtimeConfig.IngressHost
1059+
}
1060+
9961061
ingress := &networkingv1.Ingress{
9971062
ObjectMeta: metav1.ObjectMeta{
9981063
Name: serviceName,
@@ -1012,31 +1077,20 @@ func (p *KubeRuntime) createNodeIngress(ctx context.Context, serviceName string)
10121077
},
10131078
Spec: networkingv1.IngressSpec{
10141079
IngressClassName: &ingressClassName,
1015-
Rules: []networkingv1.IngressRule{
1016-
{
1017-
IngressRuleValue: networkingv1.IngressRuleValue{
1018-
HTTP: &networkingv1.HTTPIngressRuleValue{
1019-
Paths: []networkingv1.HTTPIngressPath{
1020-
{
1021-
Path: pathPattern,
1022-
PathType: &pathType,
1023-
Backend: networkingv1.IngressBackend{
1024-
Service: &networkingv1.IngressServiceBackend{
1025-
Name: serviceName,
1026-
Port: networkingv1.ServiceBackendPort{
1027-
Number: config.DefaultHTTPPort,
1028-
},
1029-
},
1030-
},
1031-
},
1032-
},
1033-
},
1034-
},
1035-
},
1036-
},
1080+
Rules: ingressRules,
10371081
},
10381082
}
10391083

1084+
// Add TLS configuration if IngressSecret is set
1085+
if len(runtimeConfig.IngressSecret) > 0 && !strings.HasPrefix(runtimeConfig.IngressHost, "localhost") {
1086+
ingress.Spec.TLS = []networkingv1.IngressTLS{
1087+
{
1088+
Hosts: []string{runtimeConfig.IngressHost},
1089+
SecretName: runtimeConfig.IngressSecret,
1090+
},
1091+
}
1092+
}
1093+
10401094
_, err = clientset.NetworkingV1().Ingresses(namespace).Create(ctx, ingress, metav1.CreateOptions{})
10411095
if err != nil {
10421096
return fmt.Errorf("failed to create Ingress: %w", err)

‎tests/fixture/tmpnet/start_kind_cluster.go

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,10 @@ func StartKindCluster(
112112
return fmt.Errorf("failed to deploy ingress controller: %w", err)
113113
}
114114

115+
if err := createDefaultsConfigMap(ctx, log, configPath, configContext, DefaultTmpnetNamespace); err != nil {
116+
return fmt.Errorf("failed to create defaults ConfigMap: %w", err)
117+
}
118+
115119
return nil
116120
}
117121

@@ -410,3 +414,48 @@ func runHelmCommand(ctx context.Context, args ...string) error {
410414
cmd.Stderr = os.Stderr
411415
return cmd.Run()
412416
}
417+
418+
// createDefaultsConfigMap creates a ConfigMap containing defaults for the tmpnet namespace.
419+
func createDefaultsConfigMap(ctx context.Context, log logging.Logger, configPath string, configContext string, namespace string) error {
420+
clientset, err := GetClientset(log, configPath, configContext)
421+
if err != nil {
422+
return fmt.Errorf("failed to get clientset: %w", err)
423+
}
424+
425+
configMapName := defaultsConfigMapName
426+
427+
// Check if configmap already exists
428+
_, err = clientset.CoreV1().ConfigMaps(namespace).Get(ctx, configMapName, metav1.GetOptions{})
429+
if err == nil {
430+
log.Info("defaults ConfigMap already exists",
431+
zap.String("namespace", namespace),
432+
zap.String("configMap", configMapName),
433+
)
434+
return nil
435+
}
436+
if !apierrors.IsNotFound(err) {
437+
return fmt.Errorf("failed to check for configmap %s/%s: %w", namespace, configMapName, err)
438+
}
439+
440+
log.Info("creating defaults ConfigMap",
441+
zap.String("namespace", namespace),
442+
zap.String("configMap", configMapName),
443+
)
444+
445+
configMap := &corev1.ConfigMap{
446+
ObjectMeta: metav1.ObjectMeta{
447+
Name: configMapName,
448+
Namespace: namespace,
449+
},
450+
Data: map[string]string{
451+
ingressHostKey: "localhost:30791",
452+
},
453+
}
454+
455+
_, err = clientset.CoreV1().ConfigMaps(namespace).Create(ctx, configMap, metav1.CreateOptions{})
456+
if err != nil {
457+
return fmt.Errorf("failed to create configmap %s/%s: %w", namespace, configMapName, err)
458+
}
459+
460+
return nil
461+
}

‎tests/fixture/tmpnet/yaml/tmpnet-rbac.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@ rules:
2828
- apiGroups: ["networking.k8s.io"]
2929
resources: ["ingresses"]
3030
verbs: ["get", "create"]
31+
- apiGroups: [""]
32+
resources: ["configmaps"]
33+
verbs: ["get"]
3134
- apiGroups: [""]
3235
resources: ["endpoints"]
3336
verbs: ["get"]

‎tests/load2/main/main.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ func main() {
6767
e2e.NewTestEnvironment(tc, flagVars, network)
6868

6969
ctx := tests.DefaultNotifyContext(0, tc.DeferCleanup)
70-
wsURIs, err := tmpnet.GetNodeWebsocketURIs(ctx, network.Nodes, blockchainID, tc.DeferCleanup)
70+
wsURIs, err := tmpnet.GetNodeWebsocketURIs(network.Nodes, blockchainID)
7171
require.NoError(err)
7272

7373
registry := prometheus.NewRegistry()

0 commit comments

Comments
 (0)