Skip to content

Commit 39c2659

Browse files
metal3-io-botmetal3-io-bot
authored
Merge pull request #2109 from Nordix/peppi-lotta/rewrite-fetch-manifest
🌱 Rewrite fetch_manifest.sh into golang
2 parents 4b67ab8 + 1661213 commit 39c2659

File tree

3 files changed

+178
-55
lines changed

3 files changed

+178
-55
lines changed

‎test/e2e/logcollector.go‎

Lines changed: 157 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,12 @@ import (
1313

1414
. "github.com/onsi/ginkgo/v2"
1515
. "github.com/onsi/gomega"
16+
"gopkg.in/yaml.v3"
1617
corev1 "k8s.io/api/core/v1"
18+
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
1719
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
20+
"k8s.io/apimachinery/pkg/runtime/schema"
21+
"k8s.io/client-go/dynamic"
1822
"k8s.io/client-go/kubernetes"
1923
"k8s.io/kubectl/pkg/describe"
2024
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
@@ -99,19 +103,98 @@ func (Metal3LogCollector) CollectMachinePoolLog(_ context.Context, _ client.Clie
99103
return fmt.Errorf("CollectMachinePoolLog not implemented")
100104
}
101105

102-
func FetchManifests(_ framework.ClusterProxy) error {
103-
return fmt.Errorf("FetchManifests not implemented")
106+
// FetchManifests fetches relevant Metal3, CAPI, and Kubernetes core resources
107+
// and dumps them to a file.
108+
func FetchManifests(clusterProxy framework.ClusterProxy, outputPath string) error {
109+
outputPath = filepath.Join(outputPath, clusterProxy.GetName())
110+
ctx := context.Background()
111+
restConfig := clusterProxy.GetRESTConfig()
112+
dynamicClient, err := dynamic.NewForConfig(restConfig)
113+
if err != nil {
114+
return fmt.Errorf("could not create dynamic client: %v", err)
115+
}
116+
117+
k8sCoreManifests := []string{
118+
// This list must contain the plural form of the Kubernetes core
119+
// resources
120+
"deployments",
121+
"replicasets",
122+
}
123+
124+
// Set group and version to get Kubernetes core resources and dump them
125+
for _, manifest := range k8sCoreManifests {
126+
gvr := schema.GroupVersionResource{
127+
Group: "apps",
128+
Version: "v1",
129+
Resource: manifest,
130+
}
131+
132+
if err := DumpGVR(ctx, dynamicClient, gvr, outputPath); err != nil {
133+
return err
134+
}
135+
}
136+
137+
manifests := []string{
138+
// This list contains all resources of interest that are NOT Kubernetes
139+
// core resources.
140+
"bmh",
141+
"hardwaredata",
142+
"cluster",
143+
"machine",
144+
"machinedeployment",
145+
"machinehealthchecks",
146+
"machinesets",
147+
"machinepools",
148+
"m3cluster",
149+
"m3machine",
150+
"metal3machinetemplate",
151+
"kubeadmconfig",
152+
"kubeadmconfigtemplates",
153+
"kubeadmcontrolplane",
154+
"ippool",
155+
"ipclaim",
156+
"ipaddress",
157+
"m3data",
158+
"m3dataclaim",
159+
"m3datatemplate",
160+
}
161+
client := clusterProxy.GetClient()
162+
163+
// Get all CustomResourceDefinitions (CRDs)
164+
crds := apiextensionsv1.CustomResourceDefinitionList{}
165+
if err := client.List(ctx, &crds); err != nil {
166+
return fmt.Errorf("could not list CRDs: %v", err)
167+
}
168+
if len(crds.Items) == 0 {
169+
return nil
170+
}
171+
172+
// Check if the resource is in the manifest list.
173+
// If it is, dump it to a file
174+
for _, crd := range crds.Items {
175+
if crdIsInList(crd, manifests) {
176+
gvr := schema.GroupVersionResource{
177+
Group: crd.Spec.Group,
178+
Version: crd.Status.StoredVersions[0],
179+
Resource: crd.Spec.Names.Plural,
180+
}
181+
182+
if err := DumpGVR(ctx, dynamicClient, gvr, outputPath); err != nil {
183+
return err
184+
}
185+
}
186+
}
187+
fmt.Printf("Successfully collected manifests for cluster %s.", clusterProxy.GetName())
188+
return nil
104189
}
105190

191+
// FetchClusterLogs fetches logs from all pods in the cluster and writes them
192+
// to files.
106193
func FetchClusterLogs(clusterProxy framework.ClusterProxy, outputPath string) error {
107194
ctx := context.Background()
108195
baseDir := filepath.Join(outputPath, clusterProxy.GetName())
109-
err := os.MkdirAll(baseDir, 0775)
110-
if err != nil {
111-
return fmt.Errorf("couldn't create directory: %v", err)
112-
}
113196

114-
// get the clientset
197+
// Get the clientset
115198
clientset := clusterProxy.GetClientSet()
116199

117200
// Print the Pods' information to file
@@ -165,16 +248,6 @@ func FetchClusterLogs(clusterProxy framework.ClusterProxy, outputPath string) er
165248
continue
166249
}
167250
for _, pod := range pods.Items {
168-
machineName := pod.Spec.NodeName
169-
// Create a directory for each pod and the path to it if
170-
// it does not exist
171-
podDir := filepath.Join(baseDir, "machines", machineName, namespace.Name, pod.Name)
172-
err = os.MkdirAll(podDir, 0775)
173-
if err != nil {
174-
fmt.Printf("couldn't write to file: %v", err)
175-
continue
176-
}
177-
178251
// Get detailed information about the Pod
179252
// This does the same thing as:
180253
// kubectl --kubeconfig="${KUBECONFIG_WORKLOAD}" describe pods -n "${NAMESPACE}" "${POD}"
@@ -190,13 +263,9 @@ func FetchClusterLogs(clusterProxy framework.ClusterProxy, outputPath string) er
190263
continue
191264
}
192265

193-
// Print the Pod information to file
194-
file := filepath.Join(podDir, "stdout_describe.log")
195-
err = os.WriteFile(file, []byte(podDescription), 0600)
196-
if err != nil {
197-
fmt.Printf("couldn't write to file: %v", err)
198-
continue
199-
}
266+
machineName := pod.Spec.NodeName
267+
podDir := filepath.Join(baseDir, "machines", machineName, namespace.Name, pod.Name)
268+
writeToFile([]byte(podDescription), "stdout_describe.log", podDir)
200269

201270
// Get containers of the Pod
202271
for _, container := range pod.Spec.Containers {
@@ -215,12 +284,9 @@ func FetchClusterLogs(clusterProxy framework.ClusterProxy, outputPath string) er
215284
return nil
216285
}
217286

287+
// CollectContainerLogs fetches logs from a specific container in a pod and
288+
// writes them to a file.
218289
func CollectContainerLogs(ctx context.Context, namespace string, podName string, containerName string, clientset *kubernetes.Clientset, outputPath string) error {
219-
err := os.MkdirAll(outputPath, 0775)
220-
if err != nil {
221-
return fmt.Errorf("couldn't create directory: %v", err)
222-
}
223-
224290
// Get logs of a container
225291
// Does the same thing as:
226292
// kubectl --kubeconfig="${KUBECONFIG_WORKLOAD}" logs -n "${NAMESPACE}" "${POD}" "${CONTAINER}"
@@ -241,12 +307,70 @@ func CollectContainerLogs(ctx context.Context, namespace string, podName string,
241307
}
242308
podStr := buf.String()
243309

244-
// Print the Pod information to file
245-
file := filepath.Join(outputPath, "stdout.log")
246-
err = os.WriteFile(file, []byte(podStr), 0600)
310+
writeToFile([]byte(podStr), "stdout.log", outputPath)
311+
312+
return nil
313+
}
314+
315+
// writeToFile writes content to a file,
316+
// creating any missing directories in the path.
317+
func writeToFile(content []byte, fileName string, filePath string) {
318+
// Create any missing directories in the path
319+
err := os.MkdirAll(filePath, 0775)
320+
if err != nil {
321+
fmt.Printf("couldn't create directory: %v", err)
322+
}
323+
// Write content to file
324+
file := filepath.Join(filePath, fileName)
325+
err = os.WriteFile(file, content, 0600)
326+
if err != nil {
327+
fmt.Printf("couldn't write to file: %v", err)
328+
}
329+
}
330+
331+
// crdIsInList checks if a CustomResourceDefinition is in the provided list of
332+
// resource names.
333+
func crdIsInList(crd apiextensionsv1.CustomResourceDefinition, list []string) bool {
334+
plural := crd.Spec.Names.Plural
335+
singular := crd.Spec.Names.Singular
336+
shortNames := crd.Spec.Names.ShortNames
337+
338+
for _, name := range list {
339+
if name == plural {
340+
return true
341+
}
342+
if name == singular {
343+
return true
344+
}
345+
for _, shortName := range shortNames {
346+
if name == shortName {
347+
return true
348+
}
349+
}
350+
}
351+
return false
352+
}
353+
354+
// DumpGVR fetches resources of a given GroupVersionResource
355+
// and writes them to files.
356+
func DumpGVR(ctx context.Context, dynamicClient *dynamic.DynamicClient, gvr schema.GroupVersionResource, outputPath string) error {
357+
resources, err := dynamicClient.Resource(gvr).List(ctx, metav1.ListOptions{})
247358
if err != nil {
248-
return fmt.Errorf("couldn't write to file: %v", err)
359+
return fmt.Errorf("could not get resources: %v", err)
360+
}
361+
if len(resources.Items) == 0 {
362+
return nil
249363
}
250364

365+
// Write resource to file
366+
for _, resource := range resources.Items {
367+
filePath := filepath.Join(outputPath, resource.GetNamespace(), resource.GetKind())
368+
fileName := fmt.Sprintf("%s.yaml", resource.GetName())
369+
content, err := yaml.Marshal(resource)
370+
if err != nil {
371+
return fmt.Errorf("could not marshal content: %v", err)
372+
}
373+
writeToFile(content, fileName, filePath)
374+
}
251375
return nil
252376
}

‎test/e2e/pivoting.go‎

Lines changed: 15 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
. "github.com/onsi/gomega"
1717
corev1 "k8s.io/api/core/v1"
1818
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
19+
"k8s.io/apimachinery/pkg/runtime"
1920
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
2021
clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3"
2122
"sigs.k8s.io/cluster-api/cmd/clusterctl/client/config"
@@ -88,13 +89,10 @@ func pivoting(ctx context.Context, inputGetter func() PivotingInput) {
8889
}
8990

9091
By("Fetch manifest for bootstrap cluster before pivot")
91-
kconfigPathBootstrap := input.BootstrapClusterProxy.GetKubeconfigPath()
92-
os.Setenv("KUBECONFIG_BOOTSTRAP", kconfigPathBootstrap)
93-
path := filepath.Join(os.Getenv("CAPM3PATH"), "scripts")
94-
cmd := exec.Command("./fetch_manifests.sh") // #nosec G204:gosec
95-
cmd.Dir = path
96-
_ = cmd.Run()
97-
92+
err = FetchManifests(input.BootstrapClusterProxy, "/tmp/manifests/")
93+
if err != nil {
94+
fmt.Printf("Error fetching manifests for bootstrap cluster before pivot: %v\n", err)
95+
}
9896
By("Fetch target cluster kubeconfig for target cluster log collection")
9997
kconfigPathWorkload := input.TargetCluster.GetKubeconfigPath()
10098
os.Setenv("KUBECONFIG_WORKLOAD", kconfigPathWorkload)
@@ -103,7 +101,7 @@ func pivoting(ctx context.Context, inputGetter func() PivotingInput) {
103101
// target log collection. There is possibility to handle the kubeconfig in better way.
104102
// KubeconfigPathTemp will be used by project-infra target log collection only incase of failed e2e test
105103
kubeconfigPathTemp := "/tmp/kubeconfig-test1.yaml"
106-
cmd = exec.Command("cp", kconfigPathWorkload, kubeconfigPathTemp) // #nosec G204:gosec
104+
cmd := exec.Command("cp", kconfigPathWorkload, kubeconfigPathTemp) // #nosec G204:gosec
107105
stdoutStderr, er := cmd.CombinedOutput()
108106
Logf("%s\n", stdoutStderr)
109107
Expect(er).ToNot(HaveOccurred(), "Cannot fetch target cluster kubeconfig")
@@ -374,10 +372,11 @@ func rePivoting(ctx context.Context, inputGetter func() RePivotingInput) {
374372
}
375373

376374
By("Fetch manifest for workload cluster after pivot")
377-
path := filepath.Join(os.Getenv("CAPM3PATH"), "scripts")
378-
cmd := exec.Command("./fetch_manifests.sh") // #nosec G204:gosec
379-
cmd.Dir = path
380-
_ = cmd.Run()
375+
workloadClusterProxy := framework.NewClusterProxy("workload-cluster-after-pivot", os.Getenv("KUBECONFIG"), runtime.NewScheme())
376+
err = FetchManifests(workloadClusterProxy, "/tmp/manifests/")
377+
if err != nil {
378+
fmt.Printf("Error fetching manifests for workload cluster after pivot: %v\n", err)
379+
}
381380
os.Unsetenv("KUBECONFIG_WORKLOAD")
382381

383382
By("Remove Ironic deployment from target cluster")
@@ -495,10 +494,10 @@ func rePivoting(ctx context.Context, inputGetter func() RePivotingInput) {
495494
})
496495

497496
By("Fetch manifest for bootstrap cluster after re-pivot")
498-
path = filepath.Join(os.Getenv("CAPM3PATH"), "scripts")
499-
cmd = exec.Command("./fetch_manifests.sh") // #nosec G204:gosec
500-
cmd.Dir = path
501-
_ = cmd.Run()
497+
err = FetchManifests(input.BootstrapClusterProxy, "/tmp/manifests/")
498+
if err != nil {
499+
fmt.Printf("Error fetching manifests for bootstrap cluster before pivot: %v\n", err)
500+
}
502501
os.Unsetenv("KUBECONFIG_BOOTSTRAP")
503502

504503
By("RE-PIVOTING TEST PASSED!")

‎test/e2e/upgrade_clusterctl_test.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -208,17 +208,17 @@ func preInitFunc(clusterProxy framework.ClusterProxy, bmoRelease string, ironicR
208208
}
209209

210210
By("Fetch manifest for bootstrap cluster")
211-
path := filepath.Join(os.Getenv("CAPM3PATH"), "scripts")
212-
cmd := exec.Command("./fetch_manifests.sh") // #nosec G204:gosec
213-
cmd.Dir = path
214-
_ = cmd.Run()
211+
err := FetchManifests(clusterProxy, "/tmp/manifests/")
212+
if err != nil {
213+
fmt.Printf("Error fetching manifests for bootstrap cluster: %v\n", err)
214+
}
215215

216216
By("Fetch target cluster kubeconfig for target cluster log collection")
217217
kconfigPathWorkload := clusterProxy.GetKubeconfigPath()
218218
os.Setenv("KUBECONFIG_WORKLOAD", kconfigPathWorkload)
219219
Logf("Save kubeconfig in temp folder for project-infra target log collection")
220220
kubeconfigPathTemp := "/tmp/kubeconfig-test1.yaml"
221-
cmd = exec.Command("cp", kconfigPathWorkload, kubeconfigPathTemp) // #nosec G204:gosec
221+
cmd := exec.Command("cp", kconfigPathWorkload, kubeconfigPathTemp) // #nosec G204:gosec
222222
stdoutStderr, er := cmd.CombinedOutput()
223223
Logf("%s\n", stdoutStderr)
224224
Expect(er).ToNot(HaveOccurred(), "Cannot fetch target cluster kubeconfig")
@@ -248,7 +248,7 @@ func preInitFunc(clusterProxy framework.ClusterProxy, bmoRelease string, ironicR
248248
ironicKustomizePath := fmt.Sprintf("IRONIC_RELEASE_%s", ironicRelease)
249249
initIronicKustomization := e2eConfig.GetVariable(ironicKustomizePath)
250250
By(fmt.Sprintf("Installing Ironic from kustomization %s on the upgrade cluster", initIronicKustomization))
251-
err := BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
251+
err = BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{
252252
Kustomization: initIronicKustomization,
253253
ClusterProxy: clusterProxy,
254254
WaitForDeployment: true,

0 commit comments

Comments
 (0)