Merge pull request #933 from wongma7/eksctldeleter

Grant EKSCTL_ADMIN_ROLE admin access to eksctl clusters

Author: k8s-ci-robot

Makefile

@@ -113,6 +113,7 @@ test-e2e-external-eks:
 	CLUSTER_TYPE=eksctl \
 	K8S_VERSION="1.20" \
 	HELM_VALUES_FILE="./hack/values_eksctl.yaml" \
+	EKSCTL_ADMIN_ROLE="Infra-prod-KopsDeleteAllLambdaServiceRoleF1578477-1ELDFIB4KCMXV" \
 	AWS_REGION=us-west-2 \
 	AWS_AVAILABILITY_ZONES=us-west-2a,us-west-2b \
 	TEST_PATH=./tests/e2e-kubernetes/... \

hack/e2e/ecr.sh

@@ -11,7 +11,7 @@ function ecr_build_and_push() {
   IMAGE_NAME=${3}
   IMAGE_TAG=${4}
   set +e
-  if docker images | grep "${IMAGE_NAME}" | grep "${IMAGE_TAG}"; then
+  if docker images --format "{{.Repository}}:{{.Tag}}" | grep "${IMAGE_NAME}:${IMAGE_TAG}"; then
     set -e
     loudecho "Assuming ${IMAGE_NAME}:${IMAGE_TAG} has been built and pushed"
   else

hack/e2e/eksctl.sh

@@ -21,6 +21,7 @@ function eksctl_create_cluster() {
   CLUSTER_FILE=${7}
   KUBECONFIG=${8}
   EKSCTL_PATCH_FILE=${9}
+  EKSCTL_ADMIN_ROLE=${10}
 
   generate_ssh_key "${SSH_KEY_PATH}"
 
@@ -55,6 +56,14 @@ function eksctl_create_cluster() {
 
   loudecho "Getting cluster ${CLUSTER_NAME}"
   ${BIN} get cluster "${CLUSTER_NAME}"
+
+  if [ -n "$EKSCTL_ADMIN_ROLE" ]; then
+    AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
+    ADMIN_ARN="arn:aws:iam::${AWS_ACCOUNT_ID}:role/${EKSCTL_ADMIN_ROLE}"
+    loudecho "Granting ${ADMIN_ARN} admin access to the cluster"
+    ${BIN} create iamidentitymapping --cluster "${CLUSTER_NAME}" --arn "${ADMIN_ARN}" --group system:masters --username admin
+  fi
+
   return $?
 }
 

hack/e2e/kops.sh

@@ -22,12 +22,14 @@ function kops_create_cluster() {
   CLUSTER_NAME=${2}
   BIN=${3}
   ZONES=${4}
-  INSTANCE_TYPE=${5}
-  K8S_VERSION=${6}
-  CLUSTER_FILE=${7}
-  KUBECONFIG=${8}
-  KOPS_PATCH_FILE=${9}
-  KOPS_STATE_FILE=${10}
+  NODE_COUNT=${5}
+  INSTANCE_TYPE=${6}
+  K8S_VERSION=${7}
+  CLUSTER_FILE=${8}
+  KUBECONFIG=${9}
+  KOPS_PATCH_FILE=${10}
+  KOPS_PATCH_NODE_FILE=${11}
+  KOPS_STATE_FILE=${12}
 
   generate_ssh_key "${SSH_KEY_PATH}"
 
@@ -39,15 +41,18 @@ function kops_create_cluster() {
     ${BIN} create cluster --state "${KOPS_STATE_FILE}" \
       --ssh-public-key="${SSH_KEY_PATH}".pub \
       --zones "${ZONES}" \
-      --node-count=3 \
+      --node-count="${NODE_COUNT}" \
       --node-size="${INSTANCE_TYPE}" \
       --kubernetes-version="${K8S_VERSION}" \
       --dry-run \
-      -o json \
+      -o yaml \
       "${CLUSTER_NAME}" > "${CLUSTER_FILE}"
 
     if test -f "$KOPS_PATCH_FILE"; then
-      kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_FILE"
+      kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_FILE" "Cluster" ""
+    fi
+    if test -f "$KOPS_PATCH_NODE_FILE"; then
+      kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_NODE_FILE" "InstanceGroup" "Node"
     fi
 
     loudecho "Creating cluster $CLUSTER_NAME with $CLUSTER_FILE"
@@ -88,36 +93,63 @@ function kops_delete_cluster() {
   ${BIN} delete cluster --name "${CLUSTER_NAME}" --state "${KOPS_STATE_FILE}" --yes
 }
 
-# TODO switch this to python or work exclusively with yaml, all this
-# hacking with jq stinks!
+# TODO switch this to python, work exclusively with yaml, use kops toolbox
+# template/kops set?, all this hacking with jq stinks!
 function kops_patch_cluster_file() {
-  CLUSTER_FILE=${1}    # input must be json
+  CLUSTER_FILE=${1}    # input must be yaml
   KOPS_PATCH_FILE=${2} # input must be yaml
+  KIND=${3}            # must be either Cluster or InstanceGroup
+  ROLE=${4}            # must be either Master or Node
 
   loudecho "Patching cluster $CLUSTER_NAME with $KOPS_PATCH_FILE"
 
-  # Temporary intermediate files for patching
+  # Temporary intermediate files for patching, don't mutate CLUSTER_FILE until
+  # the end
+  CLUSTER_FILE_JSON=$CLUSTER_FILE.json
   CLUSTER_FILE_0=$CLUSTER_FILE.0
   CLUSTER_FILE_1=$CLUSTER_FILE.1
 
-  # Output is an array of Cluster and InstanceGroups
-  jq '.[] | select(.kind=="Cluster")' "$CLUSTER_FILE" > "$CLUSTER_FILE_0"
+  # HACK convert the multiple yaml documents to an array of json objects
+  yaml_to_json "$CLUSTER_FILE" "$CLUSTER_FILE_JSON"
+
+  # Find the json objects to patch
+  FILTER=".[] | select(.kind==\"$KIND\")"
+  if [ -n "$ROLE" ]; then
+    FILTER="$FILTER | select(.spec.role==\"$ROLE\")"
+  fi
+  jq "$FILTER" "$CLUSTER_FILE_JSON" > "$CLUSTER_FILE_0"
 
-  # Patch only the Cluster
+  # Patch only the json objects
   kubectl patch -f "$CLUSTER_FILE_0" --local --type merge --patch "$(cat "$KOPS_PATCH_FILE")" -o json > "$CLUSTER_FILE_1"
   mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0"
 
-  # Write the patched Cluster back to the array
-  jq '(.[] | select(.kind=="Cluster")) = $cluster[0]' "$CLUSTER_FILE" --slurpfile cluster "$CLUSTER_FILE_0" > "$CLUSTER_FILE_1"
+  # Delete the original json objects, add the patched
+  # TODO Cluster must always be first?
+  jq "del($FILTER)" "$CLUSTER_FILE_JSON" | jq ". + \$patched | sort" --slurpfile patched "$CLUSTER_FILE_0" > "$CLUSTER_FILE_1"
   mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0"
 
-  # HACK convert the json array to multiple yaml documents
-  for ((i = 0; i < $(jq length "$CLUSTER_FILE_0"); i++)); do
-    echo "---" >> "$CLUSTER_FILE_1"
-    jq ".[$i]" "$CLUSTER_FILE_0" | kubectl patch -f - --local -p "{}" --type merge -o yaml >> "$CLUSTER_FILE_1"
-  done
+  # HACK convert the array of json objects to multiple yaml documents
+  json_to_yaml "$CLUSTER_FILE_0" "$CLUSTER_FILE_1"
   mv "$CLUSTER_FILE_1" "$CLUSTER_FILE_0"
 
-  # Done patching, overwrite original CLUSTER_FILE
+  # Done patching, overwrite original yaml CLUSTER_FILE
   mv "$CLUSTER_FILE_0" "$CLUSTER_FILE" # output is yaml
+
+  # Clean up
+  rm "$CLUSTER_FILE_JSON"
+}
+
+function yaml_to_json() {
+  IN=${1}
+  OUT=${2}
+  kubectl patch -f "$IN" --local -p "{}" --type merge -o json | jq '.' -s > "$OUT"
+}
+
+function json_to_yaml() {
+  IN=${1}
+  OUT=${2}
+  for ((i = 0; i < $(jq length "$IN"); i++)); do
+    echo "---" >> "$OUT"
+    jq ".[$i]" "$IN" | kubectl patch -f - --local -p "{}" --type merge -o yaml >> "$OUT"
+  done
 }

hack/e2e/run.sh

@@ -35,12 +35,13 @@ CLUSTER_TYPE=${CLUSTER_TYPE:-kops}
 TEST_DIR=${BASE_DIR}/csi-test-artifacts
 BIN_DIR=${TEST_DIR}/bin
 SSH_KEY_PATH=${TEST_DIR}/id_rsa
-CLUSTER_FILE=${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.json
+CLUSTER_FILE=${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.yaml
 KUBECONFIG=${KUBECONFIG:-"${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.kubeconfig"}
 
 REGION=${AWS_REGION:-us-west-2}
 ZONES=${AWS_AVAILABILITY_ZONES:-us-west-2a,us-west-2b,us-west-2c}
 FIRST_ZONE=$(echo "${ZONES}" | cut -d, -f1)
+NODE_COUNT=${NODE_COUNT:-3}
 INSTANCE_TYPE=${INSTANCE_TYPE:-c4.large}
 
 AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
@@ -54,8 +55,10 @@ K8S_VERSION=${K8S_VERSION:-1.20.6}
 KOPS_VERSION=${KOPS_VERSION:-1.20.0}
 KOPS_STATE_FILE=${KOPS_STATE_FILE:-s3://k8s-kops-csi-e2e}
 KOPS_PATCH_FILE=${KOPS_PATCH_FILE:-./hack/kops-patch.yaml}
+KOPS_PATCH_NODE_FILE=${KOPS_PATCH_NODE_FILE:-./hack/kops-patch-node.yaml}
 
 EKSCTL_PATCH_FILE=${EKSCTL_PATCH_FILE:-./hack/eksctl-patch.yaml}
+EKSCTL_ADMIN_ROLE=${EKSCTL_ADMIN_ROLE:-}
 
 HELM_VALUES_FILE=${HELM_VALUES_FILE:-./hack/values.yaml}
 
@@ -111,11 +114,13 @@ if [[ "${CLUSTER_TYPE}" == "kops" ]]; then
     "$CLUSTER_NAME" \
     "$KOPS_BIN" \
     "$ZONES" \
+    "$NODE_COUNT" \
     "$INSTANCE_TYPE" \
     "$K8S_VERSION" \
     "$CLUSTER_FILE" \
     "$KUBECONFIG" \
     "$KOPS_PATCH_FILE" \
+    "$KOPS_PATCH_NODE_FILE" \
     "$KOPS_STATE_FILE"
   if [[ $? -ne 0 ]]; then
     exit 1
@@ -130,7 +135,8 @@ elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then
     "$K8S_VERSION" \
     "$CLUSTER_FILE" \
     "$KUBECONFIG" \
-    "$EKSCTL_PATCH_FILE"
+    "$EKSCTL_PATCH_FILE" \
+    "$EKSCTL_ADMIN_ROLE"
   if [[ $? -ne 0 ]]; then
     exit 1
   fi