From de8584cce9c9d45cdf0458ce67120311103be55c Mon Sep 17 00:00:00 2001
From: Nick M <4718+rkage@users.noreply.github.com>
Date: Sun, 31 Jan 2021 16:25:01 -0500
Subject: [PATCH 1/3] first step of cluster init/join role

Signed-off-by: Nick M <4718+rkage@users.noreply.github.com>
---
 ansible/group_vars/cluster.yml                |  74 ++++++++++++
 ansible/roles/cluster/defaults/main.yml       |  89 +++++++++++++-
 ansible/roles/cluster/tasks/initialize.yml    |   8 +-
 ansible/roles/cluster/tasks/join.yml          |  12 +-
 .../cluster/templates/kubeadm-init.yaml.j2    | 111 ++++++++++++++++++
 .../cluster/templates/kubeadm-init.yml.j2     |  76 ------------
 ...beadm-join.yml.j2 => kubeadm-join.yaml.j2} |  11 +-
 ansible/roles/cluster/vars/main.yml           |   2 +
 8 files changed, 295 insertions(+), 88 deletions(-)
 create mode 100644 ansible/group_vars/cluster.yml
 create mode 100644 ansible/roles/cluster/templates/kubeadm-init.yaml.j2
 delete mode 100644 ansible/roles/cluster/templates/kubeadm-init.yml.j2
 rename ansible/roles/cluster/templates/{kubeadm-join.yml.j2 => kubeadm-join.yaml.j2} (55%)

diff --git a/ansible/group_vars/cluster.yml b/ansible/group_vars/cluster.yml
new file mode 100644
index 00000000..fa290149
--- /dev/null
+++ b/ansible/group_vars/cluster.yml
@@ -0,0 +1,74 @@
+---
+
+# Specify the Kubernetes version, current release is v1.20.0
+cluster_kubernetes_version: v1.20.2
+
+cluster_cgroup_driver: 'systemd'
+
+# Profiling; Enable profiling via web interface host:port/debug/pprof/
+cluster_profiling: false
+
+# The below parameters set the internal network for Kubernetes, this configuration
+# provides a maximum of 64 nodes in a cluster and 254 pods per node.
+
+# Kubernetes internal network for services, *must* be an unused block of space.
+cluster_service_subnet: '10.144.0.0/18'
+
+# Kubernetes internal network. IP's for pods will be drawn from this address
+# space. *must* be unused in your network infrastructure.
+cluster_pod_subnet: '10.144.64.0/18'
+
+# This sets the network node allocation size. Each node in the cluster is
+# assigned a subnet of this size.
+cluster_node_network_prefix: 24
+
+# Cluster Authorization Modes for the Kubernetes Cluster
+# Available Options: 'AlwaysAllow','AlwaysDeny','Node' and 'RBAC'
+# Order is relevant
+cluster_authorization_modes: ['Node', 'RBAC']
+
+# Vars for discoverying kubernetes api endpoints
+cluster_primary_control_node: '{{ groups["masters"]|first }}'
+cluster_apiserver_count: '{{ groups["masters"]|length }}'
+
+
+## Old variables
+####
+# Role - keepalived
+####
+keepalived_vip: 192.168.91.240
+keepalived_interface: "{{ ansible_default_ipv4['interface'] }}"
+
+####
+# Role - cluster
+####
+cluster_name: kubernetes
+cluster_extra_sans:
+  - "{{ keepalived_vip }}"
+cluster_control_plane_endpoint: "{{ keepalived_vip }}:8443"
+cluster_enable_admission_plugins: "NodeRestriction"
+cluster_etcd_heartbeat_interval: 100
+cluster_etcd_election_timeout: 1000
+
+####
+# Role - cni
+####
+cni_plugin: calico
+# cni_bgp_peer_address: 192.168.0.1
+# cni_bgp_peer_asn: 64512
+# cni_cilium_helm_version: 1.9.3
+# cni_cilium_image_version: v1.9.3
+# cni_cilium_hubble_enabled: false
+# cni_cilium_enovy_proxy_image_version: v1.16.2
+# cni_cilium_bpf_hostrouting: "true"
+# cni_cilium_bpf_tproxy: "false"
+# cni_cilium_bpf_masquerade: "true"
+# cni_cilium_endpoint_routes: "false"
+
+####
+# Role - cloudflared
+####
+# Cloudflare options for exposing Kubernetes services via HTTPS/SSH
+cloudflared_enabled: false
+# cloudflared_kube_api_server_dns: ''
+# cloudflared_version: 2020.11.11
diff --git a/ansible/roles/cluster/defaults/main.yml b/ansible/roles/cluster/defaults/main.yml
index 953c98ec..599e1b37 100644
--- a/ansible/roles/cluster/defaults/main.yml
+++ b/ansible/roles/cluster/defaults/main.yml
@@ -1,14 +1,95 @@
 ---
+# API Server Variables
+
+# API Server bind address, by default listens to all requests
+cluster_apiserver_bind_address: '0.0.0.0'
+
+# API Server secure port, by default listens to 6443
+cluster_apiserver_bind_port: 6443
+
+# Enable cluster audit log
+cluster_apiserver_enable_audit: false
+# path to the audit log
+cluster_apiserver_audit_log_path: /var/log/kubernetes/audit.log
+# audit log max age in days
+cluster_apiserver_audit_log_maxage: 30
+# number of audit logs to keep as backup
+cluster_apiserver_audit_log_maxbackup: 1
+# audit log max log size
+cluster_apiserver_audit_log_maxsize: 100
+
+# Admission Plugins to enable within the cluster
+# Default Plugins: NamespaceLifecycle, LimitRanger, ServiceAccount, TaintNodesByCondition, Priority, DefaultTolerationSeconds,
+#                  DefaultStorageClass, StorageObjectInUseProtection, PersistentVolumeClaimResize, RuntimeClass, CertificateApproval,
+#                  CertificateSigning, CertificateSubjectRestriction, DefaultIngressClass, MutatingAdmissionWebhook, ValidatingAdmissionWebhook, ResourceQuota
+cluster_apiserver_enable_admission_plugins: []
+
+# Admission Plugins to disable
+cluster_apiserver_disable_admission_plugins: []
+
+# Enable controller manager and scheduler accessible by cluster
+# necessary for metrics and monitoring; Default is '127.0.0.1'
+cluster_controller_manager_bind_address: '0.0.0.0'
+cluster_scheduler_bind_address: '0.0.0.0'
+
+# Number of terminated pods that can exist before the terminated pod
+# garbage collector starts deleting terminated pods. If <= 0, the terminated
+# pod garbage collector is disabled. Default: 12500
+cluster_controller_manager_terminated_pod_gc_threshold: 12500
+
+# Set the TLS Minimum version, Default VersionTLS11
+# Possible values: 'VersionTLS10', 'VersionTLS11', 'VersionTLS12', 'VersionTLS13'
+# cluster_tls_min_version: ''
+
+# Set the TLS Cipher Suites, Default GO cipher suites are used
+# cluster_tls_cipher_suites:
+# - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA
+# - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256
+# - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
+# - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA
+# - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
+# - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
+# - TLS_ECDHE_ECDSA_WITH_RC4_128_SHA
+# - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA
+# - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA
+# - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256
+# - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
+# - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA
+# - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
+# - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305
+# - TLS_ECDHE_RSA_WITH_RC4_128_SHA
+# - TLS_RSA_WITH_3DES_EDE_CBC_SHA
+# - TLS_RSA_WITH_AES_128_CBC_SHA
+# - TLS_RSA_WITH_AES_128_CBC_SHA256
+# - TLS_RSA_WITH_AES_128_GCM_SHA256
+# - TLS_RSA_WITH_AES_256_CBC_SHA
+# - TLS_RSA_WITH_AES_256_GCM_SHA384
+# - TLS_RSA_WITH_RC4_128_SHA
+
+cri_socket_map:
+  crio: 'unix:///var/run/crio/crio.sock'
+  containerd: 'unix:///run/containerd/containerd.sock'
+  docker: 'unix:///var/run/dockershim.sock'
+cri_socket: '{{ cri_socket_map[cri_plugin] }}'
+
+architecture_map:
+  amd64: 'amd64'
+  x86_64: 'amd64'
+  arm6l: 'arm'
+  arm7l: 'arm'
+  aarch64: 'arm64'
+  32-bit: '386'
+  64-bit: 'amd64'
+host_architecture: '{{ architecture_map[ansible_architecture] }}'
+
+
+# Old defaults
 cluster_name: kubernetes
 cluster_extra_sans:
   - "{{ keepalived_vip }}"
 cluster_control_plane_endpoint: "" # defaults to '{{ keepalived_vip }}:8443' when left empty
 cluster_enable_admission_plugins: "NodeRestriction"
 cluster_authorization_mode: "Node,RBAC"
-cluster_audit_log_path: "/var/log/kubernetes/audit.log"
-cluster_audit_log_maxage: "30"
-cluster_audit_log_maxbackup: "10"
-cluster_audit_log_maxsize: "100"
 cluster_pod_subnet: ""
 # Default etcd values, change these if you experience "leader changed" issues when running on a SD card
 cluster_etcd_heartbeat_interval: 100
diff --git a/ansible/roles/cluster/tasks/initialize.yml b/ansible/roles/cluster/tasks/initialize.yml
index bdb15a3e..fc2428ff 100644
--- a/ansible/roles/cluster/tasks/initialize.yml
+++ b/ansible/roles/cluster/tasks/initialize.yml
@@ -22,20 +22,20 @@
 
 - name: generate kubeadm configuration
   template:
-    src: kubeadm-init.yml.j2
-    dest: /etc/kubernetes/kubeadm-init.yml
+    src: kubeadm-init.yaml.j2
+    dest: /etc/kubernetes/kubeadm-init.yaml
     mode: 0644
 
 - name: initialize cluster (with kube-proxy)
   command:
-    cmd: kubeadm init --config /etc/kubernetes/kubeadm-init.yml --upload-certs
+    cmd: kubeadm init --config /etc/kubernetes/kubeadm-init.yaml --upload-certs
     creates: /etc/kubernetes/admin.conf
   when:
     - cluster_kube_proxy_enabled
 
 - name: initialize cluster (without kube-proxy)
   command:
-    cmd: kubeadm init --skip-phases=addon/kube-proxy --config /etc/kubernetes/kubeadm-init.yml --upload-certs
+    cmd: kubeadm init --skip-phases=addon/kube-proxy --config /etc/kubernetes/kubeadm-init.yaml --upload-certs
     creates: /etc/kubernetes/admin.conf
   when:
     - cluster_kube_proxy_enabled == false
diff --git a/ansible/roles/cluster/tasks/join.yml b/ansible/roles/cluster/tasks/join.yml
index 2bb10a71..80e9b4a8 100644
--- a/ansible/roles/cluster/tasks/join.yml
+++ b/ansible/roles/cluster/tasks/join.yml
@@ -5,12 +5,18 @@
 
 - name: generate kubeadm join configuration
   template:
-    src: kubeadm-join.yml.j2
-    dest: /etc/kubernetes/kubeadm-join.yml
+    src: kubeadm-join.yaml.j2
+    dest: /etc/kubernetes/kubeadm-join.yaml
     mode: 0644
   when: not cluster_node_configured
 
+- name: 'check/wait for cluster apiserver to be available'
+  ansible.builtin.wait_for:
+    host: '{{ hostvars[groups["masters"]|first]["ansible_default_ipv4"]["address"] }}'
+    port: '{{ cluster_apiserver_bind_port }}'
+    timeout: 180
+
 - name: join node to cluster
   command:
-    cmd: kubeadm join --config /etc/kubernetes/kubeadm-join.yml
+    cmd: kubeadm join --config /etc/kubernetes/kubeadm-join.yaml
     creates: /etc/kubernetes/kubelet.conf
diff --git a/ansible/roles/cluster/templates/kubeadm-init.yaml.j2 b/ansible/roles/cluster/templates/kubeadm-init.yaml.j2
new file mode 100644
index 00000000..d3a5b987
--- /dev/null
+++ b/ansible/roles/cluster/templates/kubeadm-init.yaml.j2
@@ -0,0 +1,111 @@
+#jinja2:lstrip_blocks: True
+---
+apiVersion: kubeadm.k8s.io/v1beta2
+kind: InitConfiguration
+bootstrapTokens:
+- token: {{ kubeadm_join_token }}
+  ttl: 1h
+  groups:
+  - system:bootstrappers:kubeadm:default-node-token
+  usages:
+  - signing
+  - authentication
+localAPIEndpoint:
+  advertiseAddress: {{ ansible_default_ipv4.address|default(ansible_all_ipv4_addresses[0]) }}
+  bindPort: {{ cluster_apiserver_bind_port }}
+certificateKey: {{ kubeadm_certificate_key }}
+nodeRegistration:
+  {% if inventory_hostname in groups['masters'] and inventory_hostname not in groups['workers'] %}
+  taints:
+  - effect: NoSchedule
+    key: node-role.kubernetes.io/master
+  {% else %}
+  taints: []
+  {% endif %}
+  criSocket: {{ cri_socket }}
+---
+apiVersion: kubeadm.k8s.io/v1beta2
+kind: ClusterConfiguration
+clusterName: {{ cluster_name }}
+certificatesDir: /etc/kubernetes/pki
+apiServer:
+  extraArgs:
+    authorization-mode: {{ cluster_authorization_modes | join(',') }}
+    bind-address: {{ cluster_apiserver_bind_address }}
+    apiserver-count: "{{ cluster_apiserver_count }}"
+    insecure-port: "0"
+    profiling: "{{ cluster_profiling }}"
+    {% if cluster_apiserver_enable_admission_plugins|length > 0 %}
+    enable-admission-plugins: {{ cluster_apiserver_enable_admission_plugins | join(',') }}
+    {% endif %}
+    {% if cluster_apiserver_disable_admission_plugins|length > 0 %}
+    disable-admission-plugins: {{ cluster_apiserver_disable_admission_plugins | join(',') }}
+    {% endif %}
+    {% if cluster_apiserver_enable_audit %}
+    audit-log-path: "{{ cluster_audit_log_path }}"
+    audit-log-maxage: "{{ cluster_audit_log_maxage }}"
+    audit-log-maxbackup: "{{ cluster_audit_log_maxbackup }}"
+    audit-log-maxsize: "{{ cluster_audit_log_maxsize }}"
+    {% endif %}
+    request-timeout: "1m0s"
+    service-account-lookup: "true"
+    {% if cluster_tls_min_version is defined %}
+    tls-min-version: {{ cluster_tls_min_version }}
+    {% endif %}
+    {% if cluster_tls_cipher_suites is defined %}
+    tls-cipher-suites: {{ cluster_tls_cipher_suites | join(',') }}
+    {% endif %}
+  certSANs:
+  {% for san in cluster_extra_sans %}
+  - {{ san }}
+  {% endfor %}
+  timeoutForControlPlane: "4m0s"
+etcd:
+  local:
+    {% if cluster_extra_sans | length > 0 %}
+    serverCertSANs:
+      {% for san in cluster_extra_sans %}
+      - {{ san }}
+      {% endfor %}
+    {% endif %}
+    dataDir: {{ cluster_etcd_datadir }}
+    extraArgs:
+      client-cert-auth: "true"
+      peer-client-cert-auth: "true"
+      peer-auto-tls: "false"
+      heartbeat-interval: "{{ cluster_etcd_heartbeat_interval }}"
+      election-timeout: "{{ cluster_etcd_election_timeout }}"
+imageRepository: {{ cluster_image_repository }}
+networking:
+  dnsDomain: cluster.local
+  serviceSubnet: {{ cluster_service_subnet }}
+  podSubnet: {{ cluster_pod_subnet }}
+kubernetesVersion: {{ cluster_kubernetes_version }}
+controlPlaneEndpoint: {{ cluster_control_plane_endpoint }}
+controllerManager:
+  extraArgs:
+    bind-address: {{ cluster_controller_manager_bind_address }}
+    terminated-pod-gc-threshold: "{{ cluster_controller_manager_terminated_pod_gc_threshold }}"
+    profiling: "{{ cluster_profiling }}"
+    use-service-account-credentials: "true"
+    feature-gates: "RotateKubeletServerCertificate=true"
+    {% if cluster_tls_min_version is defined %}
+    tls-min-version: {{ cluster_tls_min_version }}
+    {% endif %}
+    {% if cluster_tls_cipher_suites is defined %}
+    tls-cipher-suites: {{ cluster_tls_cipher_suites | join(',') }}
+    {% endif %}
+scheduler:
+  extraArgs:
+    bind-address: {{ cluster_scheduler_bind_address }}
+    profiling: "{{ cluster_profiling }}"
+    {% if cluster_tls_min_version is defined %}
+    tls-min-version: {{ cluster_tls_min_version }}
+    {% endif %}
+    {% if cluster_tls_cipher_suites is defined %}
+    tls-cipher-suites: {{ cluster_tls_cipher_suites | join(',') }}
+    {% endif %}
+---
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+cgroupDriver: {{ cluster_cgroup_driver }}
diff --git a/ansible/roles/cluster/templates/kubeadm-init.yml.j2 b/ansible/roles/cluster/templates/kubeadm-init.yml.j2
deleted file mode 100644
index ca13b89a..00000000
--- a/ansible/roles/cluster/templates/kubeadm-init.yml.j2
+++ /dev/null
@@ -1,76 +0,0 @@
-#jinja2:lstrip_blocks: True
----
-apiVersion: kubeadm.k8s.io/v1beta2
-kind: InitConfiguration
-bootstrapTokens:
-  - token: {{ kubeadm_join_token }}
-    ttl: 1h
-    groups:
-      - system:bootstrappers:kubeadm:default-node-token
-    usages:
-      - signing
-      - authentication
-certificateKey: {{ kubeadm_certificate_key }}
----
-apiVersion: kubeadm.k8s.io/v1beta2
-kind: ClusterConfiguration
-clusterName: {{ cluster_name }}
-certificatesDir: /etc/kubernetes/pki
-controlPlaneEndpoint: {{ cluster_control_plane_endpoint }}
-apiServer:
-  {% if cluster_extra_sans | length > 0 %}
-  certSANs:
-    {% for san in cluster_extra_sans %}
-    - {{ san }}
-    {% endfor %}
-  {% endif %}
-  extraArgs:
-    audit-log-path: "{{ cluster_audit_log_path }}"
-    audit-log-maxage: "{{ cluster_audit_log_maxage }}"
-    audit-log-maxbackup: "{{ cluster_audit_log_maxbackup }}"
-    audit-log-maxsize: "{{ cluster_audit_log_maxsize }}"
-    authorization-mode: {{ cluster_authorization_mode }}
-    enable-admission-plugins: {{ cluster_enable_admission_plugins }}
-    insecure-port: "0"
-    kubelet-https: "true"
-    profiling: "false"
-    request-timeout: "1m0s"
-    secure-port: "6443"
-    service-account-lookup: "true"
-    tls-cipher-suites: "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384"
-  timeoutForControlPlane: "4m0s"
-etcd:
-  local:
-    {% if cluster_extra_sans | length > 0 %}
-    serverCertSANs:
-      {% for san in cluster_extra_sans %}
-      - {{ san }}
-      {% endfor %}
-    {% endif %}
-    dataDir: {{ cluster_etcd_datadir }}
-    extraArgs:
-      client-cert-auth: "true"
-      peer-client-cert-auth: "true"
-      peer-auto-tls: "false"
-      heartbeat-interval: "{{ cluster_etcd_heartbeat_interval }}"
-      election-timeout: "{{ cluster_etcd_election_timeout }}"
-imageRepository: {{ cluster_image_repository }}
-networking:
-  dnsDomain: cluster.local
-  {%  if cluster_service_subnet | ipaddr %}
-  serviceSubnet: {{ cluster_service_subnet }}
-  {% endif %}
-  {%  if cluster_pod_subnet | ipaddr %}
-  podSubnet: {{ cluster_pod_subnet }}
-  {% endif %}
-controllerManager:
-  extraArgs:
-    bind-address: 127.0.0.1
-    terminated-pod-gc-threshold: "50"
-    profiling: "false"
-    use-service-account-credentials: "true"
-    feature-gates: "RotateKubeletServerCertificate=true"
-scheduler:
-  extraArgs:
-    bind-address: 127.0.0.1
-    profiling: "false"
diff --git a/ansible/roles/cluster/templates/kubeadm-join.yml.j2 b/ansible/roles/cluster/templates/kubeadm-join.yaml.j2
similarity index 55%
rename from ansible/roles/cluster/templates/kubeadm-join.yml.j2
rename to ansible/roles/cluster/templates/kubeadm-join.yaml.j2
index fa46b2f3..c27b8304 100644
--- a/ansible/roles/cluster/templates/kubeadm-join.yml.j2
+++ b/ansible/roles/cluster/templates/kubeadm-join.yaml.j2
@@ -7,11 +7,20 @@ discovery:
   bootstrapToken:
     apiServerEndpoint: {{ cluster_control_plane_endpoint }}
     token: {{ kubeadm_join_token }}
+    {% if cluster_ca_sha256 is defined %}
     unsafeSkipCAVerification: false
     caCertHashes:
-      - sha256:{{ cluster_ca_sha256 }}
+    - sha256:{{ cluster_ca_sha256 }}
+    {% else %}
+    unsafeSkipCAVerification: true
+    {% endif %}
   timeout: 5m0s
 {% if inventory_hostname in groups['masters'] %}
 controlPlane:
+  localAPIEndpoint:
+    advertiseAddress: {{ ansible_default_ipv4.address|default(ansible_all_ipv4_addresses[0]) }}
+    bindPort: {{ cluster_apiserver_bind_port }}
   certificateKey: {{ kubeadm_certificate_key }}
 {% endif %}
+nodeRegistration:
+  criSocket: {{ cri_socket }}
diff --git a/ansible/roles/cluster/vars/main.yml b/ansible/roles/cluster/vars/main.yml
index a92d4922..b0655c4f 100644
--- a/ansible/roles/cluster/vars/main.yml
+++ b/ansible/roles/cluster/vars/main.yml
@@ -1,4 +1,6 @@
 ---
+# vars file for cluster
+# Old Vars
 cluster_main_master: "{{ groups['masters'][0] }}"
 cluster_etcd_datadir: /var/lib/etcd
 cluster_image_repository: k8s.gcr.io

From 96e82ed0549a11151305c084d85fc1e879e87e96 Mon Sep 17 00:00:00 2001
From: Nick M <4718+rkage@users.noreply.github.com>
Date: Sun, 31 Jan 2021 21:22:03 -0500
Subject: [PATCH 2/3] minor changes, added variables, merged others

Signed-off-by: Nick M <4718+rkage@users.noreply.github.com>
---
 ansible/group_vars/cluster.yml                | 25 +++++++++----
 ansible/roles/cluster/defaults/main.yml       | 36 +++++++++++++------
 .../cluster/templates/kubeadm-init.yaml.j2    |  3 +-
 ansible/roles/cluster/vars/main.yml           |  7 ----
 4 files changed, 45 insertions(+), 26 deletions(-)
 delete mode 100644 ansible/roles/cluster/vars/main.yml

diff --git a/ansible/group_vars/cluster.yml b/ansible/group_vars/cluster.yml
index fa290149..0db78723 100644
--- a/ansible/group_vars/cluster.yml
+++ b/ansible/group_vars/cluster.yml
@@ -1,7 +1,10 @@
 ---
 
 # Specify the Kubernetes version, current release is v1.20.0
-cluster_kubernetes_version: v1.20.2
+cluster_kubernetes_version: 'v1.20.2'
+
+cluster_apiserver_enable_admission_plugins:
+- 'NodeRestriction'
 
 cluster_cgroup_driver: 'systemd'
 
@@ -28,9 +31,20 @@ cluster_node_network_prefix: 24
 cluster_authorization_modes: ['Node', 'RBAC']
 
 # Vars for discoverying kubernetes api endpoints
+# will refactor to eliminate this variable
 cluster_primary_control_node: '{{ groups["masters"]|first }}'
 cluster_apiserver_count: '{{ groups["masters"]|length }}'
 
+# Tuning for arm64 platforms; https://etcd.io/docs/v3.4.0/tuning/
+cluster_etcd_heartbeat_interval: 250
+cluster_etcd_election_timeout: 1250
+cluster_etcd_snapshot_count: 5000
+
+####
+# Role - cri
+####
+cri_plugin: 'containerd'
+cri_containerd_version: '1.4.3'
 
 ## Old variables
 ####
@@ -44,16 +58,13 @@ keepalived_interface: "{{ ansible_default_ipv4['interface'] }}"
 ####
 cluster_name: kubernetes
 cluster_extra_sans:
-  - "{{ keepalived_vip }}"
-cluster_control_plane_endpoint: "{{ keepalived_vip }}:8443"
-cluster_enable_admission_plugins: "NodeRestriction"
-cluster_etcd_heartbeat_interval: 100
-cluster_etcd_election_timeout: 1000
+- '{{ keepalived_vip }}'
+cluster_control_plane_endpoint: '{{ keepalived_vip }}:8443'
 
 ####
 # Role - cni
 ####
-cni_plugin: calico
+cni_plugin: 'calico'
 # cni_bgp_peer_address: 192.168.0.1
 # cni_bgp_peer_asn: 64512
 # cni_cilium_helm_version: 1.9.3
diff --git a/ansible/roles/cluster/defaults/main.yml b/ansible/roles/cluster/defaults/main.yml
index 599e1b37..9c0b146e 100644
--- a/ansible/roles/cluster/defaults/main.yml
+++ b/ansible/roles/cluster/defaults/main.yml
@@ -1,16 +1,20 @@
 ---
 # API Server Variables
 
-# API Server bind address, by default listens to all requests
+# API Server bind address, default listen to all requests
 cluster_apiserver_bind_address: '0.0.0.0'
 
-# API Server secure port, by default listens to 6443
+# API Server secure port, default listen to 6443
 cluster_apiserver_bind_port: 6443
 
+# API Server insecure port, by default this port is disabled (0)
+# This configuration item has been marked deprecated and will be removed in future.
+cluster_apiserver_insecure_port: 0
+
 # Enable cluster audit log
 cluster_apiserver_enable_audit: false
 # path to the audit log
-cluster_apiserver_audit_log_path: /var/log/kubernetes/audit.log
+cluster_apiserver_audit_log_path: '/var/log/kubernetes/audit.log'
 # audit log max age in days
 cluster_apiserver_audit_log_maxage: 30
 # number of audit logs to keep as backup
@@ -37,6 +41,19 @@ cluster_scheduler_bind_address: '0.0.0.0'
 # pod garbage collector is disabled. Default: 12500
 cluster_controller_manager_terminated_pod_gc_threshold: 12500
 
+# Time in milliseconds; Time should be adjusted for 0.5-1.5x RTT between nodes.
+# If heartbeat is too low, etcd will send unnecessary messages that increase CPU.
+# Defaults: _heartbeat_interval: 100, _election_timeout: 1000
+cluster_etcd_heartbeat_interval: 100
+cluster_etcd_election_timeout: 1000
+
+# Number of committed transactions to trigger a snapshot to disk.
+# Default: 100000
+cluster_etcd_snapshot_count: 100000
+
+# Default will install kube-proxy although some CNI plugins will recommend it be disabled
+cluster_kube_proxy_enabled: true
+
 # Set the TLS Minimum version, Default VersionTLS11
 # Possible values: 'VersionTLS10', 'VersionTLS11', 'VersionTLS12', 'VersionTLS13'
 # cluster_tls_min_version: ''
@@ -88,11 +105,8 @@ cluster_name: kubernetes
 cluster_extra_sans:
   - "{{ keepalived_vip }}"
 cluster_control_plane_endpoint: "" # defaults to '{{ keepalived_vip }}:8443' when left empty
-cluster_enable_admission_plugins: "NodeRestriction"
-cluster_authorization_mode: "Node,RBAC"
-cluster_pod_subnet: ""
-# Default etcd values, change these if you experience "leader changed" issues when running on a SD card
-cluster_etcd_heartbeat_interval: 100
-cluster_etcd_election_timeout: 1000
-# Default will install kube-proxy although some CNI plugins will recommend it be disabled
-cluster_kube_proxy_enabled: true
+
+# Old Vars
+cluster_main_master: "{{ groups['masters'][0] }}"
+cluster_etcd_datadir: /var/lib/etcd
+cluster_image_repository: k8s.gcr.io
diff --git a/ansible/roles/cluster/templates/kubeadm-init.yaml.j2 b/ansible/roles/cluster/templates/kubeadm-init.yaml.j2
index d3a5b987..6f27e44e 100644
--- a/ansible/roles/cluster/templates/kubeadm-init.yaml.j2
+++ b/ansible/roles/cluster/templates/kubeadm-init.yaml.j2
@@ -33,7 +33,7 @@ apiServer:
     authorization-mode: {{ cluster_authorization_modes | join(',') }}
     bind-address: {{ cluster_apiserver_bind_address }}
     apiserver-count: "{{ cluster_apiserver_count }}"
-    insecure-port: "0"
+    insecure-port: "{{ cluster_apiserver_insecure_port }}"
     profiling: "{{ cluster_profiling }}"
     {% if cluster_apiserver_enable_admission_plugins|length > 0 %}
     enable-admission-plugins: {{ cluster_apiserver_enable_admission_plugins | join(',') }}
@@ -75,6 +75,7 @@ etcd:
       peer-auto-tls: "false"
       heartbeat-interval: "{{ cluster_etcd_heartbeat_interval }}"
       election-timeout: "{{ cluster_etcd_election_timeout }}"
+      snapshot-count: "{{ cluster_etcd_snapshot_count }}"
 imageRepository: {{ cluster_image_repository }}
 networking:
   dnsDomain: cluster.local
diff --git a/ansible/roles/cluster/vars/main.yml b/ansible/roles/cluster/vars/main.yml
deleted file mode 100644
index b0655c4f..00000000
--- a/ansible/roles/cluster/vars/main.yml
+++ /dev/null
@@ -1,7 +0,0 @@
----
-# vars file for cluster
-# Old Vars
-cluster_main_master: "{{ groups['masters'][0] }}"
-cluster_etcd_datadir: /var/lib/etcd
-cluster_image_repository: k8s.gcr.io
-cluster_service_subnet: 10.96.0.0/12

From 826f6d1d15923021bddef8ea40234fe8854a0bd2 Mon Sep 17 00:00:00 2001
From: Nick M <4718+rkage@users.noreply.github.com>
Date: Sun, 31 Jan 2021 21:35:45 -0500
Subject: [PATCH 3/3] clean up pre-commit findings, remove ssh private key,
 EOLs, etc

Signed-off-by: Nick M <4718+rkage@users.noreply.github.com>
---
 infrastructure/gcp/install.sh                 |  1 -
 .../libvirt/ubuntu/aarch64/ubuntu-aarch64.tf  |  1 -
 .../libvirt/ubuntu/x86-64/ssh_id_shared       | 27 -------------------
 3 files changed, 29 deletions(-)
 delete mode 100644 infrastructure/libvirt/ubuntu/x86-64/ssh_id_shared

diff --git a/infrastructure/gcp/install.sh b/infrastructure/gcp/install.sh
index 48c172b6..1e53a4b9 100755
--- a/infrastructure/gcp/install.sh
+++ b/infrastructure/gcp/install.sh
@@ -57,4 +57,3 @@ gsutil versioning set on "gs://${TF_VAR_PROJ_ID}-terraform-state"
 # Configure your environment for the Google Cloud Terraform provider
 export GOOGLE_APPLICATION_CREDENTIALS=${TF_CREDS}
 export GOOGLE_PROJECT=${TF_VAR_PROJ_ID}
-
diff --git a/infrastructure/libvirt/ubuntu/aarch64/ubuntu-aarch64.tf b/infrastructure/libvirt/ubuntu/aarch64/ubuntu-aarch64.tf
index 6c23c707..a73e183d 100644
--- a/infrastructure/libvirt/ubuntu/aarch64/ubuntu-aarch64.tf
+++ b/infrastructure/libvirt/ubuntu/aarch64/ubuntu-aarch64.tf
@@ -98,4 +98,3 @@ resource "libvirt_domain" "node" {
   }
 
 }
-
diff --git a/infrastructure/libvirt/ubuntu/x86-64/ssh_id_shared b/infrastructure/libvirt/ubuntu/x86-64/ssh_id_shared
deleted file mode 100644
index 1907e13c..00000000
--- a/infrastructure/libvirt/ubuntu/x86-64/ssh_id_shared
+++ /dev/null
@@ -1,27 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-MIIEogIBAAKCAQEAthu5NMxgI3ftC84Wz3Bi5HSayfy2y6xcbV3vizwJK6YYs30c
-XWaUguw76+VXsK5Hf5H2Dm4g3Or9twx8c3IpVQthxi0FDXlHq/2BNZi+JZ1wtTN3
-G+HlPryfxsq/V1TjkXg0QwQsKLmi+yM4GDQrJtDub9s/8PboJ0S68lGNfNP6CBnh
-P2jiNR0YemDrCJ+0d0b+ecvAmEqrG5pR1ogwuomSAmbb2C2JlJgoD19hYNnk/mkW
-wErVHFBEZvIduOn7NSscsHRxyP+qNxNTTSDyENC3cyJ+Wlr0wneg+ddS+K3/5Oc2
-3cPxqpdAMoBwUf8J54AeYEgxESyn4oM3iJQinwIDAQABAoIBAAOPPSqECCkJUg/W
-Amohlz1B7jiegPAwOUpF+MWu8brxry/q9R/6PlLkzTMKM5GXhnw+3SFhE7FdwoHO
-c6xKAfbQlyOpg5zEzxrUhK8Igkb38dkI3X4lBzcXtsPK+Lx7Q0t3nwXkUfylzEK+
-aH1In8NNeCuljHVoThdeZ40O/QYAkkFOxyRuiUJHDBoAe09ofQNfCwVOCjhLJZ31
-4TlbcPNlYoISKBztDqtSqxuIZy4eQIqnliXn7VlV36qqKyFDomMiR0Do0L00XitH
-XChiwPs+etW2+RAyQY1Oib4XZqmIxSjR/f6UhiQ0TTmjSBLnWe6KAFKw06v75Bc2
-YoaobakCgYEA5GO/BW9rlL/TfMSg5D6XxRaQRhsudOt70ikXm7OS/xqxS+h+Chwu
-q1bpVxAOD/OAbxO07LbK4559ejLrh29PB4QcsPFG//jgzHS+evsctdXNSdEQwF3S
-Sm/uSBV6S/sGVMH1Fg8LNKbqIKYz48GEbivuCEaV9jLKesiiqjLgfhUCgYEAzB+m
-6MKqB4zO1A5dPcSs7AItJjbVWMxPXWwdwOF51CFGzoDlYprJlzhrYgs0zDoaYMCC
-15pfHjQ8gcCMhgOB11wwhEpBTKNVjH5HXkQ+t1WzRmy/2bse/4sTB6Ev6W1IASa2
-aW+xMy0nvdYj1qtfdmJA+i8Mr9CugGpRt0sefuMCgYA+2oSThlphEXmueJ6esesq
-vk7uPETIwJXO/KgWnZDyy/HHJjCpr5/YX47P+wcjtKu/5fiAvL2RbuixXJodJ7Dk
-LKOIvp3/jAqxud9CESGRmUlp6zNk/W0GN0+oXuDNUEQpx0cfzvwSfW0plxPotjUv
-7L8RX/udSdUXfWfm1yGZEQKBgFs7DlNbpbjIIyy92sifnZXCB0HcIhCVCCe+CbJK
-VDQOR2JGU791ZQI0UjI4xfk9QsrZX+FS+o46VIy9bVD8QkzpxDlsIrFJExQ+4FDu
-eYVhRUYFlt2Od3VJSFERpqNSl2sETWTkabNwzCFZkxz/7SOxGE95NddikouMeUdN
-yKx3AoGAOp6PfnqgxrbFp6O46NcRXtWNLKKulsi/m1mmPAuDsHQJPYKjpsi9iB8a
-jSM46H8miC4aqfCln5pDHy0kG7vG//0R0SnByzy5KW+KbAvv2eDU2WO8b9SMXkmH
-tGE3GP+Pj4qHaTLacHos6s6W2UOSMWxU1cWUGikgpVXdQngx9bY=
------END RSA PRIVATE KEY-----