diff --git a/cleanup.sh b/cleanup.sh index 73b2af93..70677d7e 100755 --- a/cleanup.sh +++ b/cleanup.sh @@ -29,3 +29,8 @@ do_footloose() { log "Deleting virtual machines" export PATH=~/.wks/bin:${PATH} do_footloose delete + +# Especially as it relates to ignite machines, we may need to remove because `wksctl apply` +# will otherwise fail. This is due to footloose IPs inrementing using CNI bridge IPAM: +# sudo cat /var/lib/cni/networks/ignite-cni-bridge/last_reserved_ip.0 +rm -f machines.yaml footloose.yaml diff --git a/cluster.yaml b/cluster.yaml index 41963cc5..0eb02cb6 100644 --- a/cluster.yaml +++ b/cluster.yaml @@ -13,7 +13,7 @@ spec: value: apiVersion: baremetalproviderspec/v1alpha1 kind: BareMetalClusterProviderSpec - sshKeyPath: cluster-key + # sshKeyPath: cluster-key user: root os: files: @@ -32,5 +32,5 @@ spec: cri: kind: docker package: docker-ce - version: 18.09.7 + version: 19.03.11 diff --git a/config.yaml b/config.yaml index 10fe09a0..ddfd3852 100644 --- a/config.yaml +++ b/config.yaml @@ -9,8 +9,49 @@ # https://github.com/weaveworks/ignite. backend: ignite -# Number of nodes allocated for the Kubernetes control plane and workers. -controlPlane: - nodes: 1 -workers: - nodes: 1 +cluster: + name: example + namespace: weavek8sops + controlPlane: + nodes: 1 + cpus: 2 # multiples of 2 + memory: 1GB + diskSize: 5GB + workers: + nodes: 1 + cpus: 4 # multiples of 2 + memory: 1GB + diskSize: 10GB +versions: + # https://kubernetes.io/docs/setup/release/version-skew-policy/ + # https://github.com/kubernetes/kubelet/releases + kubelet: 1.17.5 + # https://github.com/jkcfg/jk/releases + jk: 0.4.0 + # https://github.com/weaveworks/footloose/releases + footloose: 0.6.3 + # https://github.com/weaveworks/ignite/releases + ignite: 0.7.0 + # https://github.com/weaveworks/wksctl/releases + wksctl: 0.8.3-rc.1 + # https://github.com/docker/docker-ce/releases + docker: 19.03.11 +images: + # https://github.com/weaveworks/footloose/releases + footloose: quay.io/footloose/centos7:0.6.3 + # https://hub.docker.com/r/weaveworks/ignite-centos/tags + ignite: docker.io/weaveworks/ignite-centos:7 + # https://hub.docker.com/r/weaveworks/ignite-kernel/tags + kernel: docker.io/weaveworks/ignite-kernel:5.4.43 + # Manage cluster and machine descriptions using Git + # https://hub.docker.com/r/weaveworks/wksctl-controller/tags + wksctl: docker.io/weaveworks/wksctl-controller:v0.8.3-rc.1 + # CD propagation of images and config changes to the cluster + # https://github.com/fluxcd/flux/releases + # https://hub.docker.com/r/fluxcd/flux/tags + flux: docker.io/fluxcd/flux:1.19.0 + # General-purpose distributed memory object caching system + # https://github.com/memcached/memcached/wiki/ReleaseNotes + # https://hub.docker.com/_/memcached?tab=tags + memcached: docker.io/memcached:1.6.6 + # https://github.com/weaveworks/weave/releases/ #download/v2.6.2/weave diff --git a/flux.yaml b/flux.yaml index 3437c8e6..b901b1b3 100644 --- a/flux.yaml +++ b/flux.yaml @@ -63,7 +63,7 @@ items: - args: - -m 64 - -p 11211 - image: memcached:1.4.25 + image: docker.io/memcached:1.6.6 imagePullPolicy: IfNotPresent name: memcached ports: @@ -110,7 +110,7 @@ items: containers: - args: - --ssh-keygen-dir=/var/fluxd/keygen - - --git-url=https://github.com/weaveworks/wks-quickstart-firekube.git + - --git-url=https://github.com/roobyz/wks-quickstart-firekube.git - --git-branch=master - --git-poll-interval=30s - --git-path=. @@ -119,8 +119,7 @@ items: - --memcached-service=memcached - --listen-metrics=:3031 - --sync-garbage-collection - - --manifest-generation=true - image: fluxcd/flux:1.14.2 + image: docker.io/fluxcd/flux:1.19.0 imagePullPolicy: IfNotPresent name: flux ports: diff --git a/footloose.yaml b/footloose.yaml new file mode 100644 index 00000000..e7025653 --- /dev/null +++ b/footloose.yaml @@ -0,0 +1,46 @@ +cluster: + name: example + privateKey: cluster-key +machines: +- count: 1 + spec: + backend: ignite + ignite: + cpus: 2 + diskSize: 5GB + kernel: docker.io/weaveworks/ignite-kernel:5.4.43 + memory: 1GB + image: docker.io/weaveworks/ignite-centos:7 + name: ctrl-%d + portMappings: + - containerPort: 22 + hostPort: 2222 + - containerPort: 6443 + hostPort: 6443 + - containerPort: 30443 + hostPort: 30443 + - containerPort: 30080 + hostPort: 30080 + privileged: false + volumes: [] +- count: 1 + spec: + backend: ignite + ignite: + cpus: 4 + diskSize: 10GB + kernel: docker.io/weaveworks/ignite-kernel:5.4.43 + memory: 1GB + image: docker.io/weaveworks/ignite-centos:7 + name: work-%d + portMappings: + - containerPort: 22 + hostPort: 2222 + - containerPort: 6443 + hostPort: 6443 + - containerPort: 30443 + hostPort: 30443 + - containerPort: 30080 + hostPort: 30080 + privileged: false + volumes: [] diff --git a/lib/functions.sh b/lib/functions.sh index 1b1cd5da..720de886 100644 --- a/lib/functions.sh +++ b/lib/functions.sh @@ -188,3 +188,68 @@ git_remote_fetchurl() { git config --get "remote.${1}.url" } +parse_yaml() { + local prefix=$2 + local s='[[:space:]]*' w='[a-zA-Z0-9_]*' fs=$(echo @|tr @ '\034') + sed -ne "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \ + -e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p" $1 | + awk -F$fs '{ + indent = length($1)/2; + vname[indent] = $2; + for (i in vname) {if (i > indent) {delete vname[i]}} + if (length($3) > 0) { + vn=""; for (i=0; i "${tmp}" && \ + mv "${tmp}" config.yaml && \ + rm -f "${tmp}" +} + +set_docker_version() { + local tmp=.config.yaml.tmp + + sed -e "s/\bversion: .*$/version: ${1}/" cluster.yaml > "${tmp}" && \ + mv "${tmp}" cluster.yaml && \ + rm -f "${tmp}" +} + +do_footloose() { + if [ "$config_backend" == "ignite" ]; then + $sudo env "PATH=${PATH}" footloose "${@}" + else + footloose "${@}" + fi +} + +set_wksctl_version() { + local tmp=.config.yaml.tmp + + awk -v repl="image: ${1}" '/image:/ && /wksctl/ { + sub(/image.*/, repl) } 1' wks-controller.yaml > "${tmp}" && \ + mv "${tmp}" wks-controller.yaml && \ + rm -f "${tmp}" +} + +set_flux_version() { + local tmp=.config.yaml.tmp + + awk -v repl="image: ${1}" '/image:/ && /memcached/ { + sub(/image.*/, repl) } 1' flux.yaml > "${tmp}" && \ + mv "${tmp}" flux.yaml && \ + rm -f "${tmp}" + awk -v repl="image: ${2}" '/image:/ && /flux/ { + sub(/image.*/, repl) } 1' flux.yaml > "${tmp}" && \ + mv "${tmp}" flux.yaml && \ + rm -f "${tmp}" + awk -v repl="git-url=${3}" '/git-url=/ { + sub(/git-url=.*/, repl) } 1' flux.yaml > "${tmp}" && \ + mv "${tmp}" flux.yaml && \ + rm -f "${tmp}" +} diff --git a/machines.yaml b/machines.yaml new file mode 100644 index 00000000..66962c07 --- /dev/null +++ b/machines.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +items: +- apiVersion: cluster.k8s.io/v1alpha1 + kind: Machine + metadata: + labels: + set: master + name: master-0 + namespace: weavek8sops + spec: + providerSpec: + value: + apiVersion: baremetalproviderspec/v1alpha1 + kind: BareMetalMachineProviderSpec + private: + address: 10.61.0.6 + port: 22 + public: + address: 127.0.0.1 + port: 2222 + versions: + kubelet: 1.17.5 +- apiVersion: cluster.k8s.io/v1alpha1 + kind: Machine + metadata: + labels: + set: worker + name: worker-0 + namespace: weavek8sops + spec: + providerSpec: + value: + apiVersion: baremetalproviderspec/v1alpha1 + kind: BareMetalMachineProviderSpec + private: + address: 10.61.0.7 + port: 22 + public: + address: 127.0.0.1 + port: 2223 + versions: + kubelet: 1.17.5 +kind: List diff --git a/setup.js b/setup.js index 3dc8d5eb..809cf2b4 100644 --- a/setup.js +++ b/setup.js @@ -4,11 +4,9 @@ import * as std from '@jkcfg/std' const config = param.all(); let output = []; -const numNodes = config => config.controlPlane.nodes + config.workers.nodes; - const backend = { docker: { - image: 'quay.io/footloose/centos7:0.6.0', + image: config.images.footloose, // The below is required for dockerd to run smoothly. // See also: https://github.com/weaveworks/footloose#running-dockerd-in-container-machines privileged: true, @@ -18,33 +16,64 @@ const backend = { }] }, ignite: { - image: 'weaveworks/ignite-centos:firekube-pre3', + image: config.images.ignite, privileged: false, volumes: [], + ctrl_plane: { + cpus: config.cluster.controlPlane.cpus, + memory: config.cluster.controlPlane.memory, + diskSize: config.cluster.controlPlane.diskSize, + kernel: config.images.kernel, + }, + work_node: { + cpus: config.cluster.workers.cpus, + memory: config.cluster.workers.memory, + diskSize: config.cluster.workers.diskSize, + kernel: config.images.kernel, + }, }, }; const image = config => backend[config.backend].image; const privileged = config => backend[config.backend].privileged; const volumes = config => backend[config.backend].volumes; +const ctrl_plane = config => backend[config.backend].ctrl_plane; +const work_node = config => backend[config.backend].work_node; const footloose = config => ({ cluster: { - name: 'firekube', + name: config.cluster.name, privateKey: 'cluster-key', }, machines: [{ - count: numNodes(config), + count: config.cluster.controlPlane.nodes, spec: { + name: 'ctrl-%d', image: image(config), - name: 'node%d', backend: config.backend, - ignite: { - cpus: 2, - memory: '1GB', - diskSize: '5GB', - kernel: 'weaveworks/ignite-kernel:4.19.47', - }, + ignite: ctrl_plane(config), + portMappings: [{ + containerPort: 22, + hostPort: 2222, + }, { + containerPort: 6443, + hostPort: 6443, + }, { + containerPort: 30443, + hostPort: 30443, + }, { + containerPort: 30080, + hostPort: 30080, + }], + privileged: privileged(config), + volumes: volumes(config), + }},{ + count: config.cluster.workers.nodes, + spec: { + name: 'work-%d', + image: image(config), + backend: config.backend, + ignite: work_node(config), portMappings: [{ containerPort: 22, hostPort: 2222, @@ -82,7 +111,7 @@ const Machine = ({ id, privateIP, sshPort, role }) => ({ set: role, }, name: `${role}-${id}`, - namespace: 'weavek8sops' + namespace: config.cluster.namespace }, spec: { providerSpec: { @@ -98,6 +127,9 @@ const Machine = ({ id, privateIP, sshPort, role }) => ({ port: 22, } } + }, + versions: { + kubelet: config.versions.kubelet } } }); @@ -107,7 +139,7 @@ const sshPort = machine => machine.ports.find(p => p.guest == 22).host; if (config.machines !== undefined) { const machines = []; - for (let i = 0; i < config.controlPlane.nodes; i++ ) { + for (let i = 0; i < config.cluster.controlPlane.nodes; i++ ) { const machine = config.machines[i]; machines.push(Machine({ id: i, @@ -117,8 +149,8 @@ if (config.machines !== undefined) { })); } - for (let i = 0; i < config.workers.nodes; i++ ) { - const machine = config.machines[config.controlPlane.nodes + i]; + for (let i = 0; i < config.cluster.workers.nodes; i++ ) { + const machine = config.machines[config.cluster.controlPlane.nodes + i]; machines.push(Machine({ id: i, privateIP: machine.runtimeNetworks[0].ip, diff --git a/setup.sh b/setup.sh index 97cf1d86..1e4b9cfd 100755 --- a/setup.sh +++ b/setup.sh @@ -19,30 +19,13 @@ fi set -euo pipefail -JK_VERSION=0.3.0 -FOOTLOOSE_VERSION=0.6.2 -IGNITE_VERSION=0.7.0 -WKSCTL_VERSION=0.8.1 - -config_backend() { - sed -n -e 's/^backend: *\(.*\)/\1/p' config.yaml -} - -set_config_backend() { - local tmp=.config.yaml.tmp - - sed -e "s/^backend: .*$/backend: ${1}/" config.yaml > "${tmp}" && \ - mv "${tmp}" config.yaml && \ - rm -f "${tmp}" -} +# On macOS, we only support the docker backend. +if [ "$(goos)" == "darwin" ]; then + set_config_backend docker +fi -do_footloose() { - if [ "$(config_backend)" == "ignite" ]; then - $sudo env "PATH=${PATH}" footloose "${@}" - else - footloose "${@}" - fi -} +# Parse all the config values after checking for macOS +eval $(parse_yaml config.yaml "config_") if git_current_branch > /dev/null 2>&1; then log "Using git branch: $(git_current_branch)" @@ -121,20 +104,17 @@ if [ "${download}" == "yes" ]; then export PATH="${HOME}/.wks/bin:${PATH}" fi -# On macOS, we only support the docker backend. -if [ "$(goos)" == "darwin" ]; then - set_config_backend docker -fi - check_command docker -check_version jk "${JK_VERSION}" -check_version footloose "${FOOTLOOSE_VERSION}" -if [ "$(config_backend)" == "ignite" ]; then - check_version ignite "${IGNITE_VERSION}" +check_version jk "${config_versions_jk}" +check_version footloose "${config_versions_footloose}" +if [ "$config_backend" == "ignite" ]; then + check_version ignite "${config_versions_ignite}" fi -check_version wksctl "${WKSCTL_VERSION}" +check_version wksctl "${config_versions_wksctl}" + +set_docker_version ${config_versions_docker} -log "Creating footloose manifest" +log "Creating footloose manifests" jk generate -f config.yaml setup.js cluster_key="cluster-key" @@ -147,6 +127,7 @@ fi log "Creating virtual machines" do_footloose create +# The machines yaml is created using data from the footloose status json and the config.yaml log "Creating Cluster API manifests" status="footloose-status.yaml" do_footloose status -o json > "${status}" @@ -154,7 +135,10 @@ jk generate -f config.yaml -f "${status}" setup.js rm -f "${status}" log "Updating container images and git parameters" -wksctl init --git-url="$(git_http_url "$(git_remote_fetchurl "${git_remote}")")" --git-branch="$(git_current_branch)" +# FIXME: wksctl init fails to update flux.yaml and wks-controller.yaml +# wksctl init -e --git-url="$(git_http_url "$(git_remote_fetchurl "${git_remote}")")" --git-branch="$(git_current_branch)" +set_flux_version ${config_images_memcached} ${config_images_flux} "$(git_http_url "$(git_remote_fetchurl "${git_remote}")")" +set_wksctl_version ${config_images_wksctl} log "Pushing initial cluster configuration" git add config.yaml footloose.yaml machines.yaml flux.yaml wks-controller.yaml @@ -169,4 +153,5 @@ apply_args=( ) [ "${git_deploy_key}" ] && apply_args+=("${git_deploy_key}") wksctl apply "${apply_args[@]}" +# wksctl apply #--verbose wksctl kubeconfig diff --git a/wks-controller.yaml b/wks-controller.yaml index 9777cb47..408574fe 100644 --- a/wks-controller.yaml +++ b/wks-controller.yaml @@ -1,4 +1,3 @@ ---- apiVersion: apps/v1 kind: Deployment metadata: @@ -42,7 +41,7 @@ spec: operator: Exists containers: - name: controller - image: quay.io/wksctl/controller:latest + image: docker.io/weaveworks/wksctl-controller:v0.8.3-rc.1 command: - /bin/controller - --verbose