diff --git a/10broker-config.yml b/10broker-config.yml index af0f0374..7d296bf3 100644 --- a/10broker-config.yml +++ b/10broker-config.yml @@ -8,8 +8,21 @@ data: #!/bin/bash set -x - export KAFKA_BROKER_ID=${HOSTNAME##*-} - sed -i "s/\${KAFKA_BROKER_ID}/$KAFKA_BROKER_ID/" /etc/kafka/server.properties + KAFKA_BROKER_ID=${HOSTNAME##*-} + sed -i "s/#init#broker.id=#init#/broker.id=$KAFKA_BROKER_ID/" /etc/kafka/server.properties + + hash kubectl 2>/dev/null || { + sed -i "s/#init#broker.rack=#init#/#init#broker.rack=# kubectl not found in path/" /etc/kafka/server.properties + } && { + ZONE=$(kubectl get node "$NODE_NAME" -o=go-template='{{index .metadata.labels "failure-domain.beta.kubernetes.io/zone"}}') + if [ $? -ne 0 ]; then + sed -i "s/#init#broker.rack=#init#/#init#broker.rack=# zone lookup failed, see -c init-config logs/" /etc/kafka/server.properties + elif [ "x$ZONE" == "x" ]; then + sed -i "s/#init#broker.rack=#init#/#init#broker.rack=# zone label not found for node $NODE_NAME/" /etc/kafka/server.properties + else + sed -i "s/#init#broker.rack=#init#/broker.rack=$ZONE/" /etc/kafka/server.properties + fi + } server.properties: |- # Licensed to the Apache Software Foundation (ASF) under one or more @@ -32,10 +45,12 @@ data: ############################# Server Basics ############################# # The id of the broker. This must be set to a unique integer for each broker. - broker.id=${KAFKA_BROKER_ID} + #init#broker.id=#init# + + #init#broker.rack=#init# # Switch to enable topic deletion or not, default value is false - #delete.topic.enable=true + delete.topic.enable=true ############################# Socket Server Settings ############################# @@ -170,7 +185,7 @@ data: # Unspecified loggers and loggers with additivity=true output to server.log and stdout # Note that INFO only applies to unspecified loggers, the log level of the child logger is used otherwise - log4j.rootLogger=INFO, stdout, kafkaAppender + log4j.rootLogger=INFO, stdout log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.layout=org.apache.log4j.PatternLayout diff --git a/50kafka.yml b/50kafka.yml index 4404a6be..8f8e2837 100644 --- a/50kafka.yml +++ b/50kafka.yml @@ -15,7 +15,12 @@ spec: terminationGracePeriodSeconds: 30 initContainers: - name: init-config - image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce + image: solsson/kafka-initutils@sha256:c275d681019a0d8f01295dbd4a5bae3cfa945c8d0f7f685ae1f00f2579f08c7d + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName command: ['/bin/bash', '/etc/kafka/init.sh'] volumeMounts: - name: config @@ -43,7 +48,7 @@ spec: requests: cpu: 100m memory: 512Mi - livenessProbe: + readinessProbe: exec: command: - /bin/sh @@ -61,6 +66,8 @@ spec: volumeClaimTemplates: - metadata: name: data + annotations: + volume.beta.kubernetes.io/storage-class: kafka-broker spec: accessModes: [ "ReadWriteOnce" ] resources: diff --git a/README.md b/README.md index 9853d12e..83afdae4 100644 --- a/README.md +++ b/README.md @@ -11,11 +11,13 @@ How to use: * Kafka for real: fork and have a look at [addon](https://github.com/Yolean/kubernetes-kafka/labels/addon)s. * Join the discussion in issues and PRs. -Why? -See for yourself, but we think this project gives you better adaptability than [helm](https://github.com/kubernetes/helm) [chart](https://github.com/kubernetes/charts/tree/master/incubator/kafka)s. No single readable readme or template can properly introduce both Kafka and Kubernets. +No readable readme can properly introduce both [Kafka](http://kafka.apache.org/) and [Kubernets](https://kubernetes.io/), +but we think the combination of the two is a great backbone for microservices. Back when we read [Newman](http://samnewman.io/books/building_microservices/) we were beginners with both. Now we've read [Kleppmann](http://dataintensive.net/), [Confluent](https://www.confluent.io/blog/) and [SRE](https://landing.google.com/sre/book.html) and enjoy this "Streaming Platform" lock-in :smile:. +We also think the plain-yaml approach of this project is easier to understand and evolve than [helm](https://github.com/kubernetes/helm) [chart](https://github.com/kubernetes/charts/tree/master/incubator/kafka)s. + ## What you get Keep an eye on `kubectl --namespace kafka get pods -w`. @@ -25,6 +27,12 @@ The goal is to provide [Bootstrap servers](http://kafka.apache.org/documentation Zookeeper at `zookeeper.kafka.svc.cluster.local:2181`. +## Prepare storage classes + +For Minikube run `kubectl create -f configure-minikube/`. + +There's a similar setup for GKE, in `configure-gke` of course. You might want to tweak it before creating. + ## Start Zookeeper The [Kafka book](https://www.confluent.io/resources/kafka-definitive-guide-preview-edition/) recommends that Kafka has its own Zookeeper cluster with at least 5 instances. @@ -50,9 +58,24 @@ kubectl -n kafka logs kafka-0 | grep "Registered broker" That's it. Just add business value :wink:. For clients we tend to use [librdkafka](https://github.com/edenhill/librdkafka)-based drivers like [node-rdkafka](https://github.com/Blizzard/node-rdkafka). To use [Kafka Connect](http://kafka.apache.org/documentation/#connect) and [Kafka Streams](http://kafka.apache.org/documentation/streams/) you may want to take a look at our [sample](https://github.com/solsson/dockerfiles/tree/master/connect-files) [Dockerfile](https://github.com/solsson/dockerfiles/tree/master/streams-logfilter)s. -Don't forget the [addon](https://github.com/Yolean/kubernetes-kafka/labels/addon)s. +And don't forget the [addon](https://github.com/Yolean/kubernetes-kafka/labels/addon)s. + +## RBAC + +For clusters that enfoce [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/) there's a minimal set of policies in +``` +kubectl apply -f rbac-namespace-default/ +``` + +## Caution: `Delete` Reclaim Policy is default + +In production you likely want to [manually set Reclaim Policy](https://kubernetes.io/docs/tasks/administer-cluster/change-pv-reclaim-policy/), +our your data will be gone if the generated [volume claim](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims)s are deleted. + +This can't be done [in manifests](https://github.com/Yolean/kubernetes-kafka/pull/50), +at least not [until Kubernetes 1.8](https://github.com/kubernetes/features/issues/352). -# Tests +## Tests ``` kubectl apply -f test/ diff --git a/configure-gke/storageclass-broker-gke.yml b/configure-gke/storageclass-broker-gke.yml new file mode 100644 index 00000000..d4361485 --- /dev/null +++ b/configure-gke/storageclass-broker-gke.yml @@ -0,0 +1,7 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: kafka-broker +provisioner: kubernetes.io/gce-pd +parameters: + type: pd-standard diff --git a/configure-gke/storageclass-zookeeper-gke.yml b/configure-gke/storageclass-zookeeper-gke.yml new file mode 100644 index 00000000..44891bac --- /dev/null +++ b/configure-gke/storageclass-zookeeper-gke.yml @@ -0,0 +1,7 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: kafka-zookeeper +provisioner: kubernetes.io/gce-pd +parameters: + type: pd-ssd diff --git a/configure-minikube/storageclass-broker-minikube.yml b/configure-minikube/storageclass-broker-minikube.yml new file mode 100644 index 00000000..3cff3b21 --- /dev/null +++ b/configure-minikube/storageclass-broker-minikube.yml @@ -0,0 +1,5 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: kafka-broker +provisioner: k8s.io/minikube-hostpath diff --git a/configure-minikube/storageclass-zookeeper-minikube.yml b/configure-minikube/storageclass-zookeeper-minikube.yml new file mode 100644 index 00000000..ba89eb46 --- /dev/null +++ b/configure-minikube/storageclass-zookeeper-minikube.yml @@ -0,0 +1,5 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: kafka-zookeeper +provisioner: k8s.io/minikube-hostpath diff --git a/prod-yolean.sh b/prod-yolean.sh deleted file mode 100755 index fb48139e..00000000 --- a/prod-yolean.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -# Combines addons into what we 'kubectl apply -f' to production -set -ex - -git fetch -git checkout origin/kafka-011 -git checkout -b prod-yolean-$(date +"%Y%m%dT%H%M%S") - -for BRANCH in \ - addon-storage-classes \ - addon-metrics \ - addon-rest \ - addon-kube-events-topic -do - git merge --no-ff $BRANCH -m "prod-yolean merge $BRANCH" -done diff --git a/qa-yolean.sh b/qa-yolean.sh new file mode 100755 index 00000000..dcdd6f21 --- /dev/null +++ b/qa-yolean.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Combines addons into what we 'kubectl apply -f' to production +set -ex + +ANNOTATION_PREFIX='yolean.se/kubernetes-kafka-' +BUILD=$(basename $0) +REMOTE=origin +FROM="$REMOTE/" +START=master + +[ ! -z "$(git status --untracked-files=no -s)" ] && echo "Working copy must be clean" && exit 1 + +function annotate { + key=$1 + value=$2 + file=$3 + case $(uname) in + Darwin*) + sed -i '' 's| annotations:| annotations:\ + --next-annotation--|' $file + sed -i '' "s|--next-annotation--|${ANNOTATION_PREFIX}$key: '$value'|" $file + ;; + *) + sed -i "s| annotations:| annotations:\n ${ANNOTATION_PREFIX}$key: '$value'|" $file + ;; + esac +} + +git checkout ${FROM}$START +REVS="$START:$(git rev-parse --short ${FROM}$START)" + +git checkout -b qa-yolean-$(date +"%Y%m%dT%H%M%S") + +for BRANCH in \ + addon-storage-classes \ + rolling-update \ + addon-rest \ + addon-metrics \ + addon-kube-events-topic +do + git merge --no-ff ${FROM}$BRANCH -m "qa-yolean merge ${FROM}$BRANCH" && \ + REVS="$REVS $BRANCH:$(git rev-parse --short ${FROM}$BRANCH)" +done + +END_BRANCH_GIT=$(git rev-parse --abbrev-ref HEAD) + +for F in ./50kafka.yml ./zookeeper/50pzoo.yml ./zookeeper/51zoo.yml +do + annotate revs "$REVS" $F + annotate build "$END_BRANCH_GIT" $F +done diff --git a/rbac-namespace-default/node-reader.yml b/rbac-namespace-default/node-reader.yml new file mode 100644 index 00000000..edf3dde1 --- /dev/null +++ b/rbac-namespace-default/node-reader.yml @@ -0,0 +1,37 @@ +# To see if init containers need RBAC: +# +# $ kubectl exec kafka-0 -- cat /etc/kafka/server.properties | grep broker.rack +# #init#broker.rack=# zone lookup failed, see -c init-config logs +# $ kubectl logs -c init-config kafka-0 +# ++ kubectl get node some-node '-o=go-template={{index .metadata.labels "failure-domain.beta.kubernetes.io/zone"}}' +# Error from server (Forbidden): User "system:serviceaccount:kafka:default" cannot get nodes at the cluster scope.: "Unknown user \"system:serviceaccount:kafka:default\"" +# +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: node-reader + labels: + origin: github.com_Yolean_kubernetes-kafka +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: kafka-node-reader + labels: + origin: github.com_Yolean_kubernetes-kafka +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: node-reader +subjects: +- kind: ServiceAccount + name: default + namespace: kafka diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index f9d5c587..566335ba 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -43,12 +43,6 @@ spec: requests: cpu: 10m memory: 100Mi - livenessProbe: - exec: - command: - - /bin/sh - - -c - - '[ "imok" = "$(echo ruok | nc -w 1 127.0.0.1 2181)" ]' readinessProbe: exec: command: @@ -67,8 +61,10 @@ spec: volumeClaimTemplates: - metadata: name: data + annotations: + volume.beta.kubernetes.io/storage-class: kafka-zookeeper spec: accessModes: [ "ReadWriteOnce" ] resources: requests: - storage: 10Gi + storage: 1Gi diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index 778567db..f5d1f91e 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -46,12 +46,6 @@ spec: requests: cpu: 10m memory: 100Mi - livenessProbe: - exec: - command: - - /bin/sh - - -c - - '[ "imok" = "$(echo ruok | nc -w 1 127.0.0.1 2181)" ]' readinessProbe: exec: command: