From e0948a4a2d9602b1701d53b32be01bf52195840f Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 27 Apr 2019 21:29:44 +0200 Subject: [PATCH 1/4] Creates a kubernets-kafka cluster locally using kubernetes-sigs/kind --- quickstart-kind.Dockerfile | 66 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 quickstart-kind.Dockerfile diff --git a/quickstart-kind.Dockerfile b/quickstart-kind.Dockerfile new file mode 100644 index 00000000..676ff333 --- /dev/null +++ b/quickstart-kind.Dockerfile @@ -0,0 +1,66 @@ +FROM busybox as entrypoint + +RUN printf '#!/bin/sh\ +# Brings up a kubernetes-kafka cluster in local docker. For example: \n\ +#docker run -v /var/run/docker.sock:/var/run/docker.sock:rw --net=host --name kafka-quickstart --rm -ti solsson/kubernetes-kafka:kind \n\ +\n\ +#set -e \n\ +kind create cluster --name=$KIND_NAME \n\ +echo KUBECONFIG=$KUBECONFIG \n\ +kubectl cluster-info \n\ +sleep 10 \n\ +kubectl -n kube-system get pods \n\ +kubectl wait --for=condition=Ready -n kube-system pod/etcd-$KIND_NAME-control-plane \n\ +\n\ +kubectl apply -f 00-namespace.yml \n\ +kubectl apply -f rbac-namespace-default/ \n\ +kubectl apply -k variants/scale-1-ephemeral/ \n\ +sleep 10 \n\ +kubectl -n kafka get pods \n\ +kubectl -n kafka wait --for=condition=Ready --timeout=120s pod/kafka-0 \n\ +kubectl -n kafka get pods \n\ +\n\ +echo "To get local kubectl access (with --net=host) run for example:" \n\ +echo "docker cp kafka-quickstart:/root/.kube/kind-config-$KIND_NAME ./kubeconfig-local-kubernetes-kafka" \n\ +echo "export KUBECONFIG=\$(pwd)/kubeconfig-local-kubernetes-kafka" \n\ +\n\ +echo "Then to get kafka access (with a hosts file entry 127.0.0.1 kafka-0.broker.kafka.svc.cluster.local):" \n\ +echo "kubectl -n kafka port-forward kafka-0 9092" \n\ +\n\ +echo "To delete the cluster simply rm this container and $KIND_NAME-control-plane" \n\ +\n\ +tail -f /dev/null \n\ +'\ +>> /entrypoint + +FROM docker:18.09.5-dind@sha256:7ed03cb37cbe109867455393e670016149ae80fde931cf4773fabd8cf6284ee6 \ + as docker + +FROM lachlanevenson/k8s-kubectl:v1.14.1@sha256:8267b932d262f0fc2c901f515d7752dd9e7c60c8410086e7a9fda935bcc3ba0d \ + as kubectl + +FROM golang:1.12.4-alpine \ + as kind + +ENV KIND_VERSION=0.2.1 +ENV GO111MODULE=on +RUN set -e; \ + apk add --no-cache git; \ + go get sigs.k8s.io/kind@${KIND_VERSION} + +FROM alpine:3.9@sha256:28ef97b8686a0b5399129e9b763d5b7e5ff03576aa5580d6f4182a49c5fe1913 \ + as runner + +COPY --from=docker /usr/local/bin/docker /usr/local/bin/docker +COPY --from=kubectl /usr/local/bin/kubectl /usr/local/bin/kubectl +COPY --from=kind /go/bin/kind /usr/local/bin/kind + +WORKDIR /kubernetes-kafka +COPY . . + +ENV KIND_NAME=kafka KUBECONFIG=/root/.kube/kind-config-kafka + +COPY --from=entrypoint /entrypoint /entrypoint +RUN chmod u+x /entrypoint + +ENTRYPOINT ["/entrypoint"] From 0820185b86d3ba2d7e7be4e8fecde779abc0cf01 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sun, 28 Apr 2019 06:38:12 +0200 Subject: [PATCH 2/4] kafkacat -b localhost:9092 -L --- quickstart-kind.Dockerfile | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/quickstart-kind.Dockerfile b/quickstart-kind.Dockerfile index 676ff333..17250023 100644 --- a/quickstart-kind.Dockerfile +++ b/quickstart-kind.Dockerfile @@ -2,15 +2,20 @@ FROM busybox as entrypoint RUN printf '#!/bin/sh\ # Brings up a kubernetes-kafka cluster in local docker. For example: \n\ -#docker run -v /var/run/docker.sock:/var/run/docker.sock:rw --net=host --name kafka-quickstart --rm -ti solsson/kubernetes-kafka:kind \n\ +#docker run -v /var/run/docker.sock:/var/run/docker.sock:rw --net=host --name kafka-quickstart -ti solsson/kubernetes-kafka:kind \n\ \n\ -#set -e \n\ kind create cluster --name=$KIND_NAME \n\ echo KUBECONFIG=$KUBECONFIG \n\ kubectl cluster-info \n\ +\n\ +echo "To get local kubectl access (with --net=host) run for example:" \n\ +echo "docker cp kafka-quickstart:/root/.kube/kind-config-$KIND_NAME ./kubeconfig-local-kubernetes-kafka" \n\ +echo "export KUBECONFIG=\$(pwd)/kubeconfig-local-kubernetes-kafka" \n\ +\n\ +echo "Will now attempt an automated kafka setup ..." \n\ sleep 10 \n\ kubectl -n kube-system get pods \n\ -kubectl wait --for=condition=Ready -n kube-system pod/etcd-$KIND_NAME-control-plane \n\ +kubectl wait --for=condition=Ready --timeout=120s -n kube-system pod --all \n\ \n\ kubectl apply -f 00-namespace.yml \n\ kubectl apply -f rbac-namespace-default/ \n\ @@ -18,18 +23,11 @@ kubectl apply -k variants/scale-1-ephemeral/ \n\ sleep 10 \n\ kubectl -n kafka get pods \n\ kubectl -n kafka wait --for=condition=Ready --timeout=120s pod/kafka-0 \n\ -kubectl -n kafka get pods \n\ \n\ -echo "To get local kubectl access (with --net=host) run for example:" \n\ -echo "docker cp kafka-quickstart:/root/.kube/kind-config-$KIND_NAME ./kubeconfig-local-kubernetes-kafka" \n\ -echo "export KUBECONFIG=\$(pwd)/kubeconfig-local-kubernetes-kafka" \n\ -\n\ -echo "Then to get kafka access (with a hosts file entry 127.0.0.1 kafka-0.broker.kafka.svc.cluster.local):" \n\ +echo "Local kafka access requires a hosts file entry 127.0.0.1 kafka-0.broker.kafka.svc.cluster.local" \n\ echo "kubectl -n kafka port-forward kafka-0 9092" \n\ \n\ echo "To delete the cluster simply rm this container and $KIND_NAME-control-plane" \n\ -\n\ -tail -f /dev/null \n\ '\ >> /entrypoint From 919305d9b61ae3b2f7be5d980c848d3ae3ccdbb2 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 30 Apr 2019 17:12:11 +0200 Subject: [PATCH 3/4] Scaling Kafka down isn't trivial --- variants/scale-1-ephemeral/kafka-scale1-overrides.json | 4 +++- variants/scale-1/kafka-scale1-overrides.json | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/variants/scale-1-ephemeral/kafka-scale1-overrides.json b/variants/scale-1-ephemeral/kafka-scale1-overrides.json index 14e41745..e6c0713a 100644 --- a/variants/scale-1-ephemeral/kafka-scale1-overrides.json +++ b/variants/scale-1-ephemeral/kafka-scale1-overrides.json @@ -2,5 +2,7 @@ {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "--override"}, {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "default.replication.factor=1"}, {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "--override"}, - {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "min.insync.replicas=1"} + {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "min.insync.replicas=1"}, + {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "--override"}, + {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "offsets.topic.replication.factor=1"} ] \ No newline at end of file diff --git a/variants/scale-1/kafka-scale1-overrides.json b/variants/scale-1/kafka-scale1-overrides.json index 14e41745..e6c0713a 100644 --- a/variants/scale-1/kafka-scale1-overrides.json +++ b/variants/scale-1/kafka-scale1-overrides.json @@ -2,5 +2,7 @@ {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "--override"}, {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "default.replication.factor=1"}, {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "--override"}, - {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "min.insync.replicas=1"} + {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "min.insync.replicas=1"}, + {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "--override"}, + {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "offsets.topic.replication.factor=1"} ] \ No newline at end of file From 3b1fc25031017d62429a422e7689ea4843e7d4cf Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 1 May 2019 11:48:00 +0200 Subject: [PATCH 4/4] No reason to have 50 partitions in single-broker, i.e. dev, stacks --- variants/scale-1-ephemeral/kafka-scale1-overrides.json | 4 +++- variants/scale-1/kafka-scale1-overrides.json | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/variants/scale-1-ephemeral/kafka-scale1-overrides.json b/variants/scale-1-ephemeral/kafka-scale1-overrides.json index e6c0713a..d9d03749 100644 --- a/variants/scale-1-ephemeral/kafka-scale1-overrides.json +++ b/variants/scale-1-ephemeral/kafka-scale1-overrides.json @@ -4,5 +4,7 @@ {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "--override"}, {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "min.insync.replicas=1"}, {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "--override"}, - {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "offsets.topic.replication.factor=1"} + {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "offsets.topic.replication.factor=1"}, + {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "--override"}, + {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "offsets.topic.num.partitions=1"} ] \ No newline at end of file diff --git a/variants/scale-1/kafka-scale1-overrides.json b/variants/scale-1/kafka-scale1-overrides.json index e6c0713a..d9d03749 100644 --- a/variants/scale-1/kafka-scale1-overrides.json +++ b/variants/scale-1/kafka-scale1-overrides.json @@ -4,5 +4,7 @@ {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "--override"}, {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "min.insync.replicas=1"}, {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "--override"}, - {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "offsets.topic.replication.factor=1"} + {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "offsets.topic.replication.factor=1"}, + {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "--override"}, + {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "offsets.topic.num.partitions=1"} ] \ No newline at end of file