Merge pull request #167 from Yolean/sidecar-for-dynamic-config

Switch from init container to sidecar, with Kafka 1.1.0

Author: solsson

README.md

@@ -28,6 +28,7 @@ If you begin to rely on this kafka setup we recommend you fork, for example to e
 
 | tag   | k8s ≥ | highlights |
 | ----- | ------ | ---------- |
+| 4.x  | 1.9+    | Kafka 1.1 dynamic config |
 | v3.2 | 1.9.4, 1.8.9, 1.7.14 | Required for read-only ConfigMaps [#162](https://github.com/Yolean/kubernetes-kafka/issues/162) [#163](https://github.com/Yolean/kubernetes-kafka/pull/163) [k8s #58720](https://github.com/kubernetes/kubernetes/pull/58720) |
 | v3.1  | 1.8    | The painstaking path to `min.insync.replicas`=2 |
 | v3.0  | 1.8    | [Outside access](#78), [modern manifests](#84), [bootstrap.kafka](#52) |

kafka/10broker-config.yml

@@ -7,24 +7,23 @@ data:
   init.sh: |-
     #!/bin/bash
     set -x
+    cp /etc/kafka-configmap/log4j.properties /etc/kafka/
 
     KAFKA_BROKER_ID=${HOSTNAME##*-}
-    cp -Lur /etc/kafka-configmap/* /etc/kafka/
-    sed -i "s/#init#broker.id=#init#/broker.id=$KAFKA_BROKER_ID/" /etc/kafka/server.properties
-
+    SEDS=("s/#init#broker.id=#init#/broker.id=$KAFKA_BROKER_ID/")
     LABELS="kafka-broker-id=$KAFKA_BROKER_ID"
     ANNOTATIONS=""
 
     hash kubectl 2>/dev/null || {
-      sed -i "s/#init#broker.rack=#init#/#init#broker.rack=# kubectl not found in path/" /etc/kafka/server.properties
+      SEDS+=("s/#init#broker.rack=#init#/#init#broker.rack=# kubectl not found in path/")
     } && {
       ZONE=$(kubectl get node "$NODE_NAME" -o=go-template='{{index .metadata.labels "failure-domain.beta.kubernetes.io/zone"}}')
       if [ $? -ne 0 ]; then
-        sed -i "s/#init#broker.rack=#init#/#init#broker.rack=# zone lookup failed, see -c init-config logs/" /etc/kafka/server.properties
+        SEDS+=("s/#init#broker.rack=#init#/#init#broker.rack=# zone lookup failed, see -c init-config logs/")
       elif [ "x$ZONE" == "x<no value>" ]; then
-        sed -i "s/#init#broker.rack=#init#/#init#broker.rack=# zone label not found for node $NODE_NAME/" /etc/kafka/server.properties
+        SEDS+=("s/#init#broker.rack=#init#/#init#broker.rack=# zone label not found for node $NODE_NAME/")
       else
-        sed -i "s/#init#broker.rack=#init#/broker.rack=$ZONE/" /etc/kafka/server.properties
+        SEDS+=("s/#init#broker.rack=#init#/broker.rack=$ZONE/")
         LABELS="$LABELS kafka-broker-rack=$ZONE"
       fi
 
@@ -33,7 +32,7 @@ data:
         echo "Outside (i.e. cluster-external access) host lookup command failed"
       else
         OUTSIDE_PORT=3240${KAFKA_BROKER_ID}
-        sed -i "s|#init#advertised.listeners=OUTSIDE://#init#|advertised.listeners=OUTSIDE://${OUTSIDE_HOST}:${OUTSIDE_PORT}|" /etc/kafka/server.properties
+        SEDS+=("s|#init#advertised.listeners=OUTSIDE://#init#|advertised.listeners=OUTSIDE://${OUTSIDE_HOST}:${OUTSIDE_PORT}|")
         ANNOTATIONS="$ANNOTATIONS kafka-listener-outside-host=$OUTSIDE_HOST kafka-listener-outside-port=$OUTSIDE_PORT"
       fi
 
@@ -44,6 +43,9 @@ data:
         kubectl -n $POD_NAMESPACE annotate pod $POD_NAME $ANNOTATIONS || echo "Failed to annotate $POD_NAMESPACE.$POD_NAME - RBAC issue?"
       fi
     }
+    printf '%s\n' "${SEDS[@]}" | sed -f - /etc/kafka-configmap/server.properties > /etc/kafka/server.properties.tmp
+    [ $? -eq 0 ] && mv /etc/kafka/server.properties.tmp /etc/kafka/server.properties
+    [ -f /etc/kafka/server.properties ] && echo "Ready for Kafka start" && tail -f /dev/null
 
   server.properties: |-
     ############################# Log Basics #############################

kafka/50kafka.yml

@@ -18,8 +18,8 @@ spec:
       annotations:
     spec:
       terminationGracePeriodSeconds: 30
-      initContainers:
-      - name: init-config
+      containers:
+      - name: config
         image: solsson/kafka-initutils@sha256:18bf01c2c756b550103a99b3c14f741acccea106072cd37155c6d24be4edd6e2
         env:
         - name: NODE_NAME
@@ -40,9 +40,8 @@ spec:
           mountPath: /etc/kafka-configmap
         - name: config
           mountPath: /etc/kafka
-      containers:
       - name: broker
-        image: solsson/kafka:1.0.1@sha256:1a4689d49d6274ac59b9b740f51b0408e1c90a9b66d16ad114ee9f7193bab111
+        image: solsson/kafka:1.1@sha256:ba863ca7dc28563930584e37f93d57c2cbf3f46b1c1fa104fe8af7bcc0c31df4
         env:
         - name: KAFKA_LOG4J_OPTS
           value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties