Merge branch '4.x', Kafka 1.0.1 + new init script sed concept

Author: solsson

kafka/00namespace.yml renamed to 00-namespace.yml

@@ -13,6 +13,7 @@ To quote [@arthurk](https://github.com/Yolean/kubernetes-kafka/issues/82#issueco
 
 We suggest you `apply -f` manifests in the following order:
  * Your choice of storage classes from [./configure](./configure/)
+ * [namespace](./00-namespace.yml)
  * [./rbac-namespace-default](./rbac-namespace-default/)
  * [./zookeeper](./zookeeper/)
  * [./kafka](./kafka/)
@@ -29,6 +30,7 @@ If you begin to rely on this kafka setup we recommend you fork, for example to e
 | tag   | k8s ≥ | highlights |
 | ----- | ------ | ---------- |
 | 4.x  | 1.9+    | Kafka 1.1 dynamic config |
+| v4.1 | 1.9+    | Kafka 1.0.1 new (default)[#148] (config)[#170] |
 | v3.2 | 1.9.4, 1.8.9, 1.7.14 | Required for read-only ConfigMaps [#162](https://github.com/Yolean/kubernetes-kafka/issues/162) [#163](https://github.com/Yolean/kubernetes-kafka/pull/163) [k8s #58720](https://github.com/kubernetes/kubernetes/pull/58720) |
 | v3.1  | 1.8    | The painstaking path to `min.insync.replicas`=2 |
 | v3.0  | 1.8    | [Outside access](#78), [modern manifests](#84), [bootstrap.kafka](#52) |

kafka/test/00namespace.yml renamed to 01-test-namespace.yml

@@ -7,24 +7,23 @@ data:
   init.sh: |-
     #!/bin/bash
     set -x
+    cp /etc/kafka-configmap/log4j.properties /etc/kafka/
 
     KAFKA_BROKER_ID=${HOSTNAME##*-}
-    cp -Lur /etc/kafka-configmap/* /etc/kafka/
-    sed -i "s/#init#broker.id=#init#/broker.id=$KAFKA_BROKER_ID/" /etc/kafka/server.properties
-
+    SEDS=("s/#init#broker.id=#init#/broker.id=$KAFKA_BROKER_ID/")
     LABELS="kafka-broker-id=$KAFKA_BROKER_ID"
     ANNOTATIONS=""
 
     hash kubectl 2>/dev/null || {
-      sed -i "s/#init#broker.rack=#init#/#init#broker.rack=# kubectl not found in path/" /etc/kafka/server.properties
+      SEDS+=("s/#init#broker.rack=#init#/#init#broker.rack=# kubectl not found in path/")
     } && {
       ZONE=$(kubectl get node "$NODE_NAME" -o=go-template='{{index .metadata.labels "failure-domain.beta.kubernetes.io/zone"}}')
       if [ $? -ne 0 ]; then
-        sed -i "s/#init#broker.rack=#init#/#init#broker.rack=# zone lookup failed, see -c init-config logs/" /etc/kafka/server.properties
+        SEDS+=("s/#init#broker.rack=#init#/#init#broker.rack=# zone lookup failed, see -c init-config logs/")
       elif [ "x$ZONE" == "x<no value>" ]; then
-        sed -i "s/#init#broker.rack=#init#/#init#broker.rack=# zone label not found for node $NODE_NAME/" /etc/kafka/server.properties
+        SEDS+=("s/#init#broker.rack=#init#/#init#broker.rack=# zone label not found for node $NODE_NAME/")
       else
-        sed -i "s/#init#broker.rack=#init#/broker.rack=$ZONE/" /etc/kafka/server.properties
+        SEDS+=("s/#init#broker.rack=#init#/broker.rack=$ZONE/")
         LABELS="$LABELS kafka-broker-rack=$ZONE"
       fi
 
@@ -33,7 +32,7 @@ data:
         echo "Outside (i.e. cluster-external access) host lookup command failed"
       else
         OUTSIDE_PORT=3240${KAFKA_BROKER_ID}
-        sed -i "s|#init#advertised.listeners=OUTSIDE://#init#|advertised.listeners=OUTSIDE://${OUTSIDE_HOST}:${OUTSIDE_PORT}|" /etc/kafka/server.properties
+        SEDS+=("s|#init#advertised.listeners=OUTSIDE://#init#|advertised.listeners=OUTSIDE://${OUTSIDE_HOST}:${OUTSIDE_PORT}|")
         ANNOTATIONS="$ANNOTATIONS kafka-listener-outside-host=$OUTSIDE_HOST kafka-listener-outside-port=$OUTSIDE_PORT"
       fi
 
@@ -44,6 +43,8 @@ data:
         kubectl -n $POD_NAMESPACE annotate pod $POD_NAME $ANNOTATIONS || echo "Failed to annotate $POD_NAMESPACE.$POD_NAME - RBAC issue?"
       fi
     }
+    printf '%s\n' "${SEDS[@]}" | sed -f - /etc/kafka-configmap/server.properties > /etc/kafka/server.properties.tmp
+    [ $? -eq 0 ] && mv /etc/kafka/server.properties.tmp /etc/kafka/server.properties
 
   server.properties: |-
     ############################# Log Basics #############################
@@ -65,7 +66,7 @@ data:
 
     # The number of threads per data directory to be used for log recovery at startup and flushing at shutdown.
     # This value is recommended to be increased for installations with data dirs located in RAID array.
-    num.recovery.threads.per.data.dir=1
+    #num.recovery.threads.per.data.dir=1
 
     ############################# Server Basics #############################
 
@@ -76,7 +77,7 @@ data:
 
     ############################# Socket Server Settings #############################
 
-    # The address the socket server listens on. It will get the value returned from
+    # The address the socket server listens on. It will get the value returned from 
     # java.net.InetAddress.getCanonicalHostName() if not configured.
     #   FORMAT:
     #     listeners = listener_name://host_name:port
@@ -85,7 +86,7 @@ data:
     #listeners=PLAINTEXT://:9092
     listeners=OUTSIDE://:9094,PLAINTEXT://:9092
 
-    # Hostname and port the broker will advertise to producers and consumers. If not set,
+    # Hostname and port the broker will advertise to producers and consumers. If not set, 
     # it uses the value for "listeners" if configured.  Otherwise, it will use the value
     # returned from java.net.InetAddress.getCanonicalHostName().
     #advertised.listeners=PLAINTEXT://your.host.name:9092
@@ -97,19 +98,19 @@ data:
     inter.broker.listener.name=PLAINTEXT
 
     # The number of threads that the server uses for receiving requests from the network and sending responses to the network
-    num.network.threads=3
+    #num.network.threads=3
 
     # The number of threads that the server uses for processing requests, which may include disk I/O
-    num.io.threads=8
+    #num.io.threads=8
 
     # The send buffer (SO_SNDBUF) used by the socket server
-    socket.send.buffer.bytes=102400
+    #socket.send.buffer.bytes=102400
 
     # The receive buffer (SO_RCVBUF) used by the socket server
-    socket.receive.buffer.bytes=102400
+    #socket.receive.buffer.bytes=102400
 
     # The maximum size of a request that the socket server will accept (protection against OOM)
-    socket.request.max.bytes=104857600
+    #socket.request.max.bytes=104857600
 
     ############################# Internal Topic Settings  #############################
     # The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state"
@@ -125,7 +126,7 @@ data:
     # There are a few important trade-offs here:
     #    1. Durability: Unflushed data may be lost if you are not using replication.
     #    2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush.
-    #    3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to exceessive seeks.
+    #    3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks.
     # The settings below allow one to configure the flush policy to flush data after a period of time or
     # every N messages (or both). This can be done globally and overridden on a per-topic basis.
 
@@ -142,19 +143,22 @@ data:
     # A segment will be deleted whenever *either* of these criteria are met. Deletion always happens
     # from the end of the log.
 
+    # https://cwiki.apache.org/confluence/display/KAFKA/KIP-186%3A+Increase+offsets+retention+default+to+7+days
+    offsets.retention.minutes=10080
+
     # The minimum age of a log file to be eligible for deletion due to age
     log.retention.hours=-1
 
-    # A size-based retention policy for logs. Segments are pruned from the log as long as the remaining
-    # segments don't drop below log.retention.bytes. Functions independently of log.retention.hours.
+    # A size-based retention policy for logs. Segments are pruned from the log unless the remaining
+    # segments drop below log.retention.bytes. Functions independently of log.retention.hours.
     #log.retention.bytes=1073741824
 
     # The maximum size of a log segment file. When this size is reached a new log segment will be created.
-    log.segment.bytes=1073741824
+    #log.segment.bytes=1073741824
 
     # The interval at which log segments are checked to see if they can be deleted according
     # to the retention policies
-    log.retention.check.interval.ms=300000
+    #log.retention.check.interval.ms=300000
 
     ############################# Zookeeper #############################
 
@@ -166,7 +170,7 @@ data:
     zookeeper.connect=zookeeper:2181
 
     # Timeout in ms for connecting to zookeeper
-    zookeeper.connection.timeout.ms=6000
+    #zookeeper.connection.timeout.ms=6000
 
 
     ############################# Group Coordinator Settings #############################
@@ -176,7 +180,7 @@ data:
     # The default value for this is 3 seconds.
     # We override this to 0 here as it makes for a better out-of-the-box experience for development and testing.
     # However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup.
-    group.initial.rebalance.delay.ms=0
+    #group.initial.rebalance.delay.ms=0
 
   log4j.properties: |-
     # Unspecified loggers and loggers with additivity=true output to server.log and stdout

README.md

@@ -1,4 +1,4 @@
-apiVersion: apps/v1beta2
+apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: kafka

kafka/10broker-config.yml

@@ -83,6 +83,8 @@ spec:
         -   test-kafkacat
         - --partitions
         -   "1"
+        - --replication-factor
+        -   "3"
       restartPolicy: Never
 ---
 apiVersion: apps/v1beta2

kafka/50kafka.yml

@@ -66,6 +66,8 @@ spec:
         -   test-produce-consume
         - --partitions
         -   "1"
+        - --replication-factor
+        -   "3"
       restartPolicy: Never
 ---
 apiVersion: apps/v1beta2

kafka/test/kafkacat.yml

@@ -36,7 +36,7 @@ data:
       exit 10
     fi
 
-    echo "$(date --iso-8601='ns') $WITH_ONE partitions have one replica and WITH_TWO_OR_MORE have more"
+    echo "$(date --iso-8601='ns') $WITH_ONE partitions have one replica and $WITH_TWO_OR_MORE have more"
     exit 0
 
   quit-on-nonzero-exit.sh: |-

kafka/test/produce-consume.yml

@@ -1,4 +1,4 @@
-apiVersion: apps/v1beta2
+apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: pzoo

kafka/test/replication-config.yml

@@ -1,4 +1,4 @@
-apiVersion: apps/v1beta2
+apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   name: zoo