Merge pull request #107 from Yolean/yolean-topic-management

Embrace auto.topic.create; trust defaults for production topics

Author: solsson

README.md

@@ -68,6 +68,8 @@ For clusters that enfoce [RBAC](https://kubernetes.io/docs/admin/authorization/r
 kubectl apply -f rbac-namespace-default/
 ```
 
+For example rack awareness can fail without this, `logs -c init-config` showing `Error from server (Forbidden): pods "kafka-0" is forbidden: User "system:serviceaccount:kafka:default" cannot get pods in the namespace "kafka": Unknown user "system:serviceaccount:kafka:default"`.
+
 ## Tests
 
 Tests are based on the [kube-test](https://github.com/Yolean/kube-test) concept.

kafka/10broker-config.yml

@@ -23,6 +23,7 @@ data:
         sed -i "s/#init#broker.rack=#init#/broker.rack=$ZONE/" /etc/kafka/server.properties
       fi
 
+      # This requires additional RBAC, and won't be needed after https://github.com/kubernetes/kubernetes/pull/55329
       kubectl -n $POD_NAMESPACE label pod $POD_NAME kafka-broker-id=$KAFKA_BROKER_ID
 
       OUTSIDE_HOST=$(kubectl get node "$NODE_NAME" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}')
@@ -35,22 +36,26 @@ data:
     }
 
   server.properties: |-
-    # Licensed to the Apache Software Foundation (ASF) under one or more
-    # contributor license agreements.  See the NOTICE file distributed with
-    # this work for additional information regarding copyright ownership.
-    # The ASF licenses this file to You under the Apache License, Version 2.0
-    # (the "License"); you may not use this file except in compliance with
-    # the License.  You may obtain a copy of the License at
-    #
-    #    http://www.apache.org/licenses/LICENSE-2.0
-    #
-    # Unless required by applicable law or agreed to in writing, software
-    # distributed under the License is distributed on an "AS IS" BASIS,
-    # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    # See the License for the specific language governing permissions and
-    # limitations under the License.
-
-    # see kafka.server.KafkaConfig for additional details and defaults
+    ############################# Log Basics #############################
+
+    # A comma seperated list of directories under which to store log files
+    # Overrides log.dir
+    log.dirs=/var/lib/kafka/data/topics
+
+    # The default number of log partitions per topic. More partitions allow greater
+    # parallelism for consumption, but this will also result in more files across
+    # the brokers.
+    num.partitions=1
+
+    default.replication.factor=3
+
+    min.insync.replicas=2
+
+    auto.create.topics.enable=true
+
+    # The number of threads per data directory to be used for log recovery at startup and flushing at shutdown.
+    # This value is recommended to be increased for installations with data dirs located in RAID array.
+    num.recovery.threads.per.data.dir=1
 
     ############################# Server Basics #############################
 
@@ -96,21 +101,6 @@ data:
     # The maximum size of a request that the socket server will accept (protection against OOM)
     socket.request.max.bytes=104857600
 
-
-    ############################# Log Basics #############################
-
-    # A comma seperated list of directories under which to store log files
-    log.dirs=/tmp/kafka-logs
-
-    # The default number of log partitions per topic. More partitions allow greater
-    # parallelism for consumption, but this will also result in more files across
-    # the brokers.
-    num.partitions=1
-
-    # The number of threads per data directory to be used for log recovery at startup and flushing at shutdown.
-    # This value is recommended to be increased for installations with data dirs located in RAID array.
-    num.recovery.threads.per.data.dir=1
-
     ############################# Internal Topic Settings  #############################
     # The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state"
     # For anything other than development testing, a value greater than 1 is recommended for to ensure availability such as 3.
@@ -143,7 +133,7 @@ data:
     # from the end of the log.
 
     # The minimum age of a log file to be eligible for deletion due to age
-    log.retention.hours=168
+    log.retention.hours=-1
 
     # A size-based retention policy for logs. Segments are pruned from the log as long as the remaining
     # segments don't drop below log.retention.bytes. Functions independently of log.retention.hours.
@@ -163,7 +153,7 @@ data:
     # server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
     # You can also append an optional chroot string to the urls to specify the
     # root directory for all kafka znodes.
-    zookeeper.connect=localhost:2181
+    zookeeper.connect=zookeeper:2181
 
     # Timeout in ms for connecting to zookeeper
     zookeeper.connection.timeout.ms=6000
@@ -179,21 +169,6 @@ data:
     group.initial.rebalance.delay.ms=0
 
   log4j.properties: |-
-    # Licensed to the Apache Software Foundation (ASF) under one or more
-    # contributor license agreements.  See the NOTICE file distributed with
-    # this work for additional information regarding copyright ownership.
-    # The ASF licenses this file to You under the Apache License, Version 2.0
-    # (the "License"); you may not use this file except in compliance with
-    # the License.  You may obtain a copy of the License at
-    #
-    #    http://www.apache.org/licenses/LICENSE-2.0
-    #
-    # Unless required by applicable law or agreed to in writing, software
-    # distributed under the License is distributed on an "AS IS" BASIS,
-    # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    # See the License for the specific language governing permissions and
-    # limitations under the License.
-
     # Unspecified loggers and loggers with additivity=true output to server.log and stdout
     # Note that INFO only applies to unspecified loggers, the log level of the child logger is used otherwise
     log4j.rootLogger=INFO, stdout

kafka/50kafka.yml

@@ -57,14 +57,6 @@ spec:
         command:
         - ./bin/kafka-server-start.sh
         - /etc/kafka/server.properties
-        - --override
-        -   zookeeper.connect=zookeeper:2181
-        - --override
-        -   log.retention.hours=-1
-        - --override
-        -   log.dirs=/var/lib/kafka/data/topics
-        - --override
-        -   auto.create.topics.enable=false
         resources:
           requests:
             cpu: 100m