Merge pull request #35 from Yolean/resource-limits

Apply resource limits based on tests with little load

Author: solsson

50kafka.yml

@@ -20,6 +20,13 @@ spec:
         image: solsson/kafka-prometheus-jmx-exporter@sha256:1f7c96c287a2dbec1d909cd8f96c0656310239b55a9a90d7fd12c81f384f1f7d
         ports:
         - containerPort: 5556
+        resources:
+          requests:
+            cpu: 0m
+            memory: 40Mi
+          limits:
+            cpu: 10m
+            memory: 80Mi
       - name: broker
         image: solsson/kafka:0.11.0.0-rc2@sha256:c1316e0131f4ec83bc645ca2141e4fda94e0d28f4fb5f836e15e37a5e054bdf1
         env:
@@ -36,6 +43,10 @@ spec:
           --override log.retention.hours=-1
           --override log.dirs=/var/lib/kafka/data/topics
           --override broker.id=${HOSTNAME##*-}
+        resources:
+          requests:
+            cpu: 100m
+            memory: 512Mi
         volumeMounts:
         - name: data
           mountPath: /var/lib/kafka/data

README.md

@@ -8,20 +8,6 @@ To get consistent service DNS names `kafka-N.broker.kafka`(`.svc.cluster.local`)
 kubectl create -f 00namespace.yml
 ```
 
-## Set up volume claims
-
-You may add [storage class](http://kubernetes.io/docs/user-guide/persistent-volumes/#storageclasses)
-to the kafka StatefulSet declaration to enable automatic volume provisioning.
-
-Alternatively create [PV](http://kubernetes.io/docs/user-guide/persistent-volumes/#persistent-volumes)s and [PVC](http://kubernetes.io/docs/user-guide/persistent-volumes/#persistentvolumeclaims)s manually. For example in Minikube.
-
-```
-./bootstrap/pv.sh
-kubectl create -f ./10pvc.yml
-# check that claims are bound
-kubectl -n kafka get pvc
-```
-
 ## Set up Zookeeper
 
 The Kafka book (Definitive Guide, O'Reilly 2016) recommends that Kafka has its own Zookeeper cluster with at least 5 instances.

test/monitoring-test.yml

@@ -0,0 +1,35 @@
+# Sets up a pod that monitors itself, to test resource usage etc.
+# kubectl exec test-metrics-... -- /bin/sh -c 'apk add --no-cache curl && curl http://localhost:5556/metrics'
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: monitoring-test
+  namespace: kafka
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: monitoring-test
+      # Uncomment to test with prometheus
+      #annotations:
+      #  prometheus.io/scrape: "true"
+      #  prometheus.io/port: "5556"
+    spec:
+      containers:
+      - name: monitor
+        image: solsson/kafka-prometheus-jmx-exporter@sha256:1f7c96c287a2dbec1d909cd8f96c0656310239b55a9a90d7fd12c81f384f1f7d
+        command:
+        - java
+        - -Dcom.sun.management.jmxremote.ssl=false
+        - -Dcom.sun.management.jmxremote.authenticate=false
+        - -Dcom.sun.management.jmxremote.port=5555
+        - -jar
+        - jmx_prometheus_httpserver.jar
+        - "5556"
+        - example_configs/httpserver_sample_config.yml
+        ports:
+        - name: jmx
+          containerPort: 5555
+        - name: slashmetrics
+          containerPort: 5556

zookeeper/50pzoo.yml

@@ -20,13 +20,20 @@ spec:
       - name: metrics
         image: solsson/kafka-prometheus-jmx-exporter@sha256:1f7c96c287a2dbec1d909cd8f96c0656310239b55a9a90d7fd12c81f384f1f7d
         command:
-        - "java"
-        - "-jar"
-        - "jmx_prometheus_httpserver.jar"
+        - java
+        - -jar
+        - jmx_prometheus_httpserver.jar
         - "5556"
         - example_configs/zookeeper.yaml
         ports:
         - containerPort: 5556
+        resources:
+          requests:
+            cpu: 0m
+            memory: 40Mi
+          limits:
+            cpu: 10m
+            memory: 80Mi
       - name: zookeeper
         image: solsson/kafka:0.11.0.0-rc2@sha256:c1316e0131f4ec83bc645ca2141e4fda94e0d28f4fb5f836e15e37a5e054bdf1
         env:
@@ -49,6 +56,10 @@ spec:
           name: peer
         - containerPort: 3888
           name: leader-election
+        resources:
+          requests:
+            cpu: 10m
+            memory: 100Mi
         volumeMounts:
         - name: config
           mountPath: /usr/local/kafka/config

zookeeper/51zoo.yml

@@ -20,13 +20,20 @@ spec:
       - name: metrics
         image: solsson/kafka-prometheus-jmx-exporter@sha256:1f7c96c287a2dbec1d909cd8f96c0656310239b55a9a90d7fd12c81f384f1f7d
         command:
-        - "java"
-        - "-jar"
-        - "jmx_prometheus_httpserver.jar"
+        - java
+        - -jar
+        - jmx_prometheus_httpserver.jar
         - "5556"
         - example_configs/zookeeper.yaml
         ports:
         - containerPort: 5556
+        resources:
+          requests:
+            cpu: 0m
+            memory: 40Mi
+          limits:
+            cpu: 10m
+            memory: 80Mi
       - name: zookeeper
         image: solsson/kafka:0.11.0.0-rc2@sha256:c1316e0131f4ec83bc645ca2141e4fda94e0d28f4fb5f836e15e37a5e054bdf1
         env:
@@ -49,6 +56,10 @@ spec:
           name: peer
         - containerPort: 3888
           name: leader-election
+        resources:
+          requests:
+            cpu: 10m
+            memory: 100Mi
         volumeMounts:
         - name: config
           mountPath: /usr/local/kafka/config