Skip to content

Commit 467fbb9

Browse files
solssonsolsson
authored
Merge pull request #95 from Yolean/ops-jobs
Add Jobs and tests for common maintenance operations
2 parents 5a2b8c7 + 43c896b commit 467fbb9

File tree

5 files changed

+217
-0
lines changed

5 files changed

+217
-0
lines changed

‎maintenance/README.md

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
2+
## Re-assign Leadership
3+
4+
This is one of the cases where this repo begs to differ from traditional Kafka setups.
5+
In Kubernetes the restart of a pod, and subsequent start on a different node, should be a non-event.
6+
7+
> ”when a broker is stopped and restarted, it does not resume leadership of any partitions automatically”
8+
9+
_-- Neha Narkhede, Gwen Shapira, and Todd Palino. ”Kafka: The Definitive Guide”_
10+
11+
Create the `preferred-replica-election-job.yml` resource, after deleting any previous one.
12+
13+
## Change a Partition's Replicas
14+
15+
> ”From time to time, it may be necessary to change the replica assignments for a partition. Some examples of when this might be needed are:
16+
> * If a topic’s partitions are not balanced across the cluster, causing uneven load on brokers
17+
> * If a broker is taken offline and the partition is under-replicated
18+
> * If a new broker is added and needs to receive a share of the cluster load”
19+
20+
_-- Neha Narkhede, Gwen Shapira, and Todd Palino. ”Kafka: The Definitive Guide”_
21+
22+
Use the `reassign-paritions-job.yml`, after editing `TOPICS` and `BROKERS`.
23+
24+
## Increase a topic's replication factor
25+
26+
See https://github.com/Yolean/kubernetes-kafka/pull/140
27+
28+
Use the `replication-factor-increase-job.yml`, after editing `TOPICS` and `BROKERS`.
29+
30+
The affected topics may end up without a preferred replica. See above to fix that,
31+
or to affect only your selected topics use [Kafka Manager](https://github.com/Yolean/kubernetes-kafka/pull/83)'s topic screen,
32+
Generate Partition Assignments followed by Reassign Partitions.

‎maintenance/preferred-replica-election-job.yml

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
apiVersion: batch/v1
2+
kind: Job
3+
metadata:
4+
name: preferred-replica-election
5+
namespace: kafka
6+
spec:
7+
template:
8+
metadata:
9+
name: preferred-replica-election
10+
spec:
11+
containers:
12+
- name: kafka
13+
image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d
14+
command:
15+
- ./bin/kafka-preferred-replica-election.sh
16+
- --zookeeper
17+
- zookeeper:2181
18+
restartPolicy: Never
19+
backoffLimit: 3

‎maintenance/reassign-paritions-job.yml

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
apiVersion: batch/v1
2+
kind: Job
3+
metadata:
4+
name: reassign-partitions
5+
namespace: kafka
6+
spec:
7+
template:
8+
metadata:
9+
name: reassign-partitions
10+
spec:
11+
containers:
12+
- name: kafka
13+
image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d
14+
env:
15+
- name: ZOOKEEPER
16+
value: zookeeper.kafka:2181
17+
# the following must be edited per job
18+
- name: TOPICS
19+
value: test-produce-consume,test-kafkacat
20+
- name: BROKERS
21+
value: 0,2
22+
command:
23+
- /bin/bash
24+
- -ce
25+
- >
26+
echo '{"topics":[' > /tmp/reassign-topics.json;
27+
echo -n ' {"topic":"' >> /tmp/reassign-topics.json;
28+
echo -n $TOPICS | sed 's/,/"},\n {"topic":"/g' >> /tmp/reassign-topics.json;
29+
echo '"}' >> /tmp/reassign-topics.json;
30+
echo ']}' >> /tmp/reassign-topics.json;
31+
32+
echo "# reassign-topics.json";
33+
cat /tmp/reassign-topics.json;
34+
35+
./bin/kafka-reassign-partitions.sh
36+
--zookeeper=$ZOOKEEPER
37+
--generate
38+
--topics-to-move-json-file=/tmp/reassign-topics.json
39+
--broker-list=$BROKERS > /tmp/generated.txt;
40+
41+
tail -n 1 /tmp/generated.txt > /tmp/proposed-reassignment.json;
42+
43+
echo "# proposed-reassignment.json";
44+
cat /tmp/proposed-reassignment.json;
45+
46+
./bin/kafka-reassign-partitions.sh
47+
--zookeeper=$ZOOKEEPER
48+
--execute
49+
--reassignment-json-file=/tmp/proposed-reassignment.json;
50+
restartPolicy: Never
51+
backoffLimit: 3

‎maintenance/replication-factor-increase-job.yml

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
apiVersion: batch/v1
2+
kind: Job
3+
metadata:
4+
name: replication-factor-increase
5+
namespace: kafka
6+
spec:
7+
template:
8+
metadata:
9+
name: replication-factor-increase
10+
spec:
11+
containers:
12+
- name: kafka
13+
image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d
14+
env:
15+
- name: ZOOKEEPER
16+
value: zookeeper.kafka:2181
17+
# the following must be edited per job
18+
- name: TOPICS
19+
value: ""
20+
- name: BROKERS
21+
value: 0,1,2
22+
command:
23+
- /bin/bash
24+
- -ce
25+
- >
26+
if [ -z "$TOPICS" ]; then
27+
echo "Please set the TOPICS env (comma-separated) and re-create the job"
28+
tail -f /dev/null
29+
fi
30+
31+
echo '{"topics":[' > /tmp/reassign-topics.json;
32+
echo -n ' {"topic":"' >> /tmp/reassign-topics.json;
33+
echo -n $TOPICS | sed 's/,/"},\n {"topic":"/g' >> /tmp/reassign-topics.json;
34+
echo '"}' >> /tmp/reassign-topics.json;
35+
echo ']}' >> /tmp/reassign-topics.json;
36+
37+
echo "# reassign-topics.json";
38+
cat /tmp/reassign-topics.json;
39+
40+
./bin/kafka-reassign-partitions.sh
41+
--zookeeper=$ZOOKEEPER
42+
--generate
43+
--topics-to-move-json-file=/tmp/reassign-topics.json
44+
--broker-list=$BROKERS > /tmp/generated.txt;
45+
46+
tail -n 1 /tmp/generated.txt > /tmp/proposed-reassignment.json;
47+
48+
sleep 1;
49+
echo "# proposed-reassignment.json";
50+
cat /tmp/proposed-reassignment.json;
51+
52+
sed -i 's/"replicas":\[.\]/"replicas":[0,1,2]/g' /tmp/proposed-reassignment.json;
53+
sed -i 's/,"log_dirs":\["any"\]//g' /tmp/proposed-reassignment.json;
54+
echo "# proposed-reassignment.json modified to affect replication factor";
55+
cat /tmp/proposed-reassignment.json;
56+
57+
echo "# Triggering kafka-reassign-partitions.sh"
58+
./bin/kafka-reassign-partitions.sh
59+
--zookeeper=$ZOOKEEPER
60+
--execute
61+
--reassignment-json-file=/tmp/proposed-reassignment.json;
62+
63+
echo "# Reassignment exited. Upon success you may want to run preferred-replica-election."
64+
restartPolicy: Never
65+
backoffLimit: 3

‎maintenance/test/replicated-partitions.yml

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
apiVersion: apps/v1beta2
2+
kind: Deployment
3+
metadata:
4+
name: replicated-partitions
5+
namespace: test-kafka
6+
spec:
7+
replicas: 1
8+
selector:
9+
matchLabels:
10+
test-type: readiness
11+
test-target: under-replicated-partitions
12+
template:
13+
metadata:
14+
labels:
15+
test-type: readiness
16+
test-target: under-replicated-partitions
17+
spec:
18+
containers:
19+
- name: kafka
20+
image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d
21+
command:
22+
- /bin/bash
23+
- -ec
24+
- >
25+
touch /tmp/testlog;
26+
tail -f /tmp/testlog
27+
readinessProbe:
28+
exec:
29+
command:
30+
- /bin/bash
31+
- -c
32+
- >
33+
echo "### $(date -Ins -u) ###" >> /tmp/testlog
34+
&&
35+
[
36+
$(
37+
./bin/kafka-topics.sh
38+
--zookeeper zookeeper.kafka:2181
39+
--describe
40+
--under-replicated-partitions
41+
|
42+
tee -a /tmp/testlog
43+
|
44+
wc -l
45+
)
46+
-eq
47+
0
48+
]
49+
periodSeconds: 30
50+
timeoutSeconds: 29

0 commit comments

Comments
 (0)