Skip to content

Commit f0bbfe0

Browse files
msvticketprashanth-volvocarsPrashanth R
authored
Autoscaling selenium grid on kubernetes with scaledjobs (#1854)
* Autoscaling selenium grid on kubernetes Autoscale selenium browser nodes running in kubernetes based on the request pending in session queue using KEDA. Toggle autoscaling on/off using 'autoscalingEnabled' option in helm charts. * Added all suggested changes * feat: add support for KEDA ScaledJobs and make them the default KEDA scaling type install keda automatically set SE_NODE_GRID_URL and DRAIN_AFTER_SESSION_COUNT automatically Set graphqlurl automatically refactor out pod templates to named template conditionally adding preStop hook for deregistering node Signed-off-by: Mårten Svantesson <[email protected]> * fix: work around for limitations in helm helm seem to normally install subchart after current chart so trying this work around * fix: format * fix: extracting default node env variables to configmap --------- Signed-off-by: Mårten Svantesson <[email protected]> Co-authored-by: Prashanth R <[email protected]> Co-authored-by: Prashanth R <[email protected]>
1 parent 33bd1ab commit f0bbfe0

14 files changed

+587
-417
lines changed

‎charts/selenium-grid/Chart.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,8 @@ type: application
55
version: 0.18.1
66
appVersion: 4.10.0-20230607
77
icon: https://github.com/SeleniumHQ/docker-selenium/raw/trunk/logo.png
8+
dependencies:
9+
- repository: https://kedacore.github.io/charts
10+
version: 2.10.2
11+
name: keda
12+
condition: autoscaling.enabled

‎charts/selenium-grid/README.md

Lines changed: 186 additions & 150 deletions
Large diffs are not rendered by default.

‎charts/selenium-grid/templates/_helpers.tpl

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,3 +79,130 @@ Ingress fullname
7979
{{- define "seleniumGrid.ingress.fullname" -}}
8080
{{- default "selenium-ingress" .Values.ingress.nameOverride | trunc 63 | trimSuffix "-" -}}
8181
{{- end -}}
82+
83+
{{/*
84+
Is autoscaling using KEDA enabled
85+
*/}}
86+
{{- define "seleniumGrid.useKEDA" -}}
87+
{{- or .Values.autoscaling.enabled .Values.autoscaling.enableWithExistingKEDA | ternary "true" "" -}}
88+
{{- end -}}
89+
90+
91+
{{/*
92+
Common pod template
93+
*/}}
94+
{{- define "seleniumGrid.podTemplate" -}}
95+
template:
96+
metadata:
97+
labels:
98+
app: {{.name}}
99+
app.kubernetes.io/name: {{.name}}
100+
{{- include "seleniumGrid.commonLabels" . | nindent 6 }}
101+
{{- with .node.labels }}
102+
{{- toYaml . | nindent 6 }}
103+
{{- end }}
104+
{{- with .Values.customLabels }}
105+
{{- toYaml . | nindent 6 }}
106+
{{- end }}
107+
annotations:
108+
checksum/event-bus-configmap: {{ include (print $.Template.BasePath "/event-bus-configmap.yaml") . | sha256sum }}
109+
{{- with .node.annotations }}
110+
{{ toYaml . | nindent 6 }}
111+
{{- end }}
112+
spec:
113+
restartPolicy: {{ and (eq (include "seleniumGrid.useKEDA" .) "true") (eq .Values.autoscaling.scalingType "job") | ternary "Never" "Always" }}
114+
{{- with .node.hostAliases }}
115+
hostAliases: {{ toYaml . | nindent 6 }}
116+
{{- end }}
117+
containers:
118+
- name: {{.name}}
119+
{{- $imageTag := default .Values.global.seleniumGrid.nodesImageTag .node.imageTag }}
120+
image: {{ printf "%s:%s" .node.imageName $imageTag }}
121+
imagePullPolicy: {{ .node.imagePullPolicy }}
122+
{{- with .node.extraEnvironmentVariables }}
123+
env: {{- tpl (toYaml .) $ | nindent 10 }}
124+
{{- end }}
125+
envFrom:
126+
- configMapRef:
127+
name: {{ .Values.busConfigMap.name }}
128+
- configMapRef:
129+
name: {{ .Values.nodeConfigMap.name }}
130+
{{- with .node.extraEnvFrom }}
131+
{{- toYaml . | nindent 10 }}
132+
{{- end }}
133+
{{- if gt (len .node.ports) 0 }}
134+
ports:
135+
{{- range .node.ports }}
136+
- containerPort: {{ . }}
137+
protocol: TCP
138+
{{- end }}
139+
{{- end }}
140+
volumeMounts:
141+
- name: dshm
142+
mountPath: /dev/shm
143+
{{- if .node.extraVolumeMounts }}
144+
{{- toYaml .node.extraVolumeMounts | nindent 10 }}
145+
{{- end }}
146+
{{- with .node.resources }}
147+
resources: {{- toYaml . | nindent 10 }}
148+
{{- end }}
149+
{{- include "seleniumGrid.lifecycle" . | nindent 8 -}}
150+
{{- with .node.startupProbe }}
151+
startupProbe: {{- toYaml . | nindent 10 }}
152+
{{- end }}
153+
{{- if or .Values.global.seleniumGrid.imagePullSecret .node.imagePullSecret }}
154+
imagePullSecrets:
155+
- name: {{ default .Values.global.seleniumGrid.imagePullSecret .node.imagePullSecret }}
156+
{{- end }}
157+
{{- with .node.nodeSelector }}
158+
nodeSelector: {{- toYaml . | nindent 6 }}
159+
{{- end }}
160+
{{- with .node.tolerations }}
161+
tolerations:
162+
{{ toYaml . | nindent 4 }}
163+
{{- end }}
164+
{{- with .node.priorityClassName }}
165+
priorityClassName: {{ . }}
166+
{{- end }}
167+
terminationGracePeriodSeconds: {{ .node.terminationGracePeriodSeconds }}
168+
volumes:
169+
- name: dshm
170+
emptyDir:
171+
medium: Memory
172+
sizeLimit: {{ default "1Gi" .node.dshmVolumeSizeLimit }}
173+
{{- if .node.extraVolumes }}
174+
{{ toYaml .node.extraVolumes | nindent 6 }}
175+
{{- end }}
176+
{{- end -}}
177+
178+
{{/*
179+
Get the url of the grid. If the external url can be figured out from the ingress use that, otherwise the cluster internal url
180+
*/}}
181+
{{- define "seleniumGrid.url" -}}
182+
{{- if and .Values.ingress.enabled .Values.ingress.hostname (ne .Values.ingress.hostname "selenium-grid.local") -}}
183+
http{{if .Values.ingress.tls}}s{{end}}://{{.Values.ingress.hostname}}
184+
{{- else -}}
185+
http://{{ include ($.Values.isolateComponents | ternary "seleniumGrid.router.fullname" "seleniumGrid.hub.fullname") $ }}.{{ .Release.Namespace }}:{{ $.Values.components.router.port }}
186+
{{- end }}
187+
{{- end -}}
188+
189+
{{/*
190+
Graphql Url of the hub or the router
191+
*/}}
192+
{{- define "seleniumGrid.graphqlURL" -}}
193+
http://{{ include ($.Values.isolateComponents | ternary "seleniumGrid.router.fullname" "seleniumGrid.hub.fullname") $ }}.{{ .Release.Namespace }}:{{ $.Values.components.router.port }}/graphql
194+
{{- end -}}
195+
196+
{{/*
197+
Get the lifecycle of the pod. When KEDA is activated and the lifecycle is used for a pod of a
198+
deployment preStop hook to deregister from the selenium hub.
199+
*/}}
200+
{{- define "seleniumGrid.lifecycle" }}
201+
{{ $lifecycle := tpl (toYaml (default (dict) .node.lifecycle)) $ }}
202+
{{- if and (eq .Values.autoscaling.scalingType "deployment") (eq (include "seleniumGrid.useKEDA" .) "true") -}}
203+
{{ $lifecycle = merge ($lifecycle | fromYaml ) .Values.autoscaling.deregisterLifecycle | toYaml }}
204+
{{- end -}}
205+
{{ if and $lifecycle (ne $lifecycle "{}") -}}
206+
lifecycle: {{ $lifecycle | nindent 2 }}
207+
{{- end -}}
208+
{{- end -}}

‎charts/selenium-grid/templates/chrome-node-deployment.yaml

Lines changed: 6 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
{{- if and .Values.chromeNode.enabled .Values.chromeNode.deploymentEnabled }}
1+
{{- if and .Values.chromeNode.enabled ((eq (include "seleniumGrid.useKEDA" .) "true") | ternary (eq .Values.autoscaling.scalingType "deployment") .Values.chromeNode.deploymentEnabled) }}
22
apiVersion: apps/v1
33
kind: Deployment
44
metadata:
55
name: {{ template "seleniumGrid.chromeNode.fullname" . }}
66
namespace: {{ .Release.Namespace }}
7-
labels: &chrome_node_labels
7+
labels:
88
app: selenium-chrome-node
99
app.kubernetes.io/name: selenium-chrome-node
1010
{{- include "seleniumGrid.commonLabels" . | nindent 4 }}
@@ -20,81 +20,8 @@ spec:
2020
matchLabels:
2121
app: selenium-chrome-node
2222
app.kubernetes.io/instance: {{ .Release.Name }}
23-
template:
24-
metadata:
25-
labels: *chrome_node_labels
26-
annotations:
27-
checksum/event-bus-configmap: {{ include (print $.Template.BasePath "/event-bus-configmap.yaml") . | sha256sum }}
28-
{{- with .Values.chromeNode.annotations }}
29-
{{ toYaml . | nindent 8 }}
30-
{{- end }}
31-
spec:
32-
{{- with .Values.chromeNode.hostAliases }}
33-
hostAliases: {{ toYaml . | nindent 8 }}
34-
{{- end }}
35-
containers:
36-
- name: selenium-chrome-node
37-
{{- $imageTag := default .Values.global.seleniumGrid.nodesImageTag .Values.chromeNode.imageTag }}
38-
image: {{ printf "%s:%s" .Values.chromeNode.imageName $imageTag }}
39-
imagePullPolicy: {{ .Values.chromeNode.imagePullPolicy }}
40-
{{- with .Values.chromeNode.extraEnvironmentVariables }}
41-
env: {{- tpl (toYaml .) $ | nindent 12 }}
42-
{{- end }}
43-
envFrom:
44-
- configMapRef:
45-
name: {{ .Values.busConfigMap.name }}
46-
{{- with .Values.chromeNode.extraEnvFrom }}
47-
{{- toYaml . | nindent 12 }}
48-
{{- end }}
49-
{{- if gt (len .Values.chromeNode.ports) 0 }}
50-
ports:
51-
{{- range .Values.chromeNode.ports }}
52-
- containerPort: {{ . }}
53-
protocol: TCP
54-
{{- end }}
55-
{{- end }}
56-
volumeMounts:
57-
- name: dshm
58-
mountPath: /dev/shm
59-
{{- if .Values.chromeNode.extraVolumeMounts }}
60-
{{- toYaml .Values.chromeNode.extraVolumeMounts | nindent 12 }}
61-
{{- end }}
62-
{{- with .Values.chromeNode.resources }}
63-
resources: {{- toYaml . | nindent 12 }}
64-
{{- end }}
65-
{{- with .Values.chromeNode.securityContext }}
66-
securityContext: {{- toYaml . | nindent 12 }}
67-
{{- end }}
68-
{{- with .Values.chromeNode.lifecycle }}
69-
lifecycle: {{- toYaml . | nindent 12 }}
70-
{{- end }}
71-
{{- with .Values.chromeNode.startupProbe }}
72-
startupProbe: {{- toYaml . | nindent 12 }}
73-
{{- end }}
74-
{{- if or .Values.global.seleniumGrid.imagePullSecret .Values.chromeNode.imagePullSecret }}
75-
imagePullSecrets:
76-
- name: {{ default .Values.global.seleniumGrid.imagePullSecret .Values.chromeNode.imagePullSecret }}
77-
{{- end }}
78-
{{- with .Values.chromeNode.nodeSelector }}
79-
nodeSelector: {{- toYaml . | nindent 8 }}
80-
{{- end }}
81-
{{- if or .Values.global.seleniumGrid.affinity .Values.chromeNode.affinity }}
82-
{{- $affinityYaml := default .Values.global.seleniumGrid.affinity .Values.chromeNode.affinity }}
83-
affinity: {{- toYaml $affinityYaml | nindent 8 }}
84-
{{- end }}
85-
{{- with .Values.chromeNode.tolerations }}
86-
tolerations: {{ toYaml . | nindent 6 }}
87-
{{- end }}
88-
{{- with .Values.chromeNode.priorityClassName }}
89-
priorityClassName: {{ . }}
90-
{{- end }}
91-
terminationGracePeriodSeconds: {{ .Values.chromeNode.terminationGracePeriodSeconds }}
92-
volumes:
93-
- name: dshm
94-
emptyDir:
95-
medium: Memory
96-
sizeLimit: {{ default "1Gi" .Values.chromeNode.dshmVolumeSizeLimit }}
97-
{{- if .Values.chromeNode.extraVolumes }}
98-
{{ toYaml .Values.chromeNode.extraVolumes | nindent 8 }}
99-
{{- end }}
23+
{{- $podScope := deepCopy . -}}
24+
{{- $_ := set $podScope "name" "selenium-chrome-node" -}}
25+
{{- $_ = set $podScope "node" .Values.chromeNode -}}
26+
{{- include "seleniumGrid.podTemplate" $podScope | nindent 2 }}
10027
{{- end }}

‎charts/selenium-grid/templates/chrome-node-hpa.yaml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
{{- if and .Values.chromeNode.enabled (eq (include "seleniumGrid.useKEDA" .) "true") (eq .Values.autoscaling.scalingType "deployment") }}
2+
apiVersion: keda.sh/v1alpha1
3+
kind: ScaledObject
4+
metadata:
5+
name: selenium-grid-chrome-scaledobject
6+
namespace: {{ .Release.Namespace }}
7+
annotations:
8+
{{- with .Values.autoscaling.annotations }}
9+
{{- toYaml . | nindent 4 }}
10+
{{- end }}
11+
labels:
12+
deploymentName: {{ template "seleniumGrid.chromeNode.fullname" . }}
13+
spec:
14+
maxReplicaCount: {{ .Values.chromeNode.maxReplicaCount }}
15+
scaleTargetRef:
16+
name: {{ template "seleniumGrid.chromeNode.fullname" . }}
17+
triggers:
18+
- type: selenium-grid
19+
{{- with .Values.chromeNode.hpa }}
20+
metadata: {{- tpl (toYaml .) $ | nindent 8 }}
21+
{{- end }}
22+
{{- end }}

‎charts/selenium-grid/templates/chrome-node-scaledjobs.yaml

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
{{- if and .Values.chromeNode.enabled (include "seleniumGrid.useKEDA" .) (eq .Values.autoscaling.scalingType "job") }}
2+
apiVersion: keda.sh/v1alpha1
3+
kind: ScaledJob
4+
metadata:
5+
name: {{ template "seleniumGrid.chromeNode.fullname" . }}
6+
namespace: {{ .Release.Namespace }}
7+
annotations:
8+
{{- with .Values.autoscaling.annotations }}
9+
{{- toYaml . | nindent 4 }}
10+
{{- end }}
11+
labels:
12+
app: selenium-chrome-node
13+
app.kubernetes.io/name: selenium-chrome-node
14+
{{- include "seleniumGrid.commonLabels" . | nindent 4 }}
15+
{{- with .Values.chromeNode.labels }}
16+
{{- toYaml . | nindent 4 }}
17+
{{- end }}
18+
{{- with .Values.customLabels }}
19+
{{- toYaml . | nindent 4 }}
20+
{{- end }}
21+
spec:
22+
maxReplicaCount: {{ .Values.chromeNode.maxReplicaCount }}
23+
{{- with .Values.autoscaling.scaledJobOptions -}}
24+
{{ toYaml . | nindent 2 }}
25+
{{- end }}
26+
triggers:
27+
- type: selenium-grid
28+
{{- with .Values.chromeNode.hpa }}
29+
metadata: {{- tpl (toYaml .) $ | nindent 8 }}
30+
{{- end }}
31+
jobTargetRef:
32+
parallelism: 1
33+
completions: 1
34+
backoffLimit: 0
35+
{{- $podScope := deepCopy . -}}
36+
{{- $_ := set $podScope "name" "selenium-chrome-node" -}}
37+
{{- $_ = set $podScope "node" .Values.chromeNode -}}
38+
{{- include "seleniumGrid.podTemplate" $podScope | nindent 4 }}
39+
{{- end }}

0 commit comments

Comments
 (0)