Add liveness check against /state

alb-ingress-controller-helm/templates/controller-deployment.yaml

@@ -65,6 +65,14 @@ spec:
               scheme: HTTP
             initialDelaySeconds: 30
             periodSeconds: {{ .Values.controller.readinessProbeInterval }}
+            timeout: {{ .Values.controller.readinessProbeTimeout }}
+          livenessProbe:
+            httpGet:
+              path: /alive
+              port: 8080
+              scheme: HTTP
+            initialDelaySeconds: 30
+            periodSeconds: 60
         {{- if .Values.controller.resources }}
           resources:
 {{ toYaml .Values.controller.resources | indent 12 }}

alb-ingress-controller-helm/values.yaml

@@ -43,6 +43,9 @@ controller:
   # How often (in seconds) to check controller readiness
   readinessProbeInterval: 10
 
+  # How long to wait before timeout (in seconds) when checking controller readiness
+  readinessProbeTimeout: 1
+
   resources: {}
     # limits:
     #   cpu: 100m

cmd/main.go

@@ -56,6 +56,7 @@ func main() {
 
 	http.HandleFunc("/state", ac.StateHandler)
 	http.HandleFunc("/healthz", ac.StatusHandler)
+	http.HandleFunc("/alive", ac.AliveHandler)
 
 	defer func() {
 		logger.Infof("Shutting down ingress controller...")

pkg/aws/albec2/ec2.go

@@ -84,7 +84,7 @@ func (e *EC2) DescribeSGByPermissionGroup(sg *string) (*string, error) {
 	}
 
 	if len(o.SecurityGroups) != 1 {
-		return nil, fmt.Errorf("Found more than 1 matching (managed) instance SGs. Found %d", len(o.SecurityGroups))
+		return nil, fmt.Errorf("Didn't find exactly 1 matching (managed) instance SG. Found %d", len(o.SecurityGroups))
 	}
 
 	e.cache.Set(key, o.SecurityGroups[0].GroupId, time.Minute*5)

pkg/controller/alb-controller.go

@@ -63,7 +63,10 @@ type albController struct {
 
 var logger *log.Logger
 
+// Release contains a default value but it's also exported so that it can be overriden with buildFlags
 var Release = "1.0.0"
+
+// Build contains a default value but it's also exported so that it can be overriden with buildFlags
 var Build = "git-00000000"
 
 func init() {
@@ -315,6 +318,7 @@ func (ac *albController) StateHandler(w http.ResponseWriter, r *http.Request) {
 	ac.mutex.RLock()
 	defer ac.mutex.RUnlock()
 	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusOK)
 	_ = json.NewEncoder(w).Encode(ac.ALBIngresses)
 }
 
@@ -356,6 +360,21 @@ func (ac *albController) StatusHandler(w http.ResponseWriter, r *http.Request) {
 	encoder.Encode(checkResults)
 }
 
+// AliveHandler validates the bare-minimum internals and only returns a empty response.
+// It checks nothing downstream & should only used to ensure the controller is still running.
+func (ac *albController) AliveHandler(w http.ResponseWriter, r *http.Request) {
+	// Take a lock here as a lightweight/minimum way to check the controller is alive.
+	ac.mutex.RLock()
+	defer ac.mutex.RUnlock()
+	w.Header().Set("Content-Type", "application/json")
+	// Explicitly set a healthy response so that this handler can be used to ascertain liveness.
+	w.WriteHeader(http.StatusOK)
+
+	// Kubernetes only cares about the HTTP status code, so just return an empty body
+	w.Write([]byte("{}\n"))
+	return
+}
+
 // UpdateIngressStatus returns the hostnames for the ALB.
 func (ac *albController) UpdateIngressStatus(ing *extensions.Ingress) []api.LoadBalancerIngress {
 	id := albingress.GenerateID(ing.ObjectMeta.Namespace, ing.ObjectMeta.Name)

pkg/controller/alb-controller_test.go

@@ -313,6 +313,7 @@ func TestALBController_StateHandler(t *testing.T) {
 	ingress := albingress.ALBIngress{}
 	encodedIngressByteSlice, _ := json.Marshal(ingress)
 	expectedBody := fmt.Sprintf("[%s]\n", encodedIngressByteSlice)
+	expectedResponseCode := 200
 	ac := albController{
 		ALBIngresses: []*albingress.ALBIngress{&ingress},
 	}
@@ -323,6 +324,9 @@ func TestALBController_StateHandler(t *testing.T) {
 	if rw.Header().Get("Content-Type") != "application/json" {
 		t.Errorf("Expected header Content-Type: application-json")
 	}
+	if rw.Result().StatusCode != expectedResponseCode {
+		t.Errorf("Expected http status code to be %d, got %d", expectedResponseCode, rw.Result().StatusCode)
+	}
 	bodyString := fmt.Sprintf("%s", rw.Body.Bytes())
 	if expectedBody != bodyString {
 		t.Errorf("Expected response body to be '%s', found '%s'", expectedBody, bodyString)
@@ -366,3 +370,23 @@ func TestALBController_StatusHandler(t *testing.T) {
 		}
 	}
 }
+func TestALBController_AliveHandler(t *testing.T) {
+
+	expectedResponseCode := 200
+	expectedResponseBody := "{}\n"
+	ac := albController{}
+	w := httptest.NewRecorder()
+	ac.AliveHandler(w, nil)
+
+	if w.Header().Get("Content-Type") != "application/json" {
+		t.Errorf("Expected header Content-Type: application-json")
+	}
+	if w.Result().StatusCode != expectedResponseCode {
+		t.Errorf("Expected http status code to be %d, got %d", expectedResponseCode, w.Result().StatusCode)
+	}
+
+	responseBody := fmt.Sprintf("%s", string(w.Body.Bytes()[:w.Body.Len()]))
+	if responseBody != expectedResponseBody {
+		t.Errorf("Expected response body to be '%s', found '%s'", expectedResponseBody, responseBody)
+	}
+}