TEST: Add e2e test for non ready endpoints

In order to test if controller is not using non ready endpoints we do
the following in the corresponding e2e test:
- Create non ready pods by configuring their readiness probe that
  to never succeed
- Create one healthy pod
- Add an HAProxy rule that will return 503 if there is more than one
  server in the test backend.

Author: Mo3m3n

deploy/tests/e2e/endpoints/config/endpoints.yaml.tmpl

@@ -22,6 +22,13 @@ spec:
             - name: https
               containerPort: 8443
               protocol: TCP
+          {{- if .NotReady}}
+          readinessProbe:
+            failureThreshold: 3
+            periodSeconds: 1
+            httpGet:
+              port: 6666
+          {{- end}}
 ---
 kind: Service
 apiVersion: v1
@@ -46,6 +53,11 @@ metadata:
   name: http-echo
   annotations:
     ingress.class: haproxy
+    {{- if .NotReady}}
+    check: "false"
+    backend-config-snippet: |
+      http-request return status 503 if !{ nbsrv() eq 1 }
+    {{- end}}
 spec:
   rules:
   - host: {{ .Host }}

deploy/tests/e2e/endpoints/http_test.go

@@ -19,7 +19,6 @@ package endpoints
 import (
 	"fmt"
 	"io/ioutil"
-	"net/http"
 
 	"github.com/haproxytech/kubernetes-ingress/deploy/tests/e2e"
 )
@@ -39,7 +38,7 @@ func (suite *EndpointsSuite) Test_HTTP_Reach() {
 						res, cls, err := suite.client.Do()
 						suite.NoError(err)
 						defer cls()
-						if res.StatusCode == http.StatusOK {
+						if res.StatusCode == 200 {
 							body, err := ioutil.ReadAll(res.Body)
 							if err != nil {
 								suite.Error(err)

deploy/tests/e2e/endpoints/not_ready.go

@@ -0,0 +1,36 @@
+// Copyright 2019 HAProxy Technologies LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build e2e_sequential
+
+package endpoints
+
+import (
+	"github.com/haproxytech/kubernetes-ingress/deploy/tests/e2e"
+)
+
+func (suite *EndpointsSuite) Test_Non_Ready_Endpoints() {
+	suite.tmplData.NotReady = true
+	suite.tmplData.Replicas = 3
+	suite.NoError(suite.test.Apply("config/endpoints.yaml.tmpl", suite.test.GetNS(), suite.tmplData))
+	suite.Require().Eventually(func() bool {
+		res, cls, err := suite.client.Do()
+		if res == nil {
+			suite.T().Log(err)
+			return false
+		}
+		defer cls()
+		return res.StatusCode == 200
+	}, e2e.WaitDuration, e2e.TickDuration)
+}

deploy/tests/e2e/endpoints/suite_test.go

@@ -35,15 +35,15 @@ type EndpointsSuite struct {
 type tmplData struct {
 	Replicas int
 	Host     string
+	NotReady bool
 }
 
 func (suite *EndpointsSuite) SetupSuite() {
 	var err error
 	suite.test, err = e2e.NewTest()
 	suite.NoError(err)
 	suite.tmplData = tmplData{
-		Replicas: 1,
-		Host:     suite.test.GetNS() + ".test",
+		Host: suite.test.GetNS() + ".test",
 	}
 }
 
@@ -62,10 +62,15 @@ func (suite *EndpointsSuite) BeforeTest(suiteName, testName string) {
 	var err error
 	test := suite.test
 	switch testName {
-	case "Test_HTTP_Reach":
-		suite.client, err = e2e.NewHTTPClient(suite.tmplData.Host)
+	case "Test_TCP_Reach":
+		suite.client, err = e2e.NewHTTPSClient("tcp-service.test", 32766)
 		suite.NoError(err)
+		suite.tmplData.Replicas = 4
 		suite.NoError(test.Apply("config/endpoints.yaml.tmpl", test.GetNS(), suite.tmplData))
+		suite.NoError(test.Apply("config/tcp.yaml", "", nil))
+		test.AddTearDown(func() error {
+			return suite.test.Delete("config/tcp.yaml")
+		})
 		suite.Require().Eventually(func() bool {
 			res, cls, err := suite.client.Do()
 			if res == nil {
@@ -75,15 +80,11 @@ func (suite *EndpointsSuite) BeforeTest(suiteName, testName string) {
 			defer cls()
 			return res.StatusCode == http.StatusOK
 		}, e2e.WaitDuration, e2e.TickDuration)
-	case "Test_TCP_Reach":
-		suite.client, err = e2e.NewHTTPSClient("tcp-service.test", 32766)
+	default:
+		suite.client, err = e2e.NewHTTPClient(suite.tmplData.Host)
 		suite.NoError(err)
-		suite.tmplData.Replicas = 4
+		suite.tmplData.Replicas = 1
 		suite.NoError(test.Apply("config/endpoints.yaml.tmpl", test.GetNS(), suite.tmplData))
-		suite.NoError(test.Apply("config/tcp.yaml", "", nil))
-		test.AddTearDown(func() error {
-			return suite.test.Delete("config/tcp.yaml")
-		})
 		suite.Require().Eventually(func() bool {
 			res, cls, err := suite.client.Do()
 			if res == nil {