MAJOR: add default server service behavior to k8s services

oktalz · oktalz · commit a224187c47c6 · 2022-05-24T15:30:10.000+02:00
if no default service is defined, one will be created locally.
it will behave the same way as other default services
- 200 on healtz
- 404 on all other endpoints
diff --git a/main.go b/main.go
@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"os"
 	"os/signal"
+	"strconv"
 	"strings"
 	"syscall"
 
@@ -74,6 +75,7 @@ func main() {
 	defaultBackendSvc := fmt.Sprint(osArgs.DefaultBackendService)
 	defaultCertificate := fmt.Sprint(osArgs.DefaultCertificate)
 	annotations.SetDefaultValue("default-backend-service", defaultBackendSvc)
+	annotations.SetDefaultValue("default-backend-port", strconv.Itoa(osArgs.DefaultBackendPort))
 	annotations.SetDefaultValue("ssl-certificate", defaultCertificate)
 
 	// Start Controller
@@ -138,7 +140,11 @@ func logInfo(logger utils.Logger, osArgs utils.OSArgs) {
 	logger.Printf("Ingress class: %s", osArgs.IngressClass)
 	logger.Printf("Empty Ingress class: %t", osArgs.EmptyIngressClass)
 	logger.Printf("Publish service: %s", osArgs.PublishService)
-	logger.Printf("Default backend service: %s", osArgs.DefaultBackendService)
+	if osArgs.DefaultBackendService.String() != "" {
+		logger.Printf("Default backend service: %s", osArgs.DefaultBackendService)
+	} else {
+		logger.Printf("Using local backend service on port: %s", osArgs.DefaultBackendPort)
+	}
 	logger.Printf("Default ssl certificate: %s", osArgs.DefaultCertificate)
 	if !osArgs.DisableHTTP {
 		logger.Printf("Frontend HTTP listening on: %s:%d", osArgs.IPV4BindAddr, osArgs.HTTPBindPort)
diff --git a/pkg/controller/builder.go b/pkg/controller/builder.go
@@ -120,23 +120,23 @@ func (builder *Builder) Build() *HAProxyController {
 	}
 
 	chShutdown := make(chan struct{})
-	rtr := router.New()
 	if builder.osArgs.ControllerPort != 0 {
+		rtr := router.New()
 		var runningServices string
 		if builder.osArgs.PprofEnabled {
 			rtr.GET("/debug/pprof/{profile:*}", pprofhandler.PprofHandler)
-			runningServices += " pprof,"
+			runningServices += " pprof"
 		}
 		if builder.osArgs.PrometheusEnabled {
 			rtr.GET("/metrics", fasthttpadaptor.NewFastHTTPHandler(promhttp.Handler()))
-			runningServices += " prometheus,"
+			runningServices += ", prometheus"
 		}
-		runningServices += " default service"
 		rtr.GET("/healtz", requestHandler)
 		// all others will be 404
 		go func() {
 			server := fasthttp.Server{
-				Handler: rtr.Handler,
+				Handler:               rtr.Handler,
+				NoDefaultServerHeader: true,
 			}
 			go func() {
 				<-chShutdown
@@ -152,6 +152,29 @@ func (builder *Builder) Build() *HAProxyController {
 		}()
 	}
 
+	if builder.osArgs.DefaultBackendService.String() == "" {
+		rtr := router.New()
+		rtr.GET("/healtz", requestHandler)
+		// all others will be 404
+		go func() {
+			server := fasthttp.Server{
+				Handler:               rtr.Handler,
+				NoDefaultServerHeader: true,
+			}
+			go func() {
+				<-chShutdown
+				if err := server.Shutdown(); err != nil {
+					logger.Errorf("Could not gracefully shutdown controller data server: %v\n", err)
+				} else {
+					logger.Errorf("Gracefully shuting down controller data server")
+				}
+			}()
+			logger.Infof("running default backend server on :%d", builder.osArgs.DefaultBackendPort)
+			err := server.ListenAndServe(":" + strconv.Itoa(builder.osArgs.DefaultBackendPort))
+			logger.Error(err)
+		}()
+	}
+
 	haproxy, err := haproxy.New(builder.osArgs, builder.haproxyEnv, builder.haproxyCfgFile, builder.haproxyProcess, builder.haproxyClient, builder.haproxyRules)
 	logger.Panic(err)
 
diff --git a/pkg/controller/global.go b/pkg/controller/global.go
@@ -148,15 +148,15 @@ func (c *HAProxyController) defaultsCfg() (reload bool) {
 // handleDefaultService configures HAProy default backend provided via cli param "default-backend-service"
 func (c *HAProxyController) handleDefaultService() (reload bool) {
 	var svc *service.Service
-	ns, name, err := common.GetK8sPath("default-backend-service", c.store.ConfigMaps.Main.Annotations)
+	namespace, name, err := common.GetK8sPath("default-backend-service", c.store.ConfigMaps.Main.Annotations)
 	if err != nil {
 		logger.Errorf("default service: %s", err)
 	}
 	if name == "" {
-		return
+		return c.handleDefaultServicePort()
 	}
 	ingressPath := &store.IngressPath{
-		SvcNamespace:     ns,
+		SvcNamespace:     namespace,
 		SvcName:          name,
 		IsDefaultBackend: true,
 	}
@@ -166,7 +166,42 @@ func (c *HAProxyController) handleDefaultService() (reload bool) {
 	if err != nil {
 		logger.Errorf("default service: %s", err)
 	}
-	return
+	return false
+}
+
+// handleDefaultServicePort configures local HAProy default backend provided via cli param "default-backend-port"
+func (c *HAProxyController) handleDefaultServicePort() (reload bool) {
+	var svc *service.Service
+	_, portStr, err := common.GetK8sPath("default-backend-port", c.store.ConfigMaps.Main.Annotations)
+	if err != nil {
+		logger.Errorf("default backend port: %s", err)
+	}
+	if portStr == "" {
+		return
+	}
+	defaultLocalBackend := "default_local_backend"
+	ingressPath := &store.IngressPath{
+		SvcNamespace:     "",
+		SvcName:          defaultLocalBackend,
+		SvcPortString:    portStr,
+		IsDefaultBackend: true,
+	}
+
+	err = c.haproxy.BackendCreate(models.Backend{
+		Name: defaultLocalBackend,
+	})
+	if err != nil {
+		logger.Errorf("default backend port: %s", err)
+	}
+	backend, _ := c.haproxy.BackendGet(defaultLocalBackend)
+
+	if svc, err = service.NewLocal(c.store, ingressPath, backend, c.store.ConfigMaps.Main.Annotations); err == nil {
+		reload, err = svc.SetDefaultBackend(c.store, c.haproxy, []string{c.haproxy.FrontHTTP, c.haproxy.FrontHTTPS}, c.annotations)
+	}
+	if err != nil {
+		logger.Errorf("default service port: %s", err)
+	}
+	return true
 }
 
 // handleDefaultCert configures default/fallback HAProxy certificate to use for client HTTPS requests.
diff --git a/pkg/controller/handler.go b/pkg/controller/handler.go
@@ -54,6 +54,12 @@ func (c *HAProxyController) initHandlers() {
 	if c.osArgs.PprofEnabled {
 		c.updateHandlers = append(c.updateHandlers, handler.Pprof{})
 	}
+	if c.osArgs.DefaultBackendService.String() == "" {
+		c.updateHandlers = append(c.updateHandlers, handler.DefaultLocalService{
+			Name: "default_local_backend",
+			Port: c.osArgs.DefaultBackendPort,
+		})
+	}
 	c.updateHandlers = append(c.updateHandlers, handler.Refresh{})
 }
 
@@ -69,7 +75,8 @@ func (c *HAProxyController) startupHandlers() error {
 			HTTPSPort: c.osArgs.HTTPSBindPort,
 			IPv4Addr:  c.osArgs.IPV4BindAddr,
 			IPv6Addr:  c.osArgs.IPV6BindAddr,
-		}}
+		},
+	}
 	for _, handler := range handlers {
 		_, err := handler.Update(c.store, c.haproxy, c.annotations)
 		if err != nil {
diff --git a/pkg/handler/default-local-service.go b/pkg/handler/default-local-service.go
@@ -0,0 +1,52 @@
+// Copyright 2019 HAProxy Technologies LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package handler
+
+import (
+	"strconv"
+
+	"github.com/haproxytech/client-native/v3/models"
+
+	"github.com/haproxytech/kubernetes-ingress/pkg/annotations"
+	"github.com/haproxytech/kubernetes-ingress/pkg/haproxy"
+	"github.com/haproxytech/kubernetes-ingress/pkg/store"
+)
+
+type DefaultLocalService struct {
+	Name string
+	Port int
+}
+
+func (handler DefaultLocalService) Update(k store.K8s, api haproxy.HAProxy, ann annotations.Annotations) (reload bool, err error) {
+	_, err = api.ServerGet("ingress_controller", handler.Name)
+	if err != nil {
+		/*err = api.BackendCreate(models.Backend{
+			Name: DefaultLocalBackend,
+			Mode: "http",
+		})
+		if err != nil {
+			return false, err
+		}*/
+		err = api.BackendServerCreate(handler.Name, models.Server{
+			Name:    "ingress_controller",
+			Address: "127.0.0.1:" + strconv.Itoa(handler.Port),
+		})
+		if err != nil {
+			return false, err
+		}
+		logger.Debug("DefaultLocalService backend created")
+	}
+	return false, nil
+}
diff --git a/pkg/service/endpoints.go b/pkg/service/endpoints.go
@@ -30,6 +30,9 @@ func (s *Service) HandleHAProxySrvs(store store.K8s, client api.HAProxyClient) (
 	var srvsScaled bool
 	backend, err := s.getRuntimeBackend(store)
 	if err != nil {
+		if s.backend != nil && s.backend.Name == "default_local_backend" {
+			return
+		}
 		logger.Warningf("Ingress '%s/%s': %s", s.resource.Namespace, s.resource.Name, err)
 		if servers, _ := client.BackendServersGet(s.backend.Name); servers != nil {
 			client.BackendServerDeleteAll(s.backend.Name)
diff --git a/pkg/service/service.go b/pkg/service/service.go
@@ -63,6 +63,18 @@ func New(k store.K8s, path *store.IngressPath, certs certs.Certificates, tcpServ
 	}, nil
 }
 
+// NewLocal returns a Service instance to handle the k8s IngressPath resource given in params.
+func NewLocal(k store.K8s, path *store.IngressPath, backend *models.Backend, annList ...map[string]string) (*Service, error) {
+	return &Service{
+		path: path,
+		resource: &store.Service{
+			Annotations: map[string]string{},
+		},
+		annotations: annList,
+		backend:     backend,
+	}, nil
+}
+
 func (s *Service) GetResource() *store.Service {
 	return s.resource
 }
@@ -140,7 +152,7 @@ func (s *Service) HandleBackend(store store.K8s, client api.HAProxyClient, a ann
 	return
 }
 
-// getBackendModel checks for a corresponding custom resource before falling back to annoations
+// getBackendModel checks for a corresponding custom resource before falling back to annotations
 func (s *Service) getBackendModel(store store.K8s, a annotations.Annotations) (backend *models.Backend, err error) {
 	// Backend mode
 	mode := "http"
diff --git a/pkg/utils/flags.go b/pkg/utils/flags.go
@@ -69,6 +69,7 @@ type OSArgs struct { //nolint:maligned
 	Help                       []bool         `short:"h" long:"help" description:"show this help message"`
 	Version                    []bool         `short:"v" long:"version" description:"version"`
 	DefaultBackendService      NamespaceValue `long:"default-backend-service" default:"" description:"default service to serve 404 page. If not specified HAProxy serves http 400"`
+	DefaultBackendPort         int            `long:"default-backend-port" description:"port to use for default service" default:"6061"`
 	DefaultCertificate         NamespaceValue `long:"default-ssl-certificate" default:"" description:"secret name of the certificate"`
 	ConfigMap                  NamespaceValue `long:"configmap" description:"configmap designated for HAProxy" default:""`
 	ConfigMapTCPServices       NamespaceValue `long:"configmap-tcp-services" description:"configmap used to define tcp services" default:""`
