add e2e tests based on a vanilla ingress (#2294)

Co-authored-by: Kishor Joshi <[email protected]>

Author: M00nF1sh

go.mod

@@ -3,12 +3,15 @@ module sigs.k8s.io/aws-load-balancer-controller
 go 1.16
 
 require (
+	github.com/Masterminds/sprig/v3 v3.2.2 // indirect
 	github.com/aws/aws-sdk-go v1.40.7
+	github.com/fatih/color v1.7.0 // indirect
+	github.com/gavv/httpexpect/v2 v2.3.1 // indirect
 	github.com/go-logr/logr v0.4.0
 	github.com/golang/mock v1.6.0
 	github.com/google/go-cmp v0.5.6
 	github.com/onsi/ginkgo v1.16.4
-	github.com/onsi/gomega v1.13.0
+	github.com/onsi/gomega v1.16.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.11.0
 	github.com/spf13/pflag v1.0.5

go.sum

@@ -0,0 +1,130 @@
+package fixture
+
+import (
+	"context"
+	"github.com/pkg/errors"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"sigs.k8s.io/aws-load-balancer-controller/pkg/k8s"
+	"sigs.k8s.io/aws-load-balancer-controller/test/framework"
+	"sigs.k8s.io/aws-load-balancer-controller/test/framework/utils"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
+	"sort"
+)
+
+var (
+	defaultKindOrdering = []string{
+		"IngressClass",
+		"Namespace",
+		"Service",
+		"Deployment",
+		"Ingress",
+	}
+)
+
+// NewK8SResourceStack constructs new Resource stack.
+func NewK8SResourceStack(tf *framework.Framework, objs ...client.Object) *k8sResourceStack {
+	return &k8sResourceStack{
+		tf:              tf,
+		objs:            objs,
+		provisionedObjs: nil,
+		kindOrdering:    defaultKindOrdering,
+	}
+}
+
+type k8sResourceStack struct {
+	tf              *framework.Framework
+	objs            []client.Object
+	provisionedObjs []client.Object
+	kindOrdering    []string
+}
+
+func (s *k8sResourceStack) Setup(ctx context.Context) error {
+	scheme := s.tf.K8sClient.Scheme()
+	sortedObjs, err := s.sortedObjectsOrderingByKind(s.kindOrdering)
+	if err != nil {
+		return err
+	}
+	for _, obj := range sortedObjs {
+		objKind, _ := apiutil.GVKForObject(obj, scheme)
+		s.tf.Logger.Info("creating resource", "kind", objKind.Kind, "key", k8s.NamespacedName(obj))
+		if err := s.tf.K8sClient.Create(ctx, obj); err != nil {
+			return err
+		}
+		s.provisionedObjs = append(s.provisionedObjs, obj)
+		s.tf.Logger.Info("created resource", "kind", objKind.Kind, "key", k8s.NamespacedName(obj))
+	}
+	return nil
+}
+
+func (s *k8sResourceStack) TearDown(ctx context.Context) error {
+	scheme := s.tf.K8sClient.Scheme()
+	for i := len(s.provisionedObjs) - 1; i >= 0; i-- {
+		obj := s.provisionedObjs[i]
+		objKind, _ := apiutil.GVKForObject(obj, scheme)
+		s.tf.Logger.Info("deleting resource", "kind", objKind.Kind, "key", k8s.NamespacedName(obj))
+		if err := s.tf.K8sClient.Delete(ctx, obj); err != nil {
+			return err
+		}
+		s.tf.Logger.Info("deleted resource", "kind", objKind.Kind, "key", k8s.NamespacedName(obj))
+
+		s.tf.Logger.Info("wait resource becomes deleted", "kind", objKind.Kind, "key", k8s.NamespacedName(obj))
+		if err := s.waitUntilResourceDeleted(ctx, obj); err != nil {
+			return err
+		}
+		s.tf.Logger.Info("resource becomes deleted", "kind", objKind.Kind, "key", k8s.NamespacedName(obj))
+	}
+	return nil
+}
+
+type objWithOrder struct {
+	obj   client.Object
+	order int
+}
+
+func (s *k8sResourceStack) sortedObjectsOrderingByKind(kindOrdering []string) ([]client.Object, error) {
+	ordering := make(map[string]int, len(kindOrdering))
+	for i, k := range kindOrdering {
+		ordering[k] = i
+	}
+
+	scheme := s.tf.K8sClient.Scheme()
+	var objWithOrderList []objWithOrder
+	for _, obj := range s.objs {
+		objKind, err := apiutil.GVKForObject(obj, scheme)
+		if err != nil {
+			return nil, err
+		}
+		order, ok := ordering[objKind.Kind]
+		if !ok {
+			return nil, errors.Errorf("ordering for %v Kind is not configured", objKind)
+		}
+		objWithOrderList = append(objWithOrderList, objWithOrder{
+			obj:   obj,
+			order: order,
+		})
+	}
+	sort.SliceStable(objWithOrderList, func(i, j int) bool {
+		return objWithOrderList[i].order < objWithOrderList[j].order
+	})
+
+	var sortedObjs []client.Object
+	for _, objWithOrder := range objWithOrderList {
+		sortedObjs = append(sortedObjs, objWithOrder.obj)
+	}
+	return sortedObjs, nil
+}
+
+func (s *k8sResourceStack) waitUntilResourceDeleted(ctx context.Context, obj client.Object) error {
+	observedObj := obj.DeepCopyObject().(client.Object)
+	return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) {
+		if err := s.tf.K8sClient.Get(ctx, k8s.NamespacedName(obj), observedObj); err != nil {
+			if apierrs.IsNotFound(err) {
+				return true, nil
+			}
+			return false, err
+		}
+		return false, nil
+	}, ctx.Done())
+}

test/e2e/ingress/vanilla_ingress_test.go

@@ -15,7 +15,6 @@ import (
 	k8sresources "sigs.k8s.io/aws-load-balancer-controller/test/framework/resources/k8s"
 	"sigs.k8s.io/aws-load-balancer-controller/test/framework/utils"
 	ctrl "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -35,7 +34,7 @@ type Framework struct {
 
 	HTTPVerifier http.Verifier
 
-	Logger logr.Logger
+	Logger utils.GinkgoLogger
 }
 
 func InitFramework() (*Framework, error) {
@@ -49,24 +48,7 @@ func InitFramework() (*Framework, error) {
 	clientgoscheme.AddToScheme(k8sSchema)
 	elbv2api.AddToScheme(k8sSchema)
 
-	ctx := ctrl.SetupSignalHandler()
-	cache, err := cache.New(restCfg, cache.Options{Scheme: k8sSchema})
-	if err != nil {
-		return nil, err
-	}
-	go func() {
-		cache.Start(ctx)
-	}()
-	cache.WaitForCacheSync(ctx)
-	realClient, err := client.New(restCfg, client.Options{Scheme: k8sSchema})
-	if err != nil {
-		return nil, err
-	}
-
-	k8sClient, err := client.NewDelegatingClient(client.NewDelegatingClientInput{
-		CacheReader: cache,
-		Client:      realClient,
-	})
+	k8sClient, err := client.New(restCfg, client.Options{Scheme: k8sSchema})
 	if err != nil {
 		return nil, err
 	}

test/framework/fixture/k8s_resource_stack.go

@@ -0,0 +1,142 @@
+package manifest
+
+import (
+	"fmt"
+	"github.com/aws/aws-sdk-go/aws"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+)
+
+// NewFixedResponseServiceBuilder constructs a builder that capable to build manifest for an HTTP service with fixed response.
+func NewFixedResponseServiceBuilder() *fixedResponseServiceBuilder {
+	return &fixedResponseServiceBuilder{
+		replicas:       1,
+		httpBody:       "Hello World!",
+		port:           80,
+		targetPort:     8080,
+		svcType:        corev1.ServiceTypeNodePort,
+		svcAnnotations: nil,
+	}
+}
+
+type fixedResponseServiceBuilder struct {
+	replicas       int32
+	httpBody       string
+	port           int32
+	targetPort     int32
+	svcType        corev1.ServiceType
+	svcAnnotations map[string]string
+}
+
+func (b *fixedResponseServiceBuilder) WithReplicas(replicas int32) *fixedResponseServiceBuilder {
+	b.replicas = replicas
+	return b
+}
+
+func (b *fixedResponseServiceBuilder) WithHTTPBody(httpBody string) *fixedResponseServiceBuilder {
+	b.httpBody = httpBody
+	return b
+}
+
+func (b *fixedResponseServiceBuilder) WithPort(port int32) *fixedResponseServiceBuilder {
+	b.port = port
+	return b
+}
+
+func (b *fixedResponseServiceBuilder) WithTargetPort(targetPort int32) *fixedResponseServiceBuilder {
+	b.targetPort = targetPort
+	return b
+}
+
+func (b *fixedResponseServiceBuilder) WithServiceType(svcType corev1.ServiceType) *fixedResponseServiceBuilder {
+	b.svcType = svcType
+	return b
+}
+
+func (b *fixedResponseServiceBuilder) WithServiceAnnotations(svcAnnotations map[string]string) *fixedResponseServiceBuilder {
+	b.svcAnnotations = svcAnnotations
+	return b
+}
+
+func (b *fixedResponseServiceBuilder) Build(namespace string, name string) (*appsv1.Deployment, *corev1.Service) {
+	dp := b.buildDeployment(namespace, name)
+	svc := b.buildService(namespace, name)
+	return dp, svc
+}
+
+// TODO: have a deployment builder that been called by this component :D.
+func (b *fixedResponseServiceBuilder) buildDeployment(namespace string, name string) *appsv1.Deployment {
+	podLabels := b.buildPodLabels(name)
+	dp := &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: namespace,
+			Name:      name,
+		},
+		Spec: appsv1.DeploymentSpec{
+			Selector: &metav1.LabelSelector{
+				MatchLabels: podLabels,
+			},
+			Replicas: aws.Int32(b.replicas),
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: podLabels,
+				},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name:  "app",
+							Image: "970805265562.dkr.ecr.us-west-2.amazonaws.com/colorteller:latest",
+							Ports: []corev1.ContainerPort{
+								{
+									ContainerPort: b.targetPort,
+								},
+							},
+							Env: []corev1.EnvVar{
+								{
+									Name:  "SERVER_PORT",
+									Value: fmt.Sprintf("%d", b.targetPort),
+								},
+								{
+									Name:  "COLOR",
+									Value: b.httpBody,
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+	return dp
+}
+
+func (b *fixedResponseServiceBuilder) buildService(namespace string, name string) *corev1.Service {
+	podLabels := b.buildPodLabels(name)
+	svc := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace:   namespace,
+			Name:        name,
+			Annotations: b.svcAnnotations,
+		},
+		Spec: corev1.ServiceSpec{
+			Type:     b.svcType,
+			Selector: podLabels,
+			Ports: []corev1.ServicePort{
+				{
+					Port:       b.port,
+					TargetPort: intstr.FromInt(int(b.targetPort)),
+					Protocol:   corev1.ProtocolTCP,
+				},
+			},
+		},
+	}
+	return svc
+}
+
+func (b *fixedResponseServiceBuilder) buildPodLabels(name string) map[string]string {
+	return map[string]string{
+		"app.kubernetes.io/name": name,
+	}
+}