add test framework (#1495)

Author: M00nF1sh

test/framework/framework.go

@@ -0,0 +1,100 @@
+package framework
+
+import (
+	"github.com/go-logr/logr"
+	. "github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/runtime"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+	elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1"
+	"sigs.k8s.io/aws-load-balancer-controller/pkg/aws"
+	"sigs.k8s.io/aws-load-balancer-controller/pkg/aws/throttle"
+	k8sresources "sigs.k8s.io/aws-load-balancer-controller/test/framework/resources/k8s"
+	"sigs.k8s.io/aws-load-balancer-controller/test/framework/utils"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sync"
+)
+
+var (
+	frameworkSingleton         *Framework
+	frameworkSingletonInitOnce sync.Once
+)
+
+type Framework struct {
+	Options   Options
+	RestCfg   *rest.Config
+	K8sClient client.Client
+	Cloud     aws.Cloud
+
+	NSManager  k8sresources.NamespaceManager
+	DPManager  k8sresources.DeploymentManager
+	SVCManager k8sresources.ServiceManager
+	INGManager k8sresources.IngressManager
+
+	Logger   logr.Logger
+	StopChan <-chan struct{}
+}
+
+// New constructs new framework
+func New() *Framework {
+	frameworkSingletonInitOnce.Do(func() {
+		frameworkSingleton = initFramework()
+	})
+	return frameworkSingleton
+}
+
+func initFramework() *Framework {
+	err := globalOptions.Validate()
+	Expect(err).NotTo(HaveOccurred())
+	restCfg := ctrl.GetConfigOrDie()
+
+	k8sSchema := runtime.NewScheme()
+	clientgoscheme.AddToScheme(k8sSchema)
+	elbv2api.AddToScheme(k8sSchema)
+
+	stopChan := ctrl.SetupSignalHandler()
+	cache, err := cache.New(restCfg, cache.Options{Scheme: k8sSchema})
+	Expect(err).NotTo(HaveOccurred())
+	go func() {
+		cache.Start(stopChan)
+	}()
+
+	cache.WaitForCacheSync(stopChan)
+	realClient, err := client.New(restCfg, client.Options{Scheme: k8sSchema})
+	Expect(err).NotTo(HaveOccurred())
+	k8sClient := client.DelegatingClient{
+		Reader: &client.DelegatingReader{
+			CacheReader:  cache,
+			ClientReader: realClient,
+		},
+		Writer:       realClient,
+		StatusClient: realClient,
+	}
+	cloud, err := aws.NewCloud(aws.CloudConfig{
+		Region:         globalOptions.AWSRegion,
+		VpcID:          globalOptions.AWSVPCID,
+		MaxRetries:     3,
+		ThrottleConfig: throttle.NewDefaultServiceOperationsThrottleConfig(),
+	}, nil)
+	Expect(err).NotTo(HaveOccurred())
+
+	logger := utils.NewGinkgoLogger()
+
+	f := &Framework{
+		Options:   globalOptions,
+		RestCfg:   restCfg,
+		K8sClient: k8sClient,
+		Cloud:     cloud,
+
+		NSManager:  k8sresources.NewDefaultNamespaceManager(k8sClient, logger),
+		DPManager:  k8sresources.NewDefaultDeploymentManager(k8sClient, logger),
+		SVCManager: k8sresources.NewDefaultServiceManager(k8sClient, logger),
+		INGManager: k8sresources.NewDefaultIngressManager(k8sClient, logger),
+		Logger:     logger,
+		StopChan:   stopChan,
+	}
+
+	return f
+}

test/framework/options.go

@@ -0,0 +1,38 @@
+package framework
+
+import (
+	"flag"
+	"github.com/pkg/errors"
+)
+
+var globalOptions Options
+
+func init() {
+	globalOptions.BindFlags()
+}
+
+// configuration options
+type Options struct {
+	ClusterName string
+	AWSRegion   string
+	AWSVPCID    string
+}
+
+func (options *Options) BindFlags() {
+	flag.StringVar(&options.ClusterName, "cluster-name", "", `Kubernetes cluster name (required)`)
+	flag.StringVar(&options.AWSRegion, "aws-region", "", `AWS Region for the kubernetes cluster`)
+	flag.StringVar(&options.AWSVPCID, "aws-vpc-id", "", `AWS VPC ID for the kubernetes cluster`)
+}
+
+func (options *Options) Validate() error {
+	if len(options.ClusterName) == 0 {
+		return errors.Errorf("%s must be set!", "cluster-name")
+	}
+	if len(options.AWSRegion) == 0 {
+		return errors.Errorf("%s must be set!", "aws-region")
+	}
+	if len(options.AWSVPCID) == 0 {
+		return errors.Errorf("%s must be set!", "aws-vpc-id")
+	}
+	return nil
+}

test/framework/resources/k8s/deployment.go

@@ -0,0 +1,63 @@
+package k8s
+
+import (
+	"context"
+	"github.com/go-logr/logr"
+	appsv1 "k8s.io/api/apps/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"sigs.k8s.io/aws-load-balancer-controller/pkg/k8s"
+	"sigs.k8s.io/aws-load-balancer-controller/test/framework/utils"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// DeploymentManager is responsible for deployment resources.
+type DeploymentManager interface {
+	WaitUntilDeploymentReady(ctx context.Context, dp *appsv1.Deployment) (*appsv1.Deployment, error)
+	WaitUntilDeploymentDeleted(ctx context.Context, dp *appsv1.Deployment) error
+}
+
+// NewDefaultDeploymentManager constructs new DeploymentManager
+func NewDefaultDeploymentManager(k8sClient client.Client, logger logr.Logger) *defaultDeploymentManager {
+	return &defaultDeploymentManager{
+		k8sClient: k8sClient,
+		logger:    logger,
+	}
+}
+
+var _ DeploymentManager = &defaultDeploymentManager{}
+
+// default implementation for DeploymentManager.
+type defaultDeploymentManager struct {
+	k8sClient client.Client
+	logger    logr.Logger
+}
+
+func (m *defaultDeploymentManager) WaitUntilDeploymentReady(ctx context.Context, dp *appsv1.Deployment) (*appsv1.Deployment, error) {
+	observedDP := &appsv1.Deployment{}
+	return observedDP, wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) {
+		if err := m.k8sClient.Get(ctx, k8s.NamespacedName(dp), observedDP); err != nil {
+			return false, err
+		}
+		if observedDP.Status.UpdatedReplicas == (*dp.Spec.Replicas) &&
+			observedDP.Status.Replicas == (*dp.Spec.Replicas) &&
+			observedDP.Status.AvailableReplicas == (*dp.Spec.Replicas) &&
+			observedDP.Status.ObservedGeneration >= dp.Generation {
+			return true, nil
+		}
+		return false, nil
+	}, ctx.Done())
+}
+
+func (m *defaultDeploymentManager) WaitUntilDeploymentDeleted(ctx context.Context, dp *appsv1.Deployment) error {
+	observedDP := &appsv1.Deployment{}
+	return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) {
+		if err := m.k8sClient.Get(ctx, k8s.NamespacedName(dp), observedDP); err != nil {
+			if apierrs.IsNotFound(err) {
+				return true, nil
+			}
+			return false, err
+		}
+		return false, nil
+	}, ctx.Done())
+}

test/framework/resources/k8s/ingresses.go

@@ -0,0 +1,62 @@
+package k8s
+
+import (
+	"context"
+	"github.com/go-logr/logr"
+	networking "k8s.io/api/networking/v1beta1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"sigs.k8s.io/aws-load-balancer-controller/pkg/k8s"
+	"sigs.k8s.io/aws-load-balancer-controller/test/framework/utils"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// IngressManager is responsible for Ingress resources.
+type IngressManager interface {
+	WaitUntilIngressReady(ctx context.Context, ing *networking.Ingress) (*networking.Ingress, error)
+	WaitUntilIngressDeleted(ctx context.Context, ing *networking.Ingress) error
+}
+
+// NewDefaultIngressManager constructs new IngressManager.
+func NewDefaultIngressManager(k8sClient client.Client, logger logr.Logger) *defaultIngressManager {
+	return &defaultIngressManager{
+		k8sClient: k8sClient,
+		logger:    logger,
+	}
+}
+
+var _ IngressManager = &defaultIngressManager{}
+
+// default implementation for IngressManager.
+type defaultIngressManager struct {
+	k8sClient client.Client
+	logger    logr.Logger
+}
+
+func (m *defaultIngressManager) WaitUntilIngressReady(ctx context.Context, ing *networking.Ingress) (*networking.Ingress, error) {
+	observedING := &networking.Ingress{}
+	return observedING, wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) {
+		if err := m.k8sClient.Get(ctx, k8s.NamespacedName(ing), observedING); err != nil {
+			return false, err
+		}
+		for _, lbIngress := range observedING.Status.LoadBalancer.Ingress {
+			if len(lbIngress.Hostname) != 0 || len(lbIngress.IP) != 0 {
+				return true, nil
+			}
+		}
+		return false, nil
+	}, ctx.Done())
+}
+
+func (m *defaultIngressManager) WaitUntilIngressDeleted(ctx context.Context, ing *networking.Ingress) error {
+	observedING := &networking.Ingress{}
+	return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) {
+		if err := m.k8sClient.Get(ctx, k8s.NamespacedName(ing), observedING); err != nil {
+			if apierrs.IsNotFound(err) {
+				return true, nil
+			}
+			return false, err
+		}
+		return false, nil
+	}, ctx.Done())
+}

test/framework/resources/k8s/namespace.go

@@ -0,0 +1,82 @@
+package k8s
+
+import (
+	"context"
+	"fmt"
+	"github.com/go-logr/logr"
+	corev1 "k8s.io/api/core/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"sigs.k8s.io/aws-load-balancer-controller/pkg/k8s"
+	"sigs.k8s.io/aws-load-balancer-controller/test/framework/utils"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// NamespaceManager is responsible for namespace resources.
+type NamespaceManager interface {
+	AllocateNamespace(ctx context.Context, baseName string) (*corev1.Namespace, error)
+	WaitUntilNamespaceDeleted(ctx context.Context, ns *corev1.Namespace) error
+}
+
+// NewDefaultNamespaceManager constructs new defaultNamespaceManager.
+func NewDefaultNamespaceManager(k8sClient client.Client, logger logr.Logger) *defaultNamespaceManager {
+	return &defaultNamespaceManager{
+		k8sClient: k8sClient,
+		logger:    logger,
+	}
+}
+
+var _ NamespaceManager = &defaultNamespaceManager{}
+
+// default implementation for NamespaceManager
+type defaultNamespaceManager struct {
+	k8sClient client.Client
+	logger    logr.Logger
+}
+
+func (m *defaultNamespaceManager) AllocateNamespace(ctx context.Context, baseName string) (*corev1.Namespace, error) {
+	name, err := m.findAvailableNamespaceName(ctx, baseName)
+	if err != nil {
+		return nil, err
+	}
+
+	ns := &corev1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+	}
+	err = m.k8sClient.Create(ctx, ns)
+	return ns, err
+}
+
+func (m *defaultNamespaceManager) WaitUntilNamespaceDeleted(ctx context.Context, ns *corev1.Namespace) error {
+	gotNS := &corev1.Namespace{}
+	return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) {
+		if err := m.k8sClient.Get(ctx, k8s.NamespacedName(ns), gotNS); err != nil {
+			if apierrs.IsNotFound(err) {
+				return true, nil
+			}
+			return false, err
+		}
+		return false, nil
+	}, ctx.Done())
+}
+
+// findAvailableNamespaceName random namespace name starting with baseName.
+func (m *defaultNamespaceManager) findAvailableNamespaceName(ctx context.Context, baseName string) (string, error) {
+	var name string
+	gotNS := &corev1.Namespace{}
+	err := wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) {
+		name = fmt.Sprintf("%v-%v", baseName, utils.RandomDNS1123Label(6))
+		if err := m.k8sClient.Get(ctx, types.NamespacedName{Name: name}, gotNS); err != nil {
+			if apierrs.IsNotFound(err) {
+				return true, nil
+			}
+			return false, err
+		}
+		return false, nil
+	}, ctx.Done())
+	return name, err
+}

test/framework/resources/k8s/service.go

@@ -0,0 +1,46 @@
+package k8s
+
+import (
+	"context"
+	"github.com/go-logr/logr"
+	corev1 "k8s.io/api/core/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"sigs.k8s.io/aws-load-balancer-controller/pkg/k8s"
+	"sigs.k8s.io/aws-load-balancer-controller/test/framework/utils"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// ServiceManager is responsible for Service resources.
+type ServiceManager interface {
+	WaitUntilServiceDeleted(ctx context.Context, svc *corev1.Service) error
+}
+
+// NewDefaultServiceManager constructs new ServiceManager.
+func NewDefaultServiceManager(k8sClient client.Client, logger logr.Logger) *defaultServiceManager {
+	return &defaultServiceManager{
+		k8sClient: k8sClient,
+		logger:    logger,
+	}
+}
+
+var _ ServiceManager = &defaultServiceManager{}
+
+// default implementation for ServiceManager.
+type defaultServiceManager struct {
+	k8sClient client.Client
+	logger    logr.Logger
+}
+
+func (m *defaultServiceManager) WaitUntilServiceDeleted(ctx context.Context, svc *corev1.Service) error {
+	observedSVC := &corev1.Service{}
+	return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) {
+		if err := m.k8sClient.Get(ctx, k8s.NamespacedName(svc), observedSVC); err != nil {
+			if apierrs.IsNotFound(err) {
+				return true, nil
+			}
+			return false, err
+		}
+		return false, nil
+	}, ctx.Done())
+}