Add AddToFrameworkScheme function for the test framework (#393)

* pkg/test/framework.go: add AddToFrameworkScheme func

* pkg/test/framework.go: use DeferredDiscoveryRESTMapper

* pkg/test/framework.go: check for CRD to be ready

* test/test-framework/memcached_test.go: update example

* pkg/test: remove unused cr helper functions

Author: AlexNPavel

pkg/test/context.go

@@ -20,15 +20,12 @@ import (
 	"strings"
 	"testing"
 	"time"
-
-	"k8s.io/client-go/rest"
 )
 
 type TestCtx struct {
 	ID         string
 	CleanUpFns []finalizerFn
 	Namespace  string
-	CRClient   *rest.RESTClient
 }
 
 type finalizerFn func() error

pkg/test/framework.go

@@ -15,12 +15,18 @@
 package test
 
 import (
+	goctx "context"
 	"fmt"
+	"time"
 
 	extensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	extscheme "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme"
+	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/discovery/cached"
 	"k8s.io/client-go/kubernetes"
 	cgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
@@ -34,6 +40,8 @@ type Framework struct {
 	KubeConfig       *rest.Config
 	KubeClient       kubernetes.Interface
 	ExtensionsClient *extensions.Clientset
+	Scheme           *runtime.Scheme
+	RestMapper       *discovery.DeferredDiscoveryRESTMapper
 	DynamicClient    dynclient.Client
 	DynamicDecoder   runtime.Decoder
 	CrdManPath       *string
@@ -66,6 +74,7 @@ func setup(kubeconfigPath, crdManPath, opManPath, rbacManPath *string) error {
 		KubeConfig:       kubeconfig,
 		KubeClient:       kubeclient,
 		ExtensionsClient: extensionsClient,
+		Scheme:           scheme,
 		DynamicClient:    dynClient,
 		DynamicDecoder:   dynDec,
 		CrdManPath:       crdManPath,
@@ -74,3 +83,43 @@ func setup(kubeconfigPath, crdManPath, opManPath, rbacManPath *string) error {
 	}
 	return nil
 }
+
+type addToSchemeFunc func(*runtime.Scheme) error
+
+// AddToFrameworkScheme allows users to add the scheme for their custom resources
+// to the framework's scheme for use with the dynamic client. The user provides
+// the addToScheme function (located in the register.go file of their operator
+// project) and the List struct for their custom resource. For example, for a
+// memcached operator, the list stuct may look like:
+// &MemcachedList{
+//	TypeMeta: metav1.TypeMeta{
+//		Kind: "Memcached",
+//		APIVersion: "cache.example.com/v1alpha1",
+//		},
+//	}
+// The List object is needed because the CRD has not always been fully registered
+// by the time this function is called. If the CRD takes more than 5 seconds to
+// become ready, this function throws an error
+func AddToFrameworkScheme(addToScheme addToSchemeFunc, obj runtime.Object) error {
+	err := addToScheme(Global.Scheme)
+	if err != nil {
+		return err
+	}
+	cachedDiscoveryClient := cached.NewMemCacheClient(Global.KubeClient.Discovery())
+	Global.RestMapper = discovery.NewDeferredDiscoveryRESTMapper(cachedDiscoveryClient, meta.InterfacesForUnstructured)
+	Global.RestMapper.Reset()
+	Global.DynamicClient, err = dynclient.New(Global.KubeConfig, dynclient.Options{Scheme: Global.Scheme, Mapper: Global.RestMapper})
+	err = wait.PollImmediate(time.Second, time.Second*10, func() (done bool, err error) {
+		err = Global.DynamicClient.List(goctx.TODO(), &dynclient.ListOptions{Namespace: "default"}, obj)
+		if err != nil {
+			Global.RestMapper.Reset()
+			return false, nil
+		}
+		return true, nil
+	})
+	if err != nil {
+		return fmt.Errorf("failed to build the dynamic client: %v", err)
+	}
+	Global.DynamicDecoder = serializer.NewCodecFactory(Global.Scheme).UniversalDeserializer()
+	return nil
+}

pkg/test/resource_creator.go

@@ -17,23 +17,13 @@ package test
 import (
 	"bytes"
 	goctx "context"
-	"errors"
 	"fmt"
 	"io/ioutil"
-	"strings"
 
-	y2j "github.com/ghodss/yaml"
 	yaml "gopkg.in/yaml.v2"
 	core "k8s.io/api/core/v1"
-	crd "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
-	extensions_scheme "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apimachinery/pkg/runtime/serializer"
-	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/client-go/kubernetes/scheme"
-	"k8s.io/client-go/rest"
 )
 
 func (ctx *TestCtx) GetNamespace() (string, error) {
@@ -55,119 +45,6 @@ func (ctx *TestCtx) GetNamespace() (string, error) {
 	return ctx.Namespace, nil
 }
 
-func (ctx *TestCtx) GetCRClient(yamlCR []byte) (*rest.RESTClient, error) {
-	if ctx.CRClient != nil {
-		return ctx.CRClient, nil
-	}
-	// a user may pass nil if they expect the CRClient to already exist
-	if yamlCR == nil {
-		return nil, errors.New("ctx.CRClient does not exist; yamlCR cannot be nil")
-	}
-	// get new RESTClient for custom resources
-	crConfig := Global.KubeConfig
-	yamlMap := make(map[interface{}]interface{})
-	err := yaml.Unmarshal(yamlCR, &yamlMap)
-	if err != nil {
-		return nil, err
-	}
-	groupVersion := strings.Split(yamlMap["apiVersion"].(string), "/")
-	crGV := schema.GroupVersion{Group: groupVersion[0], Version: groupVersion[1]}
-	crConfig.GroupVersion = &crGV
-	crConfig.APIPath = "/apis"
-	crConfig.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: scheme.Codecs}
-
-	if crConfig.UserAgent == "" {
-		crConfig.UserAgent = rest.DefaultKubernetesUserAgent()
-	}
-	ctx.CRClient, err = rest.RESTClientFor(crConfig)
-	return ctx.CRClient, err
-}
-
-// TODO: Implement a way for a user to add their own scheme to us the dynamic
-// client to eliminate the need for the UpdateCR function
-
-// UpdateCR takes the name of a resource, the resource plural name,
-// the path of the field that need to be updated (e.g. /spec/size),
-// and the new value to that field and patches the resource with
-// that change
-func (ctx *TestCtx) UpdateCR(name, resourceName, path, value string) error {
-	crClient, err := ctx.GetCRClient(nil)
-	if err != nil {
-		return err
-	}
-	namespace, err := ctx.GetNamespace()
-	if err != nil {
-		return err
-	}
-	return crClient.Patch(types.JSONPatchType).
-		Namespace(namespace).
-		Resource(resourceName).
-		Name(name).
-		Body([]byte("[{\"op\": \"replace\", \"path\": \"" + path + "\", \"value\": " + value + "}]")).
-		Do().
-		Error()
-}
-
-func (ctx *TestCtx) createCRFromYAML(yamlFile []byte, resourceName string) error {
-	client, err := ctx.GetCRClient(yamlFile)
-	if err != nil {
-		return err
-	}
-	namespace, err := ctx.GetNamespace()
-	if err != nil {
-		return err
-	}
-	yamlMap := make(map[interface{}]interface{})
-	err = yaml.Unmarshal(yamlFile, &yamlMap)
-	if err != nil {
-		return err
-	}
-	// TODO: handle failure of this line without segfault
-	name := yamlMap["metadata"].(map[interface{}]interface{})["name"].(string)
-	jsonDat, err := y2j.YAMLToJSON(yamlFile)
-	err = client.Post().
-		Namespace(namespace).
-		Resource(resourceName).
-		Body(jsonDat).
-		Do().
-		Error()
-	ctx.AddFinalizerFn(func() error {
-		return client.Delete().
-			Namespace(namespace).
-			Resource(resourceName).
-			Name(name).
-			Body(metav1.NewDeleteOptions(0)).
-			Do().
-			Error()
-	})
-	return err
-}
-
-func (ctx *TestCtx) createCRDFromYAML(yamlFile []byte) error {
-	decode := extensions_scheme.Codecs.UniversalDeserializer().Decode
-	obj, _, err := decode(yamlFile, nil, nil)
-	if err != nil {
-		return err
-	}
-	switch o := obj.(type) {
-	case *crd.CustomResourceDefinition:
-		_, err = Global.ExtensionsClient.ApiextensionsV1beta1().CustomResourceDefinitions().Create(o)
-		ctx.AddFinalizerFn(func() error {
-			err = Global.ExtensionsClient.ApiextensionsV1beta1().CustomResourceDefinitions().Delete(o.Name, metav1.NewDeleteOptions(0))
-			if err != nil && !apierrors.IsNotFound(err) {
-				return err
-			}
-			return nil
-		})
-		if apierrors.IsAlreadyExists(err) {
-			return nil
-		}
-		return err
-	default:
-		return errors.New("non-CRD resource in createCRDFromYAML function")
-	}
-}
-
 func setNamespaceYAML(yamlFile []byte, namespace string) ([]byte, error) {
 	yamlMap := make(map[interface{}]interface{})
 	err := yaml.Unmarshal(yamlFile, &yamlMap)
@@ -192,17 +69,7 @@ func (ctx *TestCtx) CreateFromYAML(yamlFile []byte) error {
 
 		obj, _, err := Global.DynamicDecoder.Decode(yamlSpec, nil, nil)
 		if err != nil {
-			yamlMap := make(map[interface{}]interface{})
-			err = yaml.Unmarshal(yamlSpec, &yamlMap)
-			if err != nil {
-				return err
-			}
-			kind := yamlMap["kind"].(string)
-			err = ctx.createCRFromYAML(yamlSpec, strings.ToLower(kind)+"s")
-			if err != nil {
-				return err
-			}
-			continue
+			return err
 		}
 
 		err = Global.DynamicClient.Create(goctx.TODO(), obj)

test/test-framework/memcached_test.go

@@ -15,11 +15,17 @@
 package e2e
 
 import (
+	goctx "context"
+	"fmt"
 	"testing"
 	"time"
 
+	operator "github.com/operator-framework/operator-sdk/test/test-framework/pkg/apis/cache/v1alpha1"
 	framework "github.com/operator-framework/operator-sdk/pkg/test"
 	"github.com/operator-framework/operator-sdk/pkg/util/e2eutil"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
 )
 
 var (
@@ -28,6 +34,16 @@ var (
 )
 
 func TestMemcached(t *testing.T) {
+	memcachedList := &operator.MemcachedList{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Memcached",
+			APIVersion: "cache.example.com/v1alpha1",
+		},
+	}
+	err := framework.AddToFrameworkScheme(operator.AddToScheme, memcachedList)
+	if err != nil {
+		t.Fatalf("failed to add custom resource scheme to framework: %v", err)
+	}
 	// run subtests
 	t.Run("memcached-group", func(t *testing.T) {
 		t.Run("Cluster", MemcachedCluster)
@@ -36,13 +52,25 @@ func TestMemcached(t *testing.T) {
 }
 
 func memcachedScaleTest(t *testing.T, f *framework.Framework, ctx framework.TestCtx) error {
-	// create memcached custom resource
-	crYAML := []byte("apiVersion: \"cache.example.com/v1alpha1\"\nkind: \"Memcached\"\nmetadata:\n  name: \"example-memcached\"\nspec:\n  size: 3")
-	err := ctx.CreateFromYAML(crYAML)
+	namespace, err := ctx.GetNamespace()
 	if err != nil {
-		return err
+		return fmt.Errorf("could not get namespace: %v", err)
 	}
-	namespace, err := ctx.GetNamespace()
+	// create memcached custom resource
+	exampleMemcached := &operator.Memcached{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Memcached",
+			APIVersion: "cache.example.com/v1alpha1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "example-memcached",
+			Namespace: namespace,
+		},
+		Spec: operator.MemcachedSpec{
+			Size: 3,
+		},
+	}
+	err = f.DynamicClient.Create(goctx.TODO(), exampleMemcached)
 	if err != nil {
 		return err
 	}
@@ -52,7 +80,12 @@ func memcachedScaleTest(t *testing.T, f *framework.Framework, ctx framework.Test
 		return err
 	}
 
-	err = ctx.UpdateCR("example-memcached", "memcacheds", "/spec/size", "4")
+	err = f.DynamicClient.Get(goctx.TODO(), types.NamespacedName{Name: "example-memcached", Namespace: namespace}, exampleMemcached)
+	if err != nil {
+		return err
+	}
+	exampleMemcached.Spec.Size = 4
+	err = f.DynamicClient.Update(goctx.TODO(), exampleMemcached)
 	if err != nil {
 		return err
 	}

test/test-framework/pkg/apis/cache/v1alpha1/register.go

@@ -0,0 +1,49 @@
+// Copyright 2018 The Operator-SDK Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package v1alpha1
+
+import (
+	sdkK8sutil "github.com/operator-framework/operator-sdk/pkg/util/k8sutil"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+const (
+	version   = "v1alpha1"
+	groupName = "cache.example.com"
+)
+
+var (
+	SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
+	AddToScheme   = SchemeBuilder.AddToScheme
+	// SchemeGroupVersion is the group version used to register these objects.
+	SchemeGroupVersion = schema.GroupVersion{Group: groupName, Version: version}
+)
+
+func init() {
+	sdkK8sutil.AddToSDKScheme(AddToScheme)
+}
+
+// addKnownTypes adds the set of types defined in this package to the supplied scheme.
+func addKnownTypes(scheme *runtime.Scheme) error {
+	scheme.AddKnownTypes(SchemeGroupVersion,
+		&Memcached{},
+		&MemcachedList{},
+	)
+	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
+	return nil
+}