pkg/test: add in-cluster/in-image testing support

Issue #435

Author: AlexNPavel

Gopkg.lock

@@ -15,17 +15,31 @@
 package cmd
 
 import (
+	"bytes"
 	"fmt"
+	"io/ioutil"
+	"log"
 	"os"
 	"os/exec"
 
+	"github.com/operator-framework/operator-sdk/commands/operator-sdk/cmd/cmdutil"
 	cmdError "github.com/operator-framework/operator-sdk/commands/operator-sdk/error"
+	"github.com/operator-framework/operator-sdk/pkg/generator"
 
+	"github.com/ghodss/yaml"
 	"github.com/spf13/cobra"
 )
 
+var (
+	namespacedManBuild string
+	globalManBuild     string
+	rbacManBuild       string
+	testLocationBuild  string
+	enableTests        bool
+)
+
 func NewBuildCmd() *cobra.Command {
-	return &cobra.Command{
+	buildCmd := &cobra.Command{
 		Use:   "build <image>",
 		Short: "Compiles code and builds artifacts",
 		Long: `The operator-sdk build command compiles the code, builds the executables,
@@ -42,6 +56,74 @@ For example:
 `,
 		Run: buildFunc,
 	}
+	buildCmd.Flags().BoolVarP(&enableTests, "enable-tests", "e", false, "Enable in-cluster testing by adding test binary to the image")
+	buildCmd.Flags().StringVarP(&testLocationBuild, "test-location", "t", "./test/e2e", "Location of tests")
+	buildCmd.Flags().StringVarP(&namespacedManBuild, "namespaced", "n", "", "Path of namespaced resources for tests")
+	buildCmd.Flags().StringVarP(&globalManBuild, "global", "g", "deploy/crd.yaml", "Path of global resources for tests")
+	buildCmd.Flags().StringVarP(&rbacManBuild, "rbac", "r", "deploy/rbac.yaml", "Path of global resources for tests")
+	return buildCmd
+}
+
+func parseRoles(yamlFile []byte) ([]byte, error) {
+	res := make([]byte, 0)
+	yamlSplit := bytes.Split(yamlFile, []byte("\n---\n"))
+	for _, yamlSpec := range yamlSplit {
+		yamlMap := make(map[string]interface{})
+		err := yaml.Unmarshal(yamlSpec, &yamlMap)
+		if err != nil {
+			return nil, err
+		}
+		if yamlMap["kind"].(string) == "Role" {
+			ruleBytes, err := yaml.Marshal(yamlMap["rules"])
+			if err != nil {
+				return nil, err
+			}
+			res = append(res, ruleBytes...)
+		}
+	}
+	return res, nil
+}
+
+func verifyDeploymentImage(yamlFile []byte, imageName string) string {
+	warningMessages := ""
+	yamlSplit := bytes.Split(yamlFile, []byte("\n---\n"))
+	for _, yamlSpec := range yamlSplit {
+		yamlMap := make(map[string]interface{})
+		err := yaml.Unmarshal(yamlSpec, &yamlMap)
+		if err != nil {
+			fmt.Printf("WARNING: Could not unmarshal yaml namespaced spec")
+			return ""
+		}
+		if yamlMap["kind"].(string) == "Deployment" {
+			// this is ugly and hacky; we should probably make this cleaner
+			nestedMap, ok := yamlMap["spec"].(map[string]interface{})
+			if !ok {
+				continue
+			}
+			nestedMap, ok = nestedMap["template"].(map[string]interface{})
+			if !ok {
+				continue
+			}
+			nestedMap, ok = nestedMap["spec"].(map[string]interface{})
+			if !ok {
+				continue
+			}
+			containersArray, ok := nestedMap["containers"].([]interface{})
+			if !ok {
+				continue
+			}
+			for _, item := range containersArray {
+				image, ok := item.(map[string]interface{})["image"].(string)
+				if !ok {
+					continue
+				}
+				if image != imageName {
+					warningMessages = fmt.Sprintf("%s\nWARNING: Namespace manifest contains a deployment with image %v, which does not match the name of the image being built: %v", warningMessages, image, imageName)
+				}
+			}
+		}
+	}
+	return warningMessages
 }
 
 const (
@@ -56,18 +138,73 @@ func buildFunc(cmd *cobra.Command, args []string) {
 	}
 
 	bcmd := exec.Command(build)
+	bcmd.Env = append(os.Environ(), fmt.Sprintf("TEST_LOCATION=%v", testLocationBuild))
+	bcmd.Env = append(bcmd.Env, fmt.Sprintf("ENABLE_TESTS=%v", enableTests))
 	o, err := bcmd.CombinedOutput()
 	if err != nil {
 		cmdError.ExitWithError(cmdError.ExitError, fmt.Errorf("failed to build: (%v)", string(o)))
 	}
 	fmt.Fprintln(os.Stdout, string(o))
 
+	namespacedRolesBytes := make([]byte, 0)
+	genWarning := ""
 	image := args[0]
+	if enableTests {
+		if namespacedManBuild == "" {
+			os.Mkdir("deploy/test", os.FileMode(int(0775)))
+			namespacedManBuild = "deploy/test/namespace-manifests.yaml"
+			rbac, err := ioutil.ReadFile("deploy/rbac.yaml")
+			if err != nil {
+				log.Fatalf("could not find rbac manifest: %v", err)
+			}
+			operator, err := ioutil.ReadFile("deploy/operator.yaml")
+			if err != nil {
+				log.Fatalf("could not find operator manifest: %v", err)
+			}
+			combined := append(rbac, []byte("\n---\n")...)
+			combined = append(combined, operator...)
+			err = ioutil.WriteFile(namespacedManBuild, combined, os.FileMode(int(0664)))
+			if err != nil {
+				log.Fatalf("could not create temporary namespaced manifest file: %v", err)
+			}
+			defer func() {
+				err := os.Remove(namespacedManBuild)
+				if err != nil {
+					log.Fatalf("could not delete temporary namespace manifest file")
+				}
+			}()
+		}
+		namespacedBytes, err := ioutil.ReadFile(namespacedManBuild)
+		if err != nil {
+			log.Fatalf("could not read rbac manifest: %v", err)
+		}
+		namespacedRolesBytes, err = parseRoles(namespacedBytes)
+		if err != nil {
+			log.Fatalf("could not parse namespaced manifest file for rbac roles: %v", err)
+		}
+		genWarning = verifyDeploymentImage(namespacedBytes, image)
+		global, err := ioutil.ReadFile(globalManBuild)
+		if err != nil {
+			cmdError.ExitWithError(cmdError.ExitError, fmt.Errorf("failed to read global manifest: (%v)", err))
+		}
+		c := cmdutil.GetConfig()
+		if err = generator.RenderTestYaml(c, string(global), string(namespacedRolesBytes), image); err != nil {
+			cmdError.ExitWithError(cmdError.ExitError, fmt.Errorf("failed to generate deploy/test-gen.yaml: (%v)", err))
+		}
+		os.Link("tmp/build/dockerfiles/Dockerfile_Tests", "tmp/build/Dockerfile")
+	} else {
+		os.Link("tmp/build/dockerfiles/Dockerfile_Standard", "tmp/build/Dockerfile")
+	}
+	defer os.Remove("tmp/build/Dockerfile")
 	dbcmd := exec.Command(dockerBuild)
 	dbcmd.Env = append(os.Environ(), fmt.Sprintf("IMAGE=%v", image))
+	dbcmd.Env = append(dbcmd.Env, fmt.Sprintf("NAMESPACEDMAN=%v", namespacedManBuild))
 	o, err = dbcmd.CombinedOutput()
 	if err != nil {
 		cmdError.ExitWithError(cmdError.ExitError, fmt.Errorf("failed to output build image %v: (%v)", image, string(o)))
 	}
 	fmt.Fprintln(os.Stdout, string(o))
+	if genWarning != "" {
+		fmt.Printf("%s\n", genWarning)
+	}
 }

commands/operator-sdk/cmd/build.go

@@ -39,6 +39,7 @@ const (
 	tmpDir        = "tmp"
 	buildDir      = tmpDir + "/build"
 	codegenDir    = tmpDir + "/codegen"
+	dockerDir     = buildDir + "/dockerfiles"
 	pkgDir        = "pkg"
 	apisDir       = pkgDir + "/apis"
 	stubDir       = pkgDir + "/stub"
@@ -52,7 +53,9 @@ const (
 	types              = "types.go"
 	build              = "build.sh"
 	dockerBuild        = "docker_build.sh"
-	dockerfile         = "Dockerfile"
+	standardDockerfile = "Dockerfile_Standard"
+	testingDockerfile  = "Dockerfile_Testing"
+	goTest             = "go-test.sh"
 	boilerplate        = "boilerplate.go.txt"
 	updateGenerated    = "update-generated.sh"
 	gopkgtoml          = "Gopkg.toml"
@@ -76,6 +79,7 @@ const (
 	operatorTmplName   = "deploy/operator.yaml"
 	rbacTmplName       = "deploy/rbac.yaml"
 	crTmplName         = "deploy/cr.yaml"
+	testYamlName       = "deploy/test-gen.yaml"
 	pluralSuffix       = "s"
 )
 
@@ -246,6 +250,16 @@ func renderDeployFiles(deployDir, projectName, apiVersion, kind string) error {
 	return renderWriteFile(filepath.Join(deployDir, "operator.yaml"), operatorTmplName, operatorYamlTmpl, opTd)
 }
 
+func RenderTestYaml(c *Config, globalManifest, extraRoles, image string) error {
+	opTd := tmplData{
+		GlobalManifest: globalManifest,
+		ExtraRoles:     extraRoles,
+		ProjectName:    c.ProjectName,
+		Image:          image,
+	}
+	return renderWriteFile(filepath.Join(deployDir, "test-gen.yaml"), testYamlName, testYamlTmpl, opTd)
+}
+
 // RenderOlmCatalog generates catalog manifests "deploy/olm-catalog/*"
 // The current working directory must be the project repository root
 func RenderOlmCatalog(c *Config, image, version string) error {
@@ -343,10 +357,16 @@ func renderBuildFiles(buildDir, repoPath, projectName string) error {
 	dTd := tmplData{
 		ProjectName: projectName,
 	}
-	if err := renderFile(buf, "tmp/build/Dockerfile", dockerFileTmpl, dTd); err != nil {
+
+	if err := renderWriteFile(filepath.Join(buildDir, "dockerfiles", standardDockerfile), "tmp/build/dockerfiles/Dockerfile_Standard", standardDockerFileTmpl, dTd); err != nil {
+		return err
+	}
+
+	if err := renderWriteFile(filepath.Join(buildDir, "dockerfiles", testingDockerfile), "tmp/build/dockerfiles/Dockerfile_Testing", testingDockerFileTmpl, dTd); err != nil {
 		return err
 	}
-	return renderWriteFile(filepath.Join(buildDir, dockerfile), "tmp/build/Dockerfile", dockerFileTmpl, dTd)
+
+	return renderWriteFile(filepath.Join(buildDir, goTest), "tmp/build/go-test.sh", goTestScript, tmplData{})
 }
 
 func renderDockerBuildFile(w io.Writer) error {
@@ -459,6 +479,10 @@ type tmplData struct {
 	CRDVersion     string
 	CSVName        string
 	CatalogVersion string
+
+	// global manifest used for testing
+	GlobalManifest string
+	ExtraRoles     string
 }
 
 // Creates all the necesary directories for the generated files
@@ -471,6 +495,7 @@ func (g *Generator) generateDirStructure() error {
 		filepath.Join(g.projectName, olmCatalogDir),
 		filepath.Join(g.projectName, buildDir),
 		filepath.Join(g.projectName, codegenDir),
+		filepath.Join(g.projectName, dockerDir),
 		filepath.Join(g.projectName, versionDir),
 		filepath.Join(g.projectName, apisDir, apiDirName(g.apiVersion), version(g.apiVersion)),
 		filepath.Join(g.projectName, stubDir),

pkg/generator/generator.go

@@ -424,6 +424,54 @@ spec:
   version: {{.Version}}
 `
 
+const testYamlTmpl = `{{.GlobalManifest}}
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: {{.ProjectName}}-test
+rules:
+- apiGroups:
+  - "rbac.authorization.k8s.io"
+  resources:
+  - "*"
+  verbs:
+  - "*"
+{{.ExtraRoles}}
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: default-account-{{.ProjectName}}-test
+subjects:
+- kind: ServiceAccount
+  name: default
+roleRef:
+  kind: Role
+  name: {{.ProjectName}}-test
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: {{.ProjectName}}-test
+spec:
+  restartPolicy: Never
+  containers:
+  - name: {{.ProjectName}}-test
+    image: {{.Image}}
+    imagePullPolicy: Always
+    command: ["/go-test.sh"]
+    resources:
+      requests:
+        cpu: 1
+    env:
+      - name: TEST_NAMESPACE
+        valueFrom:
+          fieldRef:
+            fieldPath: metadata.namespace
+`
+
 const operatorYamlTmpl = `apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -541,8 +589,13 @@ mkdir -p ${BIN_DIR}
 PROJECT_NAME="{{.ProjectName}}"
 REPO_PATH="{{.RepoPath}}"
 BUILD_PATH="${REPO_PATH}/cmd/${PROJECT_NAME}"
+TEST_PATH="${REPO_PATH}/${TEST_LOCATION}"
 echo "building "${PROJECT_NAME}"..."
 GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o ${BIN_DIR}/${PROJECT_NAME} $BUILD_PATH
+if $ENABLE_TESTS ; then
+	echo "building "${PROJECT_NAME}-test"..."
+	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go test -c -o ${BIN_DIR}/${PROJECT_NAME}-test $TEST_PATH
+fi
 `
 
 const dockerBuildTmpl = `#!/usr/bin/env bash
@@ -555,15 +608,38 @@ fi
 : ${IMAGE:?"Need to set IMAGE, e.g. gcr.io/<repo>/<your>-operator"}
 
 echo "building container ${IMAGE}..."
-docker build -t "${IMAGE}" -f tmp/build/Dockerfile .
+docker build -t "${IMAGE}" -f tmp/build/Dockerfile . --build-arg NAMESPACEDMAN=$NAMESPACEDMAN
 `
 
-const dockerFileTmpl = `FROM alpine:3.6
+const goTestScript = `#!/bin/sh
+
+memcached-operator-test -test.parallel=1 -test.failfast -root=/ -kubeconfig=incluster -namespacedMan=namespaced.yaml -test.v
+`
+
+const standardDockerFileTmpl = `FROM alpine:3.6
 
 RUN adduser -D {{.ProjectName}}
 USER {{.ProjectName}}
 
 ADD tmp/_output/bin/{{.ProjectName}} /usr/local/bin/{{.ProjectName}}
+
+# just keep this to ignore warnings
+ARG NAMESPACEDMAN
+`
+
+const testingDockerFileTmpl = `FROM alpine:3.6
+
+RUN adduser -D {{.ProjectName}}
+USER {{.ProjectName}}
+
+ADD tmp/_output/bin/{{.ProjectName}} /usr/local/bin/{{.ProjectName}}
+
+# just keep this to ignore warnings
+ARG NAMESPACEDMAN
+
+ADD tmp/_output/bin/{{.ProjectName}}-test /usr/local/bin/{{.ProjectName}}-test
+ADD $NAMESPACEDMAN /namespaced.yaml
+ADD tmp/build/go-test.sh /go-test.sh
 `
 
 // apiDocTmpl is the template for apis/../doc.go