Create CRD validation specs in CRD manifests and as Go code (#869)

* commands/.../generate/*: build and call openapi-gen, and re-scaffold CRD manifests

commands/.../add/api.go: call OpenAPI generator function

pkg/scaffold/crd*: use CustomRenderer interface to write CRD manifests
with validation spec instead of a template

pkg/scaffold/gopkgtoml*: include openapi-gen deps

pkg/scaffold/types*: add openapi-gen directives

* fix comment

* only use controller-tools CRD generator if in a Go project

* change dir in unit test so Go project is detected

* add names and only overwrite dstCrd when a file at path was generated

remove status field from resulting struct

* use generated crd if one isn't present locally

* add test case for non-Go crd generation

* internal/util/*util/*: remove pkg/scaffold imports, which cause cycles

* test/e2e/memcached_test.go: symlink pkg and internal

* pkg/scaffold/*: use a testData dir for scaffold tests instead of test/test-framework

* rename test dir to avoid import cycles during e2e test

* pkg/scaffold/crd.go: add IsOperatorGo field for controller-tools crd generator condition

* use test/test-framework code as test project

* revert removing scaffold constants from internal/util

* revert SrcDir change and check IsOperatorGo first

* change function name

* pkg/scaffold/crd.go: simplify unmarshalling crd

* use Golang initialism/acronym conventions for naming CR(D) types/variables

* Gopkg.lock: revendor

* commands/.../openapi.go: return error from CLI func instead of log.Fatal

* fix kube-openapi revision

* pkg/scaffold/gopkgtoml*.go: force sigs.k8s.io/controller-tools/pkg/crd/generator vendoring

* remove extra kube-openapi override

* pkg/scaffold/types*.go: add note on adding custom validation

* commands/.../api.go: comment on what is generated in add api command usage

* commands/.../openapi.go: remove go header file from openapi-gen

* new() -> &...{}

* correct verbosity settings for deepcopy and openapi generators

* commands/operator-sdk/cmd/add/crd.go: only skip overwriting CRD's/CR's if explicitly adding them

* verbose code generation in e2e test

* add --header-file for Go boilerplate file path

* update openapi command comment

* override go-openapi/spec to avoid new project dep solve errors

* revendor

Author: estroz

Gopkg.lock

@@ -29,6 +29,6 @@ func NewAddCmd() *cobra.Command {
 
 	upCmd.AddCommand(add.NewApiCmd())
 	upCmd.AddCommand(add.NewControllerCmd())
-	upCmd.AddCommand(add.NewAddCrdCmd())
+	upCmd.AddCommand(add.NewAddCRDCmd())
 	return upCmd
 }

commands/operator-sdk/cmd/add.go

@@ -36,8 +36,17 @@ func NewApiCmd() *cobra.Command {
 		Use:   "api",
 		Short: "Adds a new api definition under pkg/apis",
 		Long: `operator-sdk add api --kind=<kind> --api-version=<group/version> creates the
-api definition for a new custom resource under pkg/apis. This command must be run from the project root directory.
-If the api already exists at pkg/apis/<group>/<version> then the command will not overwrite and return an error.
+api definition for a new custom resource under pkg/apis. This command must be
+run from the project root directory. If the api already exists at
+pkg/apis/<group>/<version> then the command will not overwrite and return an
+error.
+
+This command runs Kubernetes deepcopy and OpenAPI V3 generators on tagged
+types in all paths under pkg/apis. Go code is generated under
+pkg/apis/<group>/<version>/zz_generated.{deepcopy,openapi}.go. CRD's are
+generated, or updated if they exist for a particular group + version + kind,
+under deploy/crds/<group>_<version>_<kind>_crd.yaml; OpenAPI V3 validation YAML
+is generated as a 'validation' object.
 
 Example:
 	$ operator-sdk add api --api-version=app.example.com/v1alpha1 --kind=AppService
@@ -49,8 +58,12 @@ Example:
 		└── v1alpha1
 			├── doc.go
 			├── register.go
-			├── types.go
-
+			├── appservice_types.go
+			├── zz_generated.deepcopy.go
+			├── zz_generated.openapi.go
+	$ tree deploy/crds
+	├── deploy/crds/app_v1alpha1_appservice_cr.yaml
+	├── deploy/crds/app_v1alpha1_appservice_crd.yaml
 `,
 		RunE: apiRun,
 	}
@@ -96,8 +109,8 @@ func apiRun(cmd *cobra.Command, args []string) error {
 		&scaffold.AddToScheme{Resource: r},
 		&scaffold.Register{Resource: r},
 		&scaffold.Doc{Resource: r},
-		&scaffold.Cr{Resource: r},
-		&scaffold.Crd{Resource: r},
+		&scaffold.CR{Resource: r},
+		&scaffold.CRD{Resource: r, IsOperatorGo: projutil.IsOperatorGo()},
 	)
 	if err != nil {
 		return fmt.Errorf("api scaffold failed: (%v)", err)
@@ -113,6 +126,11 @@ func apiRun(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	// Generate a validation spec for the new CRD.
+	if err := generate.OpenAPIGen(); err != nil {
+		return err
+	}
+
 	log.Info("API generation complete.")
 	return nil
 }

commands/operator-sdk/cmd/add/api.go

@@ -28,8 +28,8 @@ import (
 	"github.com/spf13/cobra"
 )
 
-// NewAddCrdCmd - add crd command
-func NewAddCrdCmd() *cobra.Command {
+// NewAddCRDCmd - add crd command
+func NewAddCRDCmd() *cobra.Command {
 	crdCmd := &cobra.Command{
 		Use:   "crd",
 		Short: "Adds a Custom Resource Definition (CRD) and the Custom Resource (CR) files",
@@ -78,11 +78,12 @@ func crdFunc(cmd *cobra.Command, args []string) error {
 
 	s := scaffold.Scaffold{}
 	err = s.Execute(cfg,
-		&scaffold.Crd{
-			Input:    input.Input{IfExistsAction: input.Skip},
-			Resource: resource,
+		&scaffold.CRD{
+			Input:        input.Input{IfExistsAction: input.Skip},
+			Resource:     resource,
+			IsOperatorGo: projutil.IsOperatorGo(),
 		},
-		&scaffold.Cr{
+		&scaffold.CR{
 			Input:    input.Input{IfExistsAction: input.Skip},
 			Resource: resource,
 		},

commands/operator-sdk/cmd/add/crd.go

@@ -27,5 +27,6 @@ func NewGenerateCmd() *cobra.Command {
 		Long:  `The operator-sdk generate command invokes specific generator to generate code as needed.`,
 	}
 	cmd.AddCommand(generate.NewGenerateK8SCmd())
+	cmd.AddCommand(generate.NewGenerateOpenAPICmd())
 	return cmd
 }

commands/operator-sdk/cmd/generate.go

@@ -0,0 +1,106 @@
+// Copyright 2018 The Operator-SDK Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package genutil
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/operator-framework/operator-sdk/internal/util/projutil"
+	"github.com/operator-framework/operator-sdk/pkg/scaffold"
+)
+
+func BuildCodegenBinaries(genDirs []string, binDir, codegenSrcDir string) error {
+	for _, gd := range genDirs {
+		err := runGoBuildCodegen(binDir, codegenSrcDir, gd)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func runGoBuildCodegen(binDir, repoDir, genDir string) error {
+	binPath := filepath.Join(binDir, filepath.Base(genDir))
+	cmd := exec.Command("go", "build", "-o", binPath, genDir)
+	cmd.Dir = repoDir
+	if gf, ok := os.LookupEnv(projutil.GoFlagsEnv); ok && len(gf) != 0 {
+		cmd.Env = append(os.Environ(), projutil.GoFlagsEnv+"="+gf)
+	}
+
+	if projutil.IsGoVerbose() {
+		return projutil.ExecCmd(cmd)
+	}
+	cmd.Stdout = ioutil.Discard
+	cmd.Stderr = ioutil.Discard
+	return cmd.Run()
+}
+
+// ParseGroupVersions parses the layout of pkg/apis to return a map of
+// API groups to versions.
+func ParseGroupVersions() (map[string][]string, error) {
+	gvs := make(map[string][]string)
+	groups, err := ioutil.ReadDir(scaffold.ApisDir)
+	if err != nil {
+		return nil, fmt.Errorf("could not read pkg/apis directory to find api Versions: %v", err)
+	}
+
+	for _, g := range groups {
+		if g.IsDir() {
+			groupDir := filepath.Join(scaffold.ApisDir, g.Name())
+			versions, err := ioutil.ReadDir(groupDir)
+			if err != nil {
+				return nil, fmt.Errorf("could not read %s directory to find api Versions: %v", groupDir, err)
+			}
+
+			gvs[g.Name()] = make([]string, 0)
+			for _, v := range versions {
+				if v.IsDir() && scaffold.ResourceVersionRegexp.MatchString(v.Name()) {
+					gvs[g.Name()] = append(gvs[g.Name()], v.Name())
+				}
+			}
+		}
+	}
+
+	if len(gvs) == 0 {
+		return nil, fmt.Errorf("no groups or versions found in %s", scaffold.ApisDir)
+	}
+	return gvs, nil
+}
+
+// CreateFQApis return a string of all fully qualified pkg + groups + versions
+// of pkg and gvs in the format:
+// "pkg/groupA/v1,pkg/groupA/v2,pkg/groupB/v1"
+func CreateFQApis(pkg string, gvs map[string][]string) string {
+	gn := 0
+	fqb := &strings.Builder{}
+	for g, vs := range gvs {
+		for vn, v := range vs {
+			fqb.WriteString(filepath.Join(pkg, g, v))
+			if vn < len(vs)-1 {
+				fqb.WriteString(",")
+			}
+		}
+		if gn < len(gvs)-1 {
+			fqb.WriteString(",")
+		}
+		gn++
+	}
+	return fqb.String()
+}