Merge branch 'master' into refactor/controller-runtime-v0.2.0

Author: estroz

CHANGELOG.md

@@ -2,13 +2,8 @@
 
 ### Added
 
-- Document new compile-time dependency `mercurial` in user-facing documentation. ([#1683](https://github.com/operator-framework/operator-sdk/pull/1683))
-- Adds new flag `--zap-time-encoding` to the flagset provided by `pkg/log/zap`. This flag configures the timestamp format produced by the zap logger. See the [logging doc](https://github.com/operator-framework/operator-sdk/blob/master/doc/user/logging.md) for more information. ([#1529](https://github.com/operator-framework/operator-sdk/pull/1529))
-
 ### Changed
 
-- Make `ready` package idempotent. Now, a user can call `Set()` or `Unset()` to set the operator's readiness without knowing the current state. ([#1761](https://github.com/operator-framework/operator-sdk/pull/1761))
-
 ### Breaking changes
 
 - CSV config field `role-path` is now `role-paths` and takes a list of strings. Users can now specify multiple `Role` and `ClusterRole` manifests using `role-paths`. ([#1704](https://github.com/operator-framework/operator-sdk/pull/1704))
@@ -41,6 +36,21 @@
 
 ### Bug Fixes
 
+- Configure the repo path correctly in `operator-sdk add crd` and prevent the command from running outside of an operator project. ([#1660](https://github.com/operator-framework/operator-sdk/pull/1660))
+
+## v0.10.0
+
+### Added
+
+- Document new compile-time dependency `mercurial` in user-facing documentation. ([#1683](https://github.com/operator-framework/operator-sdk/pull/1683))
+- Adds new flag `--zap-time-encoding` to the flagset provided by `pkg/log/zap`. This flag configures the timestamp format produced by the zap logger. See the [logging doc](https://github.com/operator-framework/operator-sdk/blob/master/doc/user/logging.md) for more information. ([#1529](https://github.com/operator-framework/operator-sdk/pull/1529))
+
+### Changed
+
+- Make `ready` package idempotent. Now, a user can call `Set()` or `Unset()` to set the operator's readiness without knowing the current state. ([#1761](https://github.com/operator-framework/operator-sdk/pull/1761))
+
+### Bug Fixes
+
 - Check if `metadata.annotations['alm-examples']` is non-empty before creating contained CR manifests in the scorecard. ([#1789](https://github.com/operator-framework/operator-sdk/pull/1789))
 
 ## v0.9.0

cmd/operator-sdk/add/crd.go

@@ -55,8 +55,11 @@ Generated CR  filename: <project-name>/deploy/crds/<group>_<version>_<kind>_cr.y
 }
 
 func crdFunc(cmd *cobra.Command, args []string) error {
+	projutil.MustInProjectRoot()
+
 	cfg := &input.Config{
 		AbsProjectPath: projutil.MustGetwd(),
+		Repo:           projutil.GetGoPkg(),
 	}
 	if len(args) != 0 {
 		return fmt.Errorf("command %s doesn't accept any arguments", cmd.CommandPath())

cmd/operator-sdk/internal/genutil/genutil.go

@@ -15,72 +15,14 @@
 package genutil
 
 import (
-	"fmt"
 	"io/ioutil"
 	"os"
-	"path"
-	"path/filepath"
 
 	"github.com/operator-framework/operator-sdk/internal/pkg/scaffold"
 
 	log "github.com/sirupsen/logrus"
 )
 
-// ParseGroupVersions parses the layout of pkg/apis to return a map of
-// API groups to versions.
-func parseGroupVersions() (map[string][]string, error) {
-	gvs := make(map[string][]string)
-	groups, err := ioutil.ReadDir(scaffold.ApisDir)
-	if err != nil {
-		return nil, fmt.Errorf("could not read pkg/apis directory to find api Versions: %v", err)
-	}
-
-	for _, g := range groups {
-		if g.IsDir() {
-			groupDir := filepath.Join(scaffold.ApisDir, g.Name())
-			versions, err := ioutil.ReadDir(groupDir)
-			if err != nil {
-				return nil, fmt.Errorf("could not read %s directory to find api Versions: %v", groupDir, err)
-			}
-
-			gvs[g.Name()] = make([]string, 0)
-			for _, v := range versions {
-				if v.IsDir() {
-					// Ignore directories that do not contain any files, so generators
-					// do not get empty directories as arguments.
-					verDir := filepath.Join(groupDir, v.Name())
-					files, err := ioutil.ReadDir(verDir)
-					if err != nil {
-						return nil, fmt.Errorf("could not read %s directory to find api Versions: %v", verDir, err)
-					}
-					for _, f := range files {
-						if !f.IsDir() && filepath.Ext(f.Name()) == ".go" {
-							gvs[g.Name()] = append(gvs[g.Name()], filepath.ToSlash(v.Name()))
-							break
-						}
-					}
-				}
-			}
-		}
-	}
-
-	if len(gvs) == 0 {
-		return nil, fmt.Errorf("no groups or versions found in %s", scaffold.ApisDir)
-	}
-	return gvs, nil
-}
-
-// createFQAPIs return a slice of all fully qualified pkg + groups + versions
-// of pkg and gvs in the format "pkg/groupA/v1".
-func createFQAPIs(pkg string, gvs map[string][]string) (apis []string) {
-	for g, vs := range gvs {
-		for _, v := range vs {
-			apis = append(apis, path.Join(pkg, g, v))
-		}
-	}
-	return apis
-}
-
 // generateWithHeaderFile runs f with a header file path as an arguemnt.
 // If there is no project boilerplate.go.txt file, an empty header file is
 // created and its path passed as the argument.

cmd/operator-sdk/internal/genutil/k8s.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 
 	"github.com/operator-framework/operator-sdk/internal/pkg/scaffold"
+	"github.com/operator-framework/operator-sdk/internal/util/k8sutil"
 	"github.com/operator-framework/operator-sdk/internal/util/projutil"
 
 	"github.com/pkg/errors"
@@ -37,7 +38,7 @@ func K8sCodegen() error {
 
 	repoPkg := projutil.GetGoPkg()
 
-	gvMap, err := parseGroupVersions()
+	gvMap, err := k8sutil.ParseGroupSubpackages(scaffold.ApisDir)
 	if err != nil {
 		return fmt.Errorf("failed to parse group versions: (%v)", err)
 	}
@@ -49,7 +50,7 @@ func K8sCodegen() error {
 	log.Infof("Running deepcopy code-generation for Custom Resource group versions: [%v]\n", gvb.String())
 
 	apisPkg := filepath.Join(repoPkg, scaffold.ApisDir)
-	fqApis := createFQAPIs(apisPkg, gvMap)
+	fqApis := k8sutil.CreateFQAPIs(apisPkg, gvMap)
 	f := func(a string) error { return deepcopyGen(a, fqApis) }
 	if err = generateWithHeaderFile(f); err != nil {
 		return err

cmd/operator-sdk/internal/genutil/openapi.go

@@ -39,7 +39,7 @@ func OpenAPIGen() error {
 	absProjectPath := projutil.MustGetwd()
 	repoPkg := projutil.GetGoPkg()
 
-	gvMap, err := parseGroupVersions()
+	gvMap, err := k8sutil.ParseGroupSubpackages(scaffold.ApisDir)
 	if err != nil {
 		return fmt.Errorf("failed to parse group versions: (%v)", err)
 	}
@@ -51,7 +51,7 @@ func OpenAPIGen() error {
 	log.Infof("Running OpenAPI code-generation for Custom Resource group versions: [%v]\n", gvb.String())
 
 	apisPkg := filepath.Join(repoPkg, scaffold.ApisDir)
-	fqApis := createFQAPIs(apisPkg, gvMap)
+	fqApis := k8sutil.CreateFQAPIs(apisPkg, gvMap)
 	f := func(a string) error { return openAPIGen(a, fqApis) }
 	if err = generateWithHeaderFile(f); err != nil {
 		return err

cmd/operator-sdk/new/cmd.go

@@ -57,7 +57,7 @@ generates a skeletal app-operator application in $HOME/projects/example.com/app-
 	newCmd.Flags().StringVar(&kind, "kind", "", "Kubernetes CustomResourceDefintion kind. (e.g AppService) - used with \"ansible\" or \"helm\" types")
 	newCmd.Flags().StringVar(&operatorType, "type", "go", "Type of operator to initialize (choices: \"go\", \"ansible\" or \"helm\")")
 	newCmd.Flags().StringVar(&depManager, "dep-manager", "modules", `Dependency manager the new project will use (choices: "dep", "modules")`)
-	newCmd.Flags().StringVar(&repo, "repo", "", "Project repository path for Go operators. Used as the project's Go import path. This must be set if outside of $GOPATH/src with Go modules, and cannot be set if --dep-manager=dep")
+	newCmd.Flags().StringVar(&repo, "repo", "", "Project repository path for Go operators. Used as the project's Go import path. This must be set if outside of $GOPATH/src with Go modules, and cannot be set if --dep-manager=dep (e.g. github.com/example-inc/my-opertor)")
 	newCmd.Flags().BoolVar(&gitInit, "git-init", false, "Initialize the project directory as a git repository (default false)")
 	newCmd.Flags().StringVar(&headerFile, "header-file", "", "Path to file containing headers for generated Go files. Copied to hack/boilerplate.go.txt")
 	newCmd.Flags().BoolVar(&makeVendor, "vendor", false, "Use a vendor directory for dependencies. This flag only applies when --dep-manager=modules (the default)")

doc/sdk-cli-reference.md

@@ -229,7 +229,7 @@ you will need to rename it before running migrate or manually add it to your Doc
 
 * `--dep-manager` string - Dependency manager the migrated project will use (choices: "dep", "modules") (default "modules")
 * `--header-file` string - Path to file containing headers for generated Go files. Copied to hack/boilerplate.go.txt
-* `--repo` string - Project repository path for Go operators. Used as the project's Go import path. This must be set if outside of `$GOPATH/src` with Go modules, and cannot be set if `--dep-manager=dep`
+* `--repo` string - Project repository path for Go operators. Used as the project's Go import path. This must be set if outside of `$GOPATH/src` with Go modules, and cannot be set if `--dep-manager=dep` (e.g. github.com/example-inc/my-opertor)
 
 ### Example
 

doc/user/install-operator-sdk.md

@@ -18,7 +18,7 @@ $ brew install operator-sdk
 
 ```sh
 # Set the release version variable
-$ RELEASE_VERSION=v0.9.0
+$ RELEASE_VERSION=v0.10.0
 # Linux
 $ curl -OJL https://github.com/operator-framework/operator-sdk/releases/download/${RELEASE_VERSION}/operator-sdk-${RELEASE_VERSION}-x86_64-linux-gnu
 # macOS

internal/pkg/scaffold/ansible/gopkgtoml.go

@@ -39,7 +39,7 @@ const gopkgTomlTmpl = `[[constraint]]
   name = "github.com/operator-framework/operator-sdk"
   # The version rule is used for a specific release and the master branch for in between releases.
   branch = "master" #osdk_branch_annotation
-  # version = "=v0.9.0" #osdk_version_annotation
+  # version = "=v0.10.0" #osdk_version_annotation
 
 [[override]]
   name = "k8s.io/api"

internal/pkg/scaffold/gopkgtoml.go

@@ -85,7 +85,7 @@ required = [
   name = "github.com/operator-framework/operator-sdk"
   # The version rule is used for a specific release and the master branch for in between releases.
   branch = "master" #osdk_branch_annotation
-  # version = "=v0.9.0" #osdk_version_annotation
+  # version = "=v0.10.0" #osdk_version_annotation
 
 [prune]
   go-tests = true

internal/pkg/scaffold/helm/gopkgtoml.go

@@ -39,7 +39,7 @@ const gopkgTomlTmpl = `[[constraint]]
   name = "github.com/operator-framework/operator-sdk"
   # The version rule is used for a specific release and the master branch for in between releases.
   branch = "master" #osdk_branch_annotation
-  # version = "=v0.9.0" #osdk_version_annotation
+  # version = "=v0.10.0" #osdk_version_annotation
 
 [[override]]
   name = "k8s.io/api"

internal/pkg/scaffold/olm-catalog/csv.go

@@ -346,16 +346,16 @@ func (s *CSV) updateCSVFromManifestFiles(cfg *CSVConfig, csv *olmapiv1alpha1.Clu
 		scanner := yamlutil.NewYAMLScanner(yamlData)
 		for scanner.Scan() {
 			yamlSpec := scanner.Bytes()
-			typemeta, err := k8sutil.GetTypeMetaFromBytes(yamlSpec)
+			typeMeta, err := k8sutil.GetTypeMetaFromBytes(yamlSpec)
 			if err != nil {
 				return errors.Wrapf(err, "error getting type metadata from manifest %s", f)
 			}
-			found, err := store.AddToUpdater(yamlSpec, typemeta.Kind)
+			found, err := store.AddToUpdater(yamlSpec, typeMeta.Kind)
 			if err != nil {
 				return errors.Wrapf(err, "error adding manifest %s to CSV updaters", f)
 			}
 			if !found {
-				id := gvkID(typemeta.GroupVersionKind())
+				id := gvkID(typeMeta.GroupVersionKind())
 				if _, ok := otherSpecs[id]; !ok {
 					otherSpecs[id] = make([][]byte, 0)
 				}

internal/util/k8sutil/crd.go

@@ -18,13 +18,16 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
-	"strings"
+	"regexp"
 
 	yaml "github.com/ghodss/yaml"
+	"github.com/pkg/errors"
 	apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
 )
 
+// GetCRDs parses all CRD manifests in the directory crdsDir and all of its subdirectories.
 func GetCRDs(crdsDir string) ([]*apiextv1beta1.CustomResourceDefinition, error) {
 	manifests, err := GetCRDManifestPaths(crdsDir)
 	if err != nil {
@@ -45,6 +48,7 @@ func GetCRDs(crdsDir string) ([]*apiextv1beta1.CustomResourceDefinition, error)
 	return crds, nil
 }
 
+// GetCRDManifestPaths gets all CRD manifest paths in crdsDir and subdirs.
 func GetCRDManifestPaths(crdsDir string) (crdPaths []string, err error) {
 	err = filepath.Walk(crdsDir, func(path string, info os.FileInfo, werr error) error {
 		if werr != nil {
@@ -53,10 +57,98 @@ func GetCRDManifestPaths(crdsDir string) (crdPaths []string, err error) {
 		if info == nil {
 			return nil
 		}
-		if !info.IsDir() && strings.HasSuffix(path, "_crd.yaml") {
-			crdPaths = append(crdPaths, path)
+		if !info.IsDir() {
+			b, err := ioutil.ReadFile(path)
+			if err != nil {
+				return errors.Wrapf(err, "error reading manifest %s", path)
+			}
+			typeMeta, err := GetTypeMetaFromBytes(b)
+			if err != nil {
+				return errors.Wrapf(err, "error getting kind from manifest %s", path)
+			}
+			if typeMeta.Kind == "CustomResourceDefinition" {
+				crdPaths = append(crdPaths, path)
+			}
 		}
 		return nil
 	})
 	return crdPaths, err
 }
+
+// ParseGroupSubpackages parses the apisDir directory tree and returns a map of
+// all found API groups to subpackages.
+func ParseGroupSubpackages(apisDir string) (map[string][]string, error) {
+	return parseGroupSubdirs(apisDir, false)
+}
+
+// ParseGroupVersions parses the apisDir directory tree and returns a map of
+// all found API groups to versions.
+func ParseGroupVersions(apisDir string) (map[string][]string, error) {
+	return parseGroupSubdirs(apisDir, true)
+}
+
+// versionRegexp defines a kube-like version:
+// https://kubernetes.io/docs/concepts/overview/kubernetes-api/#api-versioning
+var versionRegexp = regexp.MustCompile("^v[1-9][0-9]*((alpha|beta)[1-9][0-9]*)?$")
+
+// parseGroupSubdirs searches apisDir for all groups and potential version
+// subdirs directly beneath each group dir, and returns a map of each group
+// dir name to all children version dir names. If strictVersionMatch is true,
+// all potential version dir names must strictly match versionRegexp. If
+// false, all subdir names are considered valid.
+func parseGroupSubdirs(apisDir string, strictVersionMatch bool) (map[string][]string, error) {
+	gvs := make(map[string][]string)
+	groups, err := ioutil.ReadDir(apisDir)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error reading directory %q to find API groups", apisDir)
+	}
+
+	for _, g := range groups {
+		if g.IsDir() {
+			groupDir := filepath.Join(apisDir, g.Name())
+			versions, err := ioutil.ReadDir(groupDir)
+			if err != nil {
+				return nil, errors.Wrapf(err, "error reading directory %q to find API versions", groupDir)
+			}
+
+			gvs[g.Name()] = make([]string, 0)
+			for _, v := range versions {
+				if v.IsDir() {
+					// Ignore directories that do not contain any files, so generators
+					// do not get empty directories as arguments.
+					verDir := filepath.Join(groupDir, v.Name())
+					files, err := ioutil.ReadDir(verDir)
+					if err != nil {
+						return nil, errors.Wrapf(err, "error reading directory %q to find API source files", verDir)
+					}
+					for _, f := range files {
+						if !f.IsDir() && filepath.Ext(f.Name()) == ".go" {
+							// If strictVersionMatch is true, strictly check if v.Name()
+							// is a Kubernetes API version.
+							if !strictVersionMatch || versionRegexp.MatchString(v.Name()) {
+								gvs[g.Name()] = append(gvs[g.Name()], v.Name())
+							}
+							break
+						}
+					}
+				}
+			}
+		}
+	}
+
+	if len(gvs) == 0 {
+		return nil, fmt.Errorf("no groups or versions found in %s", apisDir)
+	}
+	return gvs, nil
+}
+
+// CreateFQAPIs return a slice of all fully qualified pkg + groups + versions
+// of pkg and gvs in the format "pkg/groupA/v1".
+func CreateFQAPIs(pkg string, gvs map[string][]string) (apis []string) {
+	for g, vs := range gvs {
+		for _, v := range vs {
+			apis = append(apis, path.Join(pkg, g, v))
+		}
+	}
+	return apis
+}