Use --crd-dir path to read and parse owned CRDs

Author: hasbro17

cmd/operator-sdk/generate/csv.go

@@ -58,18 +58,18 @@ has already generated a CSV manifest you want to use as a base, supply its
 version to --from-version. Otherwise the SDK will scaffold a new CSV manifest.
 
 CSV input flags:
-	--deploy-dir:	The CSV file contents will be generated from the operator manifests
-					present in this directory.
-
-	--apis-dir:		The CSV annotation comments will be parsed from the Go types under this path to 
-					fill out metadata for owned APIs in spec.customresourcedefinitions.owned.
-
-	--crd-dir:		The CRD manifests are updated from this path to the CSV bundle directory.
-					Note: The CSV generator only uses this to copy the CRD manifests.
-					The CSV contents for spec.customresourcedefinitions.owned will still be generated
-					from the CRD manifests in the deploy directory specified by --deploy-dir.
-					If unset, it defaults to the same value as --deploy-dir.
-
+	--deploy-dir:
+		The CSV's install strategy and permissions will be generated from the operator manifests
+		(Deployment and Role/ClusterRole) present in this directory.
+
+	--apis-dir:
+		The CSV annotation comments will be parsed from the Go types under this path to 
+		fill out metadata for owned APIs in spec.customresourcedefinitions.owned.
+
+	--crd-dir:
+		The CSV's spec.customresourcedefinitions.owned field is generated from the CRD manifests
+		in this path.These CRD manifests are also copied over to the bundle directory if --update-crds is set.
+		Additionally the CR manifests will be used to populate the CSV example CRs.
 `,
 		Example: `
 		##### Generate CSV from default input paths #####
@@ -170,12 +170,16 @@ CSV input flags:
 	cmd.Flags().StringVar(&c.fromVersion, "from-version", "",
 		"Semantic version of an existing CSV to use as a base")
 
+	// TODO: Allow multiple paths
+	// Deployment and RBAC manifests might be in different dirs e.g kubebuilder
 	cmd.Flags().StringVar(&c.deployDir, "deploy-dir", "deploy",
-		`Project relative path to root directory for operator manifests (Deployment, RBAC, CRDs)`)
+		`Project relative path to root directory for operator manifests (Deployment and RBAC)`)
 	cmd.Flags().StringVar(&c.apisDir, "apis-dir", filepath.Join("pkg", "apis"),
 		`Project relative path to root directory for API type defintions`)
-	cmd.Flags().StringVar(&c.crdDir, "crd-dir", "",
-		`Project relative path to root directory for for CRD manifests`)
+	// TODO: Allow multiple paths
+	// CRD and CR manifests might be in different dirs e.g kubebuilder
+	cmd.Flags().StringVar(&c.crdDir, "crd-dir", filepath.Join("deploy", "crds"),
+		`Project relative path to root directory for CRD and CR manifests`)
 
 	cmd.Flags().StringVar(&c.outputDir, "output-dir", scaffold.DeployDir,
 		"Base directory to output generated CSV. The resulting CSV bundle directory "+
@@ -206,6 +210,7 @@ func (c csvCmd) run() error {
 		Inputs: map[string]string{
 			gencatalog.DeployDirKey: c.deployDir,
 			gencatalog.APIsDirKey:   c.apisDir,
+			gencatalog.CRDsDirKey:   c.crdDir,
 		},
 		OutputDir: c.outputDir,
 	}
@@ -221,11 +226,6 @@ func (c csvCmd) run() error {
 
 	// Write CRD's to the new or updated CSV package dir.
 	if c.updateCRDs {
-		// TODO(hasbro17): Reconsider the crd-dir flag since it only lets you control
-		// where the CRD manifests are copied from but the CSV generator above
-		// will uses the CRD manifests in deploy dir to build the CSV contents for
-		// spec.customresourcedefinitions.owned
-		// Need to reconcile this disparity.
 		crdManifestSet, err := findCRDFileSet(c.crdDir)
 		if err != nil {
 			return fmt.Errorf("failed to update CRD's: %v", err)
@@ -277,28 +277,43 @@ func validateVersion(version string) error {
 	return nil
 }
 
-// findCRDFileSet searches all directories and files in path for CRD manifests,
+// findCRDFileSet searches files in the given directory path for CRD manifests,
 // returning a map of paths to file contents.
-func findCRDFileSet(path string) (map[string][]byte, error) {
+func findCRDFileSet(crdDir string) (map[string][]byte, error) {
 	crdFileSet := map[string][]byte{}
-	info, err := os.Stat(path)
+	info, err := os.Stat(crdDir)
 	if err != nil {
 		return nil, err
 	}
 	if !info.IsDir() {
-		return nil, fmt.Errorf("crd's must be read from a directory. %s is a file", path)
+		return nil, fmt.Errorf("crd's must be read from a directory. %s is a file", crdDir)
 	}
-
-	// Get CRD manifest paths from path recursively but ignore olm-catalog subdir
-	// if it is present in the search directory
-	crdPaths, err := k8sutil.GetCRDManifestPaths(path, gencatalog.OLMCatalogChildDir)
+	files, err := ioutil.ReadDir(crdDir)
 	if err != nil {
 		return nil, err
 	}
-	for _, crdPath := range crdPaths {
+
+	wd := projutil.MustGetwd()
+	for _, f := range files {
+		if f.IsDir() {
+			continue
+		}
+
+		crdPath := filepath.Join(wd, crdDir, f.Name())
 		b, err := ioutil.ReadFile(crdPath)
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("error reading manifest %s: %v", crdPath, err)
+		}
+		// Skip files in crdsDir that aren't k8s manifests since we do not know
+		// what other files are in crdsDir.
+		typeMeta, err := k8sutil.GetTypeMetaFromBytes(b)
+		if err != nil {
+			log.Debugf("Skipping non-manifest file %s: %v", crdPath, err)
+			continue
+		}
+		if typeMeta.Kind != "CustomResourceDefinition" {
+			log.Debugf("Skipping non CRD manifest %s", crdPath)
+			continue
 		}
 		crdFileSet[filepath.Base(crdPath)] = b
 	}

doc/cli/operator-sdk_generate_csv.md

@@ -12,18 +12,18 @@ has already generated a CSV manifest you want to use as a base, supply its
 version to --from-version. Otherwise the SDK will scaffold a new CSV manifest.
 
 CSV input flags:
-	--deploy-dir:	The CSV file contents will be generated from the operator manifests
-					present in this directory.
+	--deploy-dir:
+		The CSV's install strategy and permissions will be generated from the operator manifests
+		(Deployment and Role/ClusterRole) present in this directory.
 
-	--apis-dir:		The CSV annotation comments will be parsed from the Go types under this path to 
-					fill out metadata for owned APIs in spec.customresourcedefinitions.owned.
-
-	--crd-dir:		The CRD manifests are updated from this path to the CSV bundle directory.
-					Note: The CSV generator only uses this to copy the CRD manifests.
-					The CSV contents for spec.customresourcedefinitions.owned will still be generated
-					from the CRD manifests in the deploy directory specified by --deploy-dir.
-					If unset, it defaults to the same value as --deploy-dir.
+	--apis-dir:
+		The CSV annotation comments will be parsed from the Go types under this path to 
+		fill out metadata for owned APIs in spec.customresourcedefinitions.owned.
 
+	--crd-dir:
+		The CSV's spec.customresourcedefinitions.owned field is generated from the CRD manifests
+		in this path.These CRD manifests are also copied over to the bundle directory if --update-crds is set.
+		Additionally the CR manifests will be used to populate the CSV example CRs.
 
 
 ```
@@ -105,11 +105,11 @@ operator-sdk generate csv [flags]
 
 ```
       --apis-dir string        Project relative path to root directory for API type defintions (default "pkg/apis")
-      --crd-dir string         Project relative path to root directory for for CRD manifests
+      --crd-dir string         Project relative path to root directory for CRD and CR manifests (default "deploy/crds")
       --csv-channel string     Channel the CSV should be registered under in the package manifest
       --csv-version string     Semantic version of the CSV
       --default-channel        Use the channel passed to --csv-channel as the package manifests' default channel. Only valid when --csv-channel is set
-      --deploy-dir string      Project relative path to root directory for operator manifests (Deployment, RBAC, CRDs) (default "deploy")
+      --deploy-dir string      Project relative path to root directory for operator manifests (Deployment and RBAC) (default "deploy")
       --from-version string    Semantic version of an existing CSV to use as a base
   -h, --help                   help for csv
       --operator-name string   Operator name to use while generating CSV

internal/generate/olm-catalog/csv.go

@@ -28,6 +28,7 @@ import (
 	"github.com/operator-framework/operator-sdk/internal/scaffold"
 	"github.com/operator-framework/operator-sdk/internal/util/fileutil"
 	"github.com/operator-framework/operator-sdk/internal/util/k8sutil"
+	"github.com/operator-framework/operator-sdk/internal/util/projutil"
 	"github.com/operator-framework/operator-sdk/internal/util/yamlutil"
 
 	"github.com/blang/semver"
@@ -51,9 +52,17 @@ const (
 	// Input keys for CSV generator whose values are the filepaths for the respective input directories
 
 	// DeployDirKey is for the location of the operator manifests directory e.g "deploy/production"
+	// The Deployment and RBAC manifests from this directory will be used to populate the CSV
+	// install strategy: spec.install
 	DeployDirKey = "deploy"
 	// APIsDirKey is for the location of the API types directory e.g "pkg/apis"
+	// The CSV annotation comments will be parsed from the types under this path.
 	APIsDirKey = "apis"
+	// CRDsDirKey is for the location of the CRD manifests directory e.g "deploy/crds"
+	// Both the CRD and CR manifests from this path will be used to populate CSV fields
+	// metadata.annotations.alm-examples for CR examples
+	// and spec.customresourcedefinitions.owned for owned CRDs
+	CRDsDirKey = "crds"
 )
 
 type csvGenerator struct {
@@ -107,6 +116,10 @@ func NewCSV(cfg gen.Config, csvVersion, fromVersion string) gen.Generator {
 		g.Inputs[APIsDirKey] = scaffold.ApisDir
 	}
 
+	if crdsDir, ok := g.Inputs[CRDsDirKey]; !ok || crdsDir == "" {
+		g.Inputs[CRDsDirKey] = filepath.Join(g.Inputs[DeployDirKey], "crds")
+	}
+
 	return g
 }
 
@@ -366,60 +379,24 @@ func (g csvGenerator) updateCSVVersions(csv *olmapiv1alpha1.ClusterServiceVersio
 // user-defined manifests and updates csv.
 func (g csvGenerator) updateCSVFromManifests(csv *olmapiv1alpha1.ClusterServiceVersion) (err error) {
 	kindManifestMap := map[schema.GroupVersionKind][][]byte{}
-	crGVKSet := map[schema.GroupVersionKind]struct{}{}
-	err = filepath.Walk(g.Inputs[DeployDirKey], func(path string, info os.FileInfo, werr error) error {
-		if werr != nil {
-			log.Debugf("Failed to walk dir: %v", werr)
-			return werr
-		}
-		// Only read manifest from files, not directories
-		if info.IsDir() {
-			// Skip walking olm-catalog dir if it's present in the deploy directory
-			if info.Name() == OLMCatalogChildDir {
-				return filepath.SkipDir
-			}
-			return nil
-		}
 
-		b, err := ioutil.ReadFile(path)
-		if err != nil {
-			return err
-		}
-		scanner := yamlutil.NewYAMLScanner(b)
-		for scanner.Scan() {
-			manifest := scanner.Bytes()
-			typeMeta, err := k8sutil.GetTypeMetaFromBytes(manifest)
-			if err != nil {
-				log.Infof("No TypeMeta in %s, skipping file", path)
-				continue
-			}
-			gvk := typeMeta.GroupVersionKind()
-			kindManifestMap[gvk] = append(kindManifestMap[gvk], manifest)
-			switch typeMeta.Kind {
-			case "CustomResourceDefinition":
-				// Collect CRD kinds to filter them out from unsupported manifest types.
-				// The CRD version type doesn't matter as long as it has a group, kind,
-				// and versions in the expected fields.
-				crd := v1beta1.CustomResourceDefinition{}
-				if err = yaml.Unmarshal(manifest, &crd); err != nil {
-					return err
-				}
-				for _, ver := range crd.Spec.Versions {
-					crGVK := schema.GroupVersionKind{
-						Group:   crd.Spec.Group,
-						Version: ver.Name,
-						Kind:    crd.Spec.Names.Kind,
-					}
-					crGVKSet[crGVK] = struct{}{}
-				}
-			}
-		}
-		return scanner.Err()
-	})
+	// Read CRD and CR manifests from CRD dir
+	if err := updateFromManifests(g.Inputs[CRDsDirKey], kindManifestMap); err != nil {
+		return err
+	}
+
+	// Get owned CRDs from CRD manifests
+	ownedCRDs, err := getOwnedCRDs(kindManifestMap)
 	if err != nil {
-		return fmt.Errorf("failed to walk manifests directory for CSV updates: %v", err)
+		return err
+	}
+
+	// Read Deployment and RBAC manifests from Deploy dir
+	if err := updateFromManifests(g.Inputs[DeployDirKey], kindManifestMap); err != nil {
+		return err
 	}
 
+	// Update CSV from all manifest types
 	crUpdaters := crs{}
 	for gvk, manifests := range kindManifestMap {
 		// We don't necessarily care about sorting by a field value, more about
@@ -437,7 +414,8 @@ func (g csvGenerator) updateCSVFromManifests(csv *olmapiv1alpha1.ClusterServiceV
 		case "CustomResourceDefinition":
 			err = crds(manifests).apply(csv)
 		default:
-			if _, ok := crGVKSet[gvk]; ok {
+			// Only update CR examples for owned CRD types
+			if _, ok := ownedCRDs[gvk]; ok {
 				crUpdaters = append(crUpdaters, crs(manifests)...)
 			} else {
 				log.Infof("Skipping manifest %s", gvk)
@@ -462,3 +440,65 @@ func (g csvGenerator) updateCSVFromManifests(csv *olmapiv1alpha1.ClusterServiceV
 	}
 	return nil
 }
+
+func updateFromManifests(dir string, kindManifestMap map[schema.GroupVersionKind][][]byte) error {
+	files, err := ioutil.ReadDir(dir)
+	if err != nil {
+		return err
+	}
+	// Read and scan all files into kindManifestMap
+	wd := projutil.MustGetwd()
+	for _, f := range files {
+		if f.IsDir() {
+			continue
+		}
+		path := filepath.Join(wd, dir, f.Name())
+		b, err := ioutil.ReadFile(path)
+		if err != nil {
+			return err
+		}
+		scanner := yamlutil.NewYAMLScanner(b)
+		for scanner.Scan() {
+			manifest := scanner.Bytes()
+			typeMeta, err := k8sutil.GetTypeMetaFromBytes(manifest)
+			if err != nil {
+				log.Infof("No TypeMeta in %s, skipping file", path)
+				continue
+			}
+
+			gvk := typeMeta.GroupVersionKind()
+			kindManifestMap[gvk] = append(kindManifestMap[gvk], manifest)
+		}
+		if scanner.Err() != nil {
+			return scanner.Err()
+		}
+	}
+	return nil
+}
+
+func getOwnedCRDs(kindManifestMap map[schema.GroupVersionKind][][]byte) (map[schema.GroupVersionKind]struct{}, error) {
+	ownedCRDs := map[schema.GroupVersionKind]struct{}{}
+	for gvk, manifests := range kindManifestMap {
+		if gvk.Kind != "CustomResourceDefinition" {
+			continue
+		}
+		// Collect CRD kinds to filter them out from unsupported manifest types.
+		// The CRD version type doesn't matter as long as it has a group, kind,
+		// and versions in the expected fields.
+		for _, manifest := range manifests {
+			crd := v1beta1.CustomResourceDefinition{}
+			if err := yaml.Unmarshal(manifest, &crd); err != nil {
+				return ownedCRDs, err
+			}
+			for _, ver := range crd.Spec.Versions {
+				crGVK := schema.GroupVersionKind{
+					Group:   crd.Spec.Group,
+					Version: ver.Name,
+					Kind:    crd.Spec.Names.Kind,
+				}
+				ownedCRDs[crGVK] = struct{}{}
+			}
+		}
+	}
+	return ownedCRDs, nil
+}