generate csv: replace config with CSV and package manifest generators

internal/generate/olm-catalog: implement CSV and package manifest
generators, moving most of the code from internal/scaffold/olm-catalog
verbatim. Also removed CSV config

doc: remove CSVConfig documentation, update CLI docs

Co-authored-by: Eric Stroczynski <[email protected]>

Author: hasbro17

CHANGELOG.md

@@ -5,13 +5,16 @@
 - Add a new option to set the minimum log level that triggers stack trace generation in logs (`--zap-stacktrace-level`) ([#2319](https://github.com/operator-framework/operator-sdk/pull/2319))
 - Added `pkg/status` with several new types and interfaces that can be used in `Status` structs to simplify handling of [status conditions](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties). ([#1143](https://github.com/operator-framework/operator-sdk/pull/1143))
 - Added support for relative Ansible roles and playbooks paths in the Ansible operator's file. ([#2273](https://github.com/operator-framework/operator-sdk/pull/2273))
+- Added the [`generate csv --include`](doc/cli/operator-sdk_generate_csv.md#options) option to include files as input to the CSV generator in lieu of a config. ([#2249](https://github.com/operator-framework/operator-sdk/pull/2249))
 
 ### Changed
 
 ### Deprecated
 
 ### Removed
 
+- **Breaking Change:** Removed CSV configuration file support (defaulting to deploy/olm-catalog/csv-config.yaml) in favor of including files as input to the generator using [`generate csv --include`](doc/cli/operator-sdk_generate_csv.md#options), defaulting to the `deploy/` directory. ([#2249](https://github.com/operator-framework/operator-sdk/pull/2249))
+
 ### Bug Fixes
 
 - Fixed issue with Go dependencies caused by removed tag in `openshift/api` repository ([#2466](https://github.com/operator-framework/operator-sdk/issues/2466))

cmd/operator-sdk/bundle/create.go

@@ -21,7 +21,7 @@ import (
 	"path/filepath"
 	"strings"
 
-	catalog "github.com/operator-framework/operator-sdk/internal/scaffold/olm-catalog"
+	catalog "github.com/operator-framework/operator-sdk/internal/generate/olm-catalog"
 	"github.com/operator-framework/operator-sdk/internal/util/projutil"
 
 	"github.com/operator-framework/operator-registry/pkg/lib/bundle"

cmd/operator-sdk/generate/csv.go

@@ -17,13 +17,12 @@ package generate
 import (
 	"fmt"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 
 	"github.com/operator-framework/operator-sdk/internal/generate/gen"
 	gencatalog "github.com/operator-framework/operator-sdk/internal/generate/olm-catalog"
 	"github.com/operator-framework/operator-sdk/internal/scaffold"
-	"github.com/operator-framework/operator-sdk/internal/scaffold/input"
-	catalog "github.com/operator-framework/operator-sdk/internal/scaffold/olm-catalog"
 	"github.com/operator-framework/operator-sdk/internal/util/fileutil"
 	"github.com/operator-framework/operator-sdk/internal/util/k8sutil"
 	"github.com/operator-framework/operator-sdk/internal/util/projutil"
@@ -37,8 +36,9 @@ type csvCmd struct {
 	csvVersion     string
 	csvChannel     string
 	fromVersion    string
-	csvConfigPath  string
 	operatorName   string
+	outputDir      string
+	includePaths   []string
 	updateCRDs     bool
 	defaultChannel bool
 }
@@ -53,9 +53,8 @@ for the operator. This file is used to publish the operator to the OLM Catalog.
 
 A CSV semantic version is supplied via the --csv-version flag. If your operator
 has already generated a CSV manifest you want to use as a base, supply its
-version to --from-version. Otherwise the SDK will scaffold a new CSV manifest.
+version to --from-version. Otherwise the SDK will scaffold a new CSV manifest.`,
 
-Configure CSV generation by writing a config file 'deploy/olm-catalog/csv-config.yaml`,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			// The CSV generator assumes that the deploy and pkg directories are
 			// present at runtime, so this command must be run in a project's root.
@@ -74,14 +73,20 @@ Configure CSV generation by writing a config file 'deploy/olm-catalog/csv-config
 		},
 	}
 
-	cmd.Flags().StringVar(&c.csvVersion, "csv-version", "", "Semantic version of the CSV")
+	cmd.Flags().StringVar(&c.csvVersion, "csv-version", "",
+		"Semantic version of the CSV")
 	if err := cmd.MarkFlagRequired("csv-version"); err != nil {
 		log.Fatalf("Failed to mark `csv-version` flag for `generate csv` subcommand as required: %v", err)
 	}
 	cmd.Flags().StringVar(&c.fromVersion, "from-version", "",
 		"Semantic version of an existing CSV to use as a base")
-	cmd.Flags().StringVar(&c.csvConfigPath, "csv-config", "",
-		"Path to CSV config file. Defaults to deploy/olm-catalog/csv-config.yaml")
+	cmd.Flags().StringSliceVar(&c.includePaths, "include", []string{scaffold.DeployDir},
+		"Paths to include in CSV generation, ex. \"deploy/prod,deploy/test\". "+
+			"If this flag is set and you want to enable default behavior, "+
+			"you must include \"deploy/\" in the argument list")
+	cmd.Flags().StringVar(&c.outputDir, "output-dir", scaffold.DeployDir,
+		"Base directory to output generated CSV. The resulting CSV bundle directory"+
+			"will be \"<output-dir>/olm-catalog/<operator-name>/<csv-version>\"")
 	cmd.Flags().BoolVar(&c.updateCRDs, "update-crds", false,
 		"Update CRD manifests in deploy/{operator-name}/{csv-version} the using latest API's")
 	cmd.Flags().StringVar(&c.operatorName, "operator-name", "",
@@ -97,60 +102,42 @@ Configure CSV generation by writing a config file 'deploy/olm-catalog/csv-config
 
 func (c csvCmd) run() error {
 
-	absProjectPath := projutil.MustGetwd()
-	cfg := &input.Config{
-		AbsProjectPath: absProjectPath,
-		ProjectName:    filepath.Base(absProjectPath),
-	}
-	if projutil.IsOperatorGo() {
-		cfg.Repo = projutil.GetGoPkg()
-	}
-
 	log.Infof("Generating CSV manifest version %s", c.csvVersion)
 
-	csvCfg, err := catalog.GetCSVConfig(c.csvConfigPath)
-	if err != nil {
-		return err
-	}
 	if c.operatorName == "" {
-		// Use config operator name if not set by CLI, i.e. prefer CLI value over
-		// config value.
-		if c.operatorName = csvCfg.OperatorName; c.operatorName == "" {
-			// Default to using project name if both are empty.
-			c.operatorName = filepath.Base(absProjectPath)
-		}
+		c.operatorName = filepath.Base(projutil.MustGetwd())
 	}
-
-	s := &scaffold.Scaffold{}
-	csv := &catalog.CSV{
-		CSVVersion:     c.csvVersion,
-		FromVersion:    c.fromVersion,
-		ConfigFilePath: c.csvConfigPath,
-		OperatorName:   c.operatorName,
-	}
-	err = s.Execute(cfg, csv)
-	if err != nil {
-		return fmt.Errorf("catalog scaffold failed: %v", err)
+	cfg := gen.Config{
+		OperatorName: c.operatorName,
+		OutputDir:    c.outputDir,
+		Filters:      gen.MakeFilters(c.includePaths...),
 	}
 
-	gcfg := gen.Config{
-		OperatorName: c.operatorName,
-		OutputDir:    filepath.Join(gencatalog.OLMCatalogDir, c.operatorName),
+	csv := gencatalog.NewCSV(cfg, c.csvVersion, c.fromVersion)
+	if err := csv.Generate(); err != nil {
+		return fmt.Errorf("error generating CSV: %v", err)
 	}
-	pkg := gencatalog.NewPackageManifest(gcfg, c.csvVersion, c.csvChannel, c.defaultChannel)
+	pkg := gencatalog.NewPackageManifest(cfg, c.csvVersion, c.csvChannel, c.defaultChannel)
 	if err := pkg.Generate(); err != nil {
 		return fmt.Errorf("error generating package manifest: %v", err)
 	}
 
 	// Write CRD's to the new or updated CSV package dir.
 	if c.updateCRDs {
-		input, err := csv.GetInput()
+		crdManifestSet, err := findCRDs(c.includePaths...)
 		if err != nil {
 			return err
 		}
-		err = writeCRDsToDir(csvCfg.CRDCRPaths, filepath.Dir(input.Path))
-		if err != nil {
-			return err
+		baseDir := c.outputDir
+		if baseDir == "" {
+			baseDir = gencatalog.OLMCatalogDir
+		}
+		bundleDir := filepath.Join(baseDir, c.operatorName, c.csvVersion)
+		for path, b := range crdManifestSet {
+			path = filepath.Join(bundleDir, path)
+			if err = ioutil.WriteFile(path, b, fileutil.DefaultFileMode); err != nil {
+				return err
+			}
 		}
 	}
 
@@ -191,25 +178,41 @@ func validateVersion(version string) error {
 	return nil
 }
 
-func writeCRDsToDir(crdPaths []string, toDir string) error {
-	for _, p := range crdPaths {
-		b, err := ioutil.ReadFile(p)
-		if err != nil {
-			return err
-		}
-		typeMeta, err := k8sutil.GetTypeMetaFromBytes(b)
+// findCRDs searches directories and files in paths for CRD manifest paths,
+// returning a map of paths to file contents.
+func findCRDs(paths ...string) (map[string][]byte, error) {
+	crdFileSet := map[string][]byte{}
+	for _, path := range paths {
+		info, err := os.Stat(path)
 		if err != nil {
-			return err
-		}
-		if typeMeta.Kind != "CustomResourceDefinition" {
-			continue
+			return nil, err
 		}
-
-		path := filepath.Join(toDir, filepath.Base(p))
-		err = ioutil.WriteFile(path, b, fileutil.DefaultFileMode)
-		if err != nil {
-			return err
+		if info.IsDir() {
+			subsetPaths, err := k8sutil.GetCRDManifestPaths(path)
+			if err != nil {
+				return nil, err
+			}
+			for _, crdPath := range subsetPaths {
+				b, err := ioutil.ReadFile(crdPath)
+				if err != nil {
+					return nil, err
+				}
+				crdFileSet[filepath.Base(crdPath)] = b
+			}
+		} else {
+			b, err := ioutil.ReadFile(path)
+			if err != nil {
+				return nil, err
+			}
+			typeMeta, err := k8sutil.GetTypeMetaFromBytes(b)
+			if err != nil {
+				log.Infof("Skipping non-manifest file %s", path)
+				continue
+			}
+			if typeMeta.Kind == "CustomResourceDefinition" {
+				crdFileSet[filepath.Base(path)] = b
+			}
 		}
 	}
-	return nil
+	return crdFileSet, nil
 }

cmd/operator-sdk/run/cmd.go

@@ -19,8 +19,8 @@ import (
 	"fmt"
 	"path/filepath"
 
+	olmcatalog "github.com/operator-framework/operator-sdk/internal/generate/olm-catalog"
 	olmoperator "github.com/operator-framework/operator-sdk/internal/olm/operator"
-	olmcatalog "github.com/operator-framework/operator-sdk/internal/scaffold/olm-catalog"
 	k8sinternal "github.com/operator-framework/operator-sdk/internal/util/k8sutil"
 	"github.com/operator-framework/operator-sdk/internal/util/projutil"
 	aoflags "github.com/operator-framework/operator-sdk/pkg/ansible/flags"

doc/cli/operator-sdk_generate_csv.md

@@ -11,8 +11,6 @@ A CSV semantic version is supplied via the --csv-version flag. If your operator
 has already generated a CSV manifest you want to use as a base, supply its
 version to --from-version. Otherwise the SDK will scaffold a new CSV manifest.
 
-Configure CSV generation by writing a config file 'deploy/olm-catalog/csv-config.yaml
-
 ```
 operator-sdk generate csv [flags]
 ```
@@ -21,12 +19,13 @@ operator-sdk generate csv [flags]
 
 ```
       --csv-channel string     Channel the CSV should be registered under in the package manifest
-      --csv-config string      Path to CSV config file. Defaults to deploy/olm-catalog/csv-config.yaml
       --csv-version string     Semantic version of the CSV
       --default-channel        Use the channel passed to --csv-channel as the package manifests' default channel. Only valid when --csv-channel is set
       --from-version string    Semantic version of an existing CSV to use as a base
   -h, --help                   help for csv
+      --include strings        Paths to include in CSV generation, ex. "deploy/prod,deploy/test". If this flag is set and you want to enable default behavior, you must include "deploy/" in the argument list (default [deploy])
       --operator-name string   Operator name to use while generating CSV
+      --output-dir string      Base directory to output generated CSV. The resulting CSV bundle directory will be "<output-dir>/olm-catalog/<operator-name>/<csv-version>" (default "deploy")
       --update-crds            Update CRD manifests in deploy/{operator-name}/{csv-version} the using latest API's
 ```
 

doc/user/olm-catalog/generating-a-csv.md

@@ -13,40 +13,20 @@ This document describes how to manage the following lifecycle for your Operator
 Operator SDK projects have an expected [project layout][doc-project-layout]. In particular, a few manifests are expected to be present in the `deploy` directory:
 
 * Roles: `role.yaml`
+* ClusterRoles: `cluster_role.yaml`
 * Deployments: `operator.yaml`
 * Custom Resources (CR's): `crds/<full group>_<version>_<kind>_cr.yaml`
-* Custom Resource Definitions (CRD's): `crds/<full group>_<resource>_crd.yaml`.
+* CustomResourceDefinitions (CRD's): `crds/<full group>_<resource>_crd.yaml`.
 
-`generate csv` reads these files and adds their data to a CSV in an alternate form.
-
-The following example config containing default values should be copied and written to `deploy/olm-catalog/csv-config.yaml`:
-
-```yaml
-crd-cr-paths:
-- deploy/crds
-operator-path: deploy/operator.yaml
-role-paths:
-- deploy/role.yaml
-```
-
-Explanation of all config fields:
-
-- `crd-cr-paths`: list of strings - a list of CRD and CR manifest file/directory paths. Defaults to `[deploy/crds]`.
-- `operator-path`: string - the operator `Deployment` manifest file path. Defaults to `deploy/operator.yaml`.
-- `role-paths`: list of strings - Role and ClusterRole manifest file paths. Defaults to `[deploy/role.yaml]`.
-- `operator-name`: string - the name used to create the CSV and manifest file names. Defaults to the project's name.
-
-**Note**: The [design doc][doc-csv-design] has outdated field information which should not be referenced.
-
-Fields in this config file can be modified to point towards alternate manifest locations, and passed to `generate csv --csv-config=<path>` to configure CSV generation. For example, if I have one set of production CR/CRD manifests under `deploy/crds/production`, and a set of test manifests under `deploy/crds/test`, and I only want to include production manifests in my CSV, I can set `crd-cr-paths: [deploy/crds/production]`. `generate csv` will then ignore `deploy/crds/test` when getting CR/CRD data.
+`generate csv` extracts manifests from files in `deploy/` by default that match the kinds above and adds them to the CSV. If your manifest files are not in `deploy/`, you can use the `--include=[list of paths]` option to instruct the command to extract manifests from files at those paths, ex. `--include="deploy/prod,deploy/test"`. Setting `--include` overrides default behavior; if you still want default behavior, you must append `deploy/` to the list of paths passed to `--include`.
 
 ## Versioning
 
 CSV's are versioned in path, file name, and in their `metadata.name` field. For example, running `operator-sdk generate csv --csv-version 0.0.1` will generate a CSV at `deploy/olm-catalog/<operator-name>/0.0.1/<operator-name>.v0.0.1.clusterserviceversion.yaml`. A versioned directory such as `deploy/olm-catalog/<operator-name>/0.0.1` is known as a [*bundle*][doc-bundle]. Versions allow the OLM to upgrade or downgrade your Operator at runtime, i.e. in a cluster. A valid semantic version is required.
 
 `generate csv` allows you to upgrade your CSV using the `--from-version` flag. If you have an existing CSV with version `0.0.1` and want to write a new version `0.0.2`, you can run `operator-sdk generate csv --csv-version 0.0.2 --from-version 0.0.1`. This will write a new CSV manifest to `deploy/olm-catalog/<operator-name>/0.0.2/<operator-name>.v0.0.2.clusterserviceversion.yaml` containing user-defined data from `0.0.1` and any modifications you've made to `roles.yaml`, `operator.yaml`, CR's, or CRD's.
 
-The SDK can manage CRD's in your Operator bundle as well. You can pass the `--update-crds` flag to `generate csv` to add or update your CRD's in your bundle by copying manifests pointed to by `crd-cr-paths` in your config. CRD's in a bundle are not updated by default.
+The SDK can manage CRD's in your Operator bundle as well. You can pass the `--update-crds` flag to `generate csv` to add or update your CRD's in your bundle by copying manifests in `deploy/crds` to your bundle. CRD's in a bundle are not updated by default.
 
 ## First Generation
 

internal/generate/crd/crd_test.go

@@ -151,7 +151,7 @@ func TestCRDNonGo(t *testing.T) {
 			}
 			cfg := gen.Config{
 				Inputs: map[string]string{
-					CRDsDirKey: c.crdsDir,
+					CRDsDirKey: c.crdsDir
 				},
 			}
 			g := NewCRDNonGo(cfg, *r)