Remove CSVConfig and use CSV Generator for configurable inputs and output (#2511)

Replaces config file for input and output flags 

Co-authored-by: Eric Stroczynski <[email protected]>

Author: hasbro17

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ### Added
 
+- Added the [`generate csv --deploy-dir --apis-dir --crd-dir`](doc/cli/operator-sdk_generate_csv.md#options) flags to allow configuring input locations for operator manifests and API types directories to the CSV generator in lieu of a config. See the CLI reference doc or `generate csv -h` help text for more details. ([#2511](https://github.com/operator-framework/operator-sdk/pull/2511))
+- Added the [`generate csv --output-dir`](doc/cli/operator-sdk_generate_csv.md#options) flag to allow configuring the output location for the catalog directory. ([#2511](https://github.com/operator-framework/operator-sdk/pull/2511))
 - The flag `--watch-namespace` and `--operator-namespace` was added to `operator-sdk run --local`, `operator-sdk test --local` and `operator-sdk cleanup` commands in order to replace the flag `--namespace` which was  deprecated.([#2617](https://github.com/operator-framework/operator-sdk/pull/2617))
 - The methods `ctx.GetOperatorNamespace()` and `ctx.GetWatchNamespace()` was added `pkg/test` in order to replace `ctx.GetNamespace()` which is  deprecated. ([#2617](https://github.com/operator-framework/operator-sdk/pull/2617))
 - The `--crd-version` flag was added to the `new`, `add api`, `add crd`, and `generate crds` commands so that users can opt-in to `v1` CRDs. ([#2684](https://github.com/operator-framework/operator-sdk/pull/2684))
@@ -29,6 +31,7 @@
 
 - **Breaking Change:** remove `pkg/restmapper` which was deprecated in `v0.14.0`. Projects that use this package must switch to the `DynamicRESTMapper` implementation in [controller-runtime](https://godoc.org/github.com/kubernetes-sigs/controller-runtime/pkg/client/apiutil#NewDynamicRESTMapper). ([#2544](https://github.com/operator-framework/operator-sdk/pull/2544))
 - **Breaking Change:** remove deprecated `operator-sdk generate openapi` subcommand. ([#2740](https://github.com/operator-framework/operator-sdk/pull/2740)) 
+- **Breaking Change:** Removed CSV configuration file support (defaulting to deploy/olm-catalog/csv-config.yaml) in favor of specifying inputs to the generator via [`generate csv --deploy-dir --apis-dir --crd-dir`](doc/cli/operator-sdk_generate_csv.md#options), and configuring output locations via [`generate csv --output-dir`](doc/cli/operator-sdk_generate_csv.md#options). ([#2511](https://github.com/operator-framework/operator-sdk/pull/2511))
 
 ### Bug Fixes
 

cmd/operator-sdk/bundle/create.go

@@ -22,7 +22,7 @@ import (
 	"os"
 	"path/filepath"
 
-	catalog "github.com/operator-framework/operator-sdk/internal/scaffold/olm-catalog"
+	catalog "github.com/operator-framework/operator-sdk/internal/generate/olm-catalog"
 	"github.com/operator-framework/operator-sdk/internal/util/projutil"
 
 	"github.com/operator-framework/operator-registry/pkg/lib/bundle"

cmd/operator-sdk/generate/csv.go

@@ -17,13 +17,12 @@ package generate
 import (
 	"fmt"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 
 	"github.com/operator-framework/operator-sdk/internal/generate/gen"
 	gencatalog "github.com/operator-framework/operator-sdk/internal/generate/olm-catalog"
 	"github.com/operator-framework/operator-sdk/internal/scaffold"
-	"github.com/operator-framework/operator-sdk/internal/scaffold/input"
-	catalog "github.com/operator-framework/operator-sdk/internal/scaffold/olm-catalog"
 	"github.com/operator-framework/operator-sdk/internal/util/fileutil"
 	"github.com/operator-framework/operator-sdk/internal/util/k8sutil"
 	"github.com/operator-framework/operator-sdk/internal/util/projutil"
@@ -37,8 +36,11 @@ type csvCmd struct {
 	csvVersion     string
 	csvChannel     string
 	fromVersion    string
-	csvConfigPath  string
 	operatorName   string
+	outputDir      string
+	deployDir      string
+	apisDir        string
+	crdDir         string
 	updateCRDs     bool
 	defaultChannel bool
 }
@@ -55,33 +57,133 @@ A CSV semantic version is supplied via the --csv-version flag. If your operator
 has already generated a CSV manifest you want to use as a base, supply its
 version to --from-version. Otherwise the SDK will scaffold a new CSV manifest.
 
-Configure CSV generation by writing a config file 'deploy/olm-catalog/csv-config.yaml`,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			// The CSV generator assumes that the deploy and pkg directories are
-			// present at runtime, so this command must be run in a project's root.
-			projutil.MustInProjectRoot()
+CSV input flags:
+	--deploy-dir:
+		The CSV's install strategy and permissions will be generated from the operator manifests
+		(Deployment and Role/ClusterRole) present in this directory.
+
+	--apis-dir:
+		The CSV annotation comments will be parsed from the Go types under this path to 
+		fill out metadata for owned APIs in spec.customresourcedefinitions.owned.
+
+	--crd-dir:
+		The CSV's spec.customresourcedefinitions.owned field is generated from the CRD manifests
+		in this path.These CRD manifests are also copied over to the bundle directory if --update-crds is set.
+		Additionally the CR manifests will be used to populate the CSV example CRs.
+`,
+		Example: `
+		##### Generate CSV from default input paths #####
+		$ tree pkg/apis/ deploy/
+		pkg/apis/
+		├── ...
+		└── cache
+			├── group.go
+			└── v1alpha1
+				├── ...
+				└── memcached_types.go
+		deploy/
+		├── crds
+		│   ├── cache.example.com_memcacheds_crd.yaml
+		│   └── cache.example.com_v1alpha1_memcached_cr.yaml
+		├── operator.yaml
+		├── role.yaml
+		├── role_binding.yaml
+		└── service_account.yaml
+
+		$ operator-sdk generate csv --csv-version=0.0.1 --update-crds
+		INFO[0000] Generating CSV manifest version 0.0.1
+		...
+
+		$ tree deploy/
+		deploy/
+		...
+		├── olm-catalog
+		│   └── memcached-operator
+		│       ├── 0.0.1
+		│       │   ├── cache.example.com_memcacheds_crd.yaml
+		│       │   └── memcached-operator.v0.0.1.clusterserviceversion.yaml
+		│       └── memcached-operator.package.yaml
+		...
+
+
 
+		##### Generate CSV from custom input paths #####
+		$ operator-sdk generate csv --csv-version=0.0.1 --update-crds \
+		--deploy-dir=config --apis-dir=api --output-dir=production
+		INFO[0000] Generating CSV manifest version 0.0.1
+		...
+
+		$ tree config/ api/ production/
+		config/
+		├── crds
+		│   ├── cache.example.com_memcacheds_crd.yaml
+		│   └── cache.example.com_v1alpha1_memcached_cr.yaml
+		├── operator.yaml
+		├── role.yaml
+		├── role_binding.yaml
+		└── service_account.yaml
+		api/
+		├── ...
+		└── cache
+			├── group.go
+			└── v1alpha1
+				├── ...
+				└── memcached_types.go
+		production/
+		└── olm-catalog
+			└── memcached-operator
+				├── 0.0.1
+				│   ├── cache.example.com_memcacheds_crd.yaml
+				│   └── memcached-operator.v0.0.1.clusterserviceversion.yaml
+				└── memcached-operator.package.yaml
+`,
+
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if len(args) != 0 {
 				return fmt.Errorf("command %s doesn't accept any arguments", cmd.CommandPath())
 			}
 			if err := c.validate(); err != nil {
 				return fmt.Errorf("error validating command flags: %v", err)
 			}
+
+			if err := projutil.CheckProjectRoot(); err != nil {
+				log.Warn("Could not detect project root. Ensure that this command " +
+					"runs from the project root directory.")
+			}
+
+			// Default for crd dir if unset
+			if c.crdDir == "" {
+				c.crdDir = c.deployDir
+			}
 			if err := c.run(); err != nil {
 				log.Fatal(err)
 			}
 			return nil
 		},
 	}
 
-	cmd.Flags().StringVar(&c.csvVersion, "csv-version", "", "Semantic version of the CSV")
+	cmd.Flags().StringVar(&c.csvVersion, "csv-version", "",
+		"Semantic version of the CSV")
 	if err := cmd.MarkFlagRequired("csv-version"); err != nil {
 		log.Fatalf("Failed to mark `csv-version` flag for `generate csv` subcommand as required: %v", err)
 	}
 	cmd.Flags().StringVar(&c.fromVersion, "from-version", "",
 		"Semantic version of an existing CSV to use as a base")
-	cmd.Flags().StringVar(&c.csvConfigPath, "csv-config", "",
-		"Path to CSV config file. Defaults to deploy/olm-catalog/csv-config.yaml")
+
+	// TODO: Allow multiple paths
+	// Deployment and RBAC manifests might be in different dirs e.g kubebuilder
+	cmd.Flags().StringVar(&c.deployDir, "deploy-dir", "deploy",
+		`Project relative path to root directory for operator manifests (Deployment and RBAC)`)
+	cmd.Flags().StringVar(&c.apisDir, "apis-dir", filepath.Join("pkg", "apis"),
+		`Project relative path to root directory for API type defintions`)
+	// TODO: Allow multiple paths
+	// CRD and CR manifests might be in different dirs e.g kubebuilder
+	cmd.Flags().StringVar(&c.crdDir, "crd-dir", filepath.Join("deploy", "crds"),
+		`Project relative path to root directory for CRD and CR manifests`)
+
+	cmd.Flags().StringVar(&c.outputDir, "output-dir", scaffold.DeployDir,
+		"Base directory to output generated CSV. The resulting CSV bundle directory "+
+			"will be \"<output-dir>/olm-catalog/<operator-name>/<csv-version>\".")
 	cmd.Flags().BoolVar(&c.updateCRDs, "update-crds", false,
 		"Update CRD manifests in deploy/{operator-name}/{csv-version} the using latest API's")
 	cmd.Flags().StringVar(&c.operatorName, "operator-name", "",
@@ -96,61 +198,45 @@ Configure CSV generation by writing a config file 'deploy/olm-catalog/csv-config
 }
 
 func (c csvCmd) run() error {
-
-	absProjectPath := projutil.MustGetwd()
-	cfg := &input.Config{
-		AbsProjectPath: absProjectPath,
-		ProjectName:    filepath.Base(absProjectPath),
-	}
-	if projutil.IsOperatorGo() {
-		cfg.Repo = projutil.GetGoPkg()
-	}
-
 	log.Infof("Generating CSV manifest version %s", c.csvVersion)
 
-	csvCfg, err := catalog.GetCSVConfig(c.csvConfigPath)
-	if err != nil {
-		return err
-	}
 	if c.operatorName == "" {
-		// Use config operator name if not set by CLI, i.e. prefer CLI value over
-		// config value.
-		if c.operatorName = csvCfg.OperatorName; c.operatorName == "" {
-			// Default to using project name if both are empty.
-			c.operatorName = filepath.Base(absProjectPath)
-		}
-	}
-
-	s := &scaffold.Scaffold{}
-	csv := &catalog.CSV{
-		CSVVersion:     c.csvVersion,
-		FromVersion:    c.fromVersion,
-		ConfigFilePath: c.csvConfigPath,
-		OperatorName:   c.operatorName,
+		c.operatorName = filepath.Base(projutil.MustGetwd())
 	}
-	err = s.Execute(cfg, csv)
-	if err != nil {
-		return fmt.Errorf("catalog scaffold failed: %v", err)
+	cfg := gen.Config{
+		OperatorName: c.operatorName,
+		// TODO(hasbro17): Remove the Input key map when the Generator input keys
+		// are removed in favour of config fields in the csvGenerator
+		Inputs: map[string]string{
+			gencatalog.DeployDirKey: c.deployDir,
+			gencatalog.APIsDirKey:   c.apisDir,
+			gencatalog.CRDsDirKey:   c.crdDir,
+		},
+		OutputDir: c.outputDir,
 	}
 
-	gcfg := gen.Config{
-		OperatorName: c.operatorName,
-		OutputDir:    filepath.Join(gencatalog.OLMCatalogDir, c.operatorName),
+	csv := gencatalog.NewCSV(cfg, c.csvVersion, c.fromVersion)
+	if err := csv.Generate(); err != nil {
+		return fmt.Errorf("error generating CSV: %v", err)
 	}
-	pkg := gencatalog.NewPackageManifest(gcfg, c.csvVersion, c.csvChannel, c.defaultChannel)
+	pkg := gencatalog.NewPackageManifest(cfg, c.csvVersion, c.csvChannel, c.defaultChannel)
 	if err := pkg.Generate(); err != nil {
 		return fmt.Errorf("error generating package manifest: %v", err)
 	}
 
 	// Write CRD's to the new or updated CSV package dir.
 	if c.updateCRDs {
-		input, err := csv.GetInput()
+		crdManifestSet, err := findCRDFileSet(c.crdDir)
 		if err != nil {
-			return err
+			return fmt.Errorf("failed to update CRD's: %v", err)
 		}
-		err = writeCRDsToDir(csvCfg.CRDCRPaths, filepath.Dir(input.Path))
-		if err != nil {
-			return err
+		// TODO: This path should come from the CSV generator field csvOutputDir
+		bundleDir := filepath.Join(c.outputDir, gencatalog.OLMCatalogChildDir, c.operatorName, c.csvVersion)
+		for path, b := range crdManifestSet {
+			path = filepath.Join(bundleDir, path)
+			if err = ioutil.WriteFile(path, b, fileutil.DefaultFileMode); err != nil {
+				return fmt.Errorf("failed to update CRD's: %v", err)
+			}
 		}
 	}
 
@@ -191,25 +277,45 @@ func validateVersion(version string) error {
 	return nil
 }
 
-func writeCRDsToDir(crdPaths []string, toDir string) error {
-	for _, p := range crdPaths {
-		b, err := ioutil.ReadFile(p)
+// findCRDFileSet searches files in the given directory path for CRD manifests,
+// returning a map of paths to file contents.
+func findCRDFileSet(crdDir string) (map[string][]byte, error) {
+	crdFileSet := map[string][]byte{}
+	info, err := os.Stat(crdDir)
+	if err != nil {
+		return nil, err
+	}
+	if !info.IsDir() {
+		return nil, fmt.Errorf("crd's must be read from a directory. %s is a file", crdDir)
+	}
+	files, err := ioutil.ReadDir(crdDir)
+	if err != nil {
+		return nil, err
+	}
+
+	wd := projutil.MustGetwd()
+	for _, f := range files {
+		if f.IsDir() {
+			continue
+		}
+
+		crdPath := filepath.Join(wd, crdDir, f.Name())
+		b, err := ioutil.ReadFile(crdPath)
 		if err != nil {
-			return err
+			return nil, fmt.Errorf("error reading manifest %s: %v", crdPath, err)
 		}
+		// Skip files in crdsDir that aren't k8s manifests since we do not know
+		// what other files are in crdsDir.
 		typeMeta, err := k8sutil.GetTypeMetaFromBytes(b)
 		if err != nil {
-			return fmt.Errorf("error in %s : %v", p, err)
+			log.Debugf("Skipping non-manifest file %s: %v", crdPath, err)
+			continue
 		}
 		if typeMeta.Kind != "CustomResourceDefinition" {
+			log.Debugf("Skipping non CRD manifest %s", crdPath)
 			continue
 		}
-
-		path := filepath.Join(toDir, filepath.Base(p))
-		err = ioutil.WriteFile(path, b, fileutil.DefaultFileMode)
-		if err != nil {
-			return err
-		}
+		crdFileSet[filepath.Base(crdPath)] = b
 	}
-	return nil
+	return crdFileSet, nil
 }

cmd/operator-sdk/run/cmd.go

@@ -19,8 +19,8 @@ import (
 	"fmt"
 	"path/filepath"
 
+	olmcatalog "github.com/operator-framework/operator-sdk/internal/generate/olm-catalog"
 	olmoperator "github.com/operator-framework/operator-sdk/internal/olm/operator"
-	olmcatalog "github.com/operator-framework/operator-sdk/internal/scaffold/olm-catalog"
 	k8sinternal "github.com/operator-framework/operator-sdk/internal/util/k8sutil"
 	"github.com/operator-framework/operator-sdk/internal/util/projutil"
 	aoflags "github.com/operator-framework/operator-sdk/pkg/ansible/flags"