add scorecard bundle validation feature (#2326)

* add scorecard bundle validation feature

* tweaks from PR reviews

* Update doc/test-framework/scorecard.md

Co-Authored-By: Camila Macedo <[email protected]>

Author: Jeff McCormick

CHANGELOG.md

@@ -3,6 +3,9 @@
 
 ### Added
 
+- Added new `--bundle` flag to the `operator-sdk scorecard` command to support bundle validation testing using the validation API (https://github.com/operator-framework/api). ([#1916](https://github.com/operator-framework/operator-sdk/pull/1916)
+- Added new `log` field to the `operator-sdk scorecard` v1alpha2 output to support tests that produce logging. ([#1916](https://github.com/operator-framework/operator-sdk/pull/1916)
+- Added new `bundle validation` test to the `operator-sdk scorecard` OLM tests. ([#1916](https://github.com/operator-framework/operator-sdk/pull/1916)
 
 ### Changed
 

cmd/operator-sdk/scorecard/cmd.go

@@ -41,6 +41,7 @@ func NewCmd() *cobra.Command {
 	scorecardCmd.Flags().String(schelpers.VersionOpt, schelpers.DefaultScorecardVersion, "scorecard version. Valid values: v1alpha1, v1alpha2")
 	scorecardCmd.Flags().StringP(scorecard.SelectorOpt, "l", "", "selector (label query) to filter tests on (only valid when version is v1alpha2)")
 	scorecardCmd.Flags().BoolP(scorecard.ListOpt, "L", false, "If true, only print the test names that would be run based on selector filtering (only valid when version is v1alpha2)")
+	scorecardCmd.Flags().StringP(scorecard.BundleOpt, "b", "", "OLM bundle directory path, when specified runs bundle validation")
 
 	// TODO: make config file global and make this a top level flag
 	if err := viper.BindPFlag(scorecard.ConfigOpt, scorecardCmd.Flags().Lookup(scorecard.ConfigOpt)); err != nil {
@@ -61,5 +62,8 @@ func NewCmd() *cobra.Command {
 	if err := viper.BindPFlag("scorecard."+scorecard.ListOpt, scorecardCmd.Flags().Lookup(scorecard.ListOpt)); err != nil {
 		log.Fatalf("Unable to add list :%v", err)
 	}
+	if err := viper.BindPFlag("scorecard."+scorecard.BundleOpt, scorecardCmd.Flags().Lookup(scorecard.BundleOpt)); err != nil {
+		log.Fatalf("Unable to add bundle :%v", err)
+	}
 	return scorecardCmd
 }

doc/cli/operator-sdk_scorecard.md

@@ -14,6 +14,7 @@ operator-sdk scorecard [flags]
 ### Options
 
 ```
+  -b, --bundle string       OLM bundle directory path, when specified runs bundle validation
       --config string       config file (default is '<project_dir>/.osdk-scorecard'; the config file's extension and format can be .yaml, .json, or .toml)
   -h, --help                help for scorecard
       --kubeconfig string   Path to kubeconfig of custom resource created in cluster