scorecard - Add list flag (#2137)

* add initial list flag logic to scorecard

* fix the list flag - v1alpha1 check

* refactor text output and list flag logic

* clean up code, add quotes around labels

* slight refactor of applying selectors when listing

* add test for label and selector flags for scorecard

* change test selector to use suite=basic

Author: Jeff McCormick

CHANGELOG.md

@@ -52,6 +52,7 @@
 - Added support for `ppc64le-linux` for the `operator-sdk` binary and the Helm operator base image. ([#1533](https://github.com/operator-framework/operator-sdk/pull/1533))
 - Added new `--version` flag to the `operator-sdk scorecard` command to support a new output format for the scorecard. ([#1916](https://github.com/operator-framework/operator-sdk/pull/1916)
 - Added new `--selector` flag to the `operator-sdk scorecard` command to support filtering scorecard tests based on labels added to each test. ([#1916](https://github.com/operator-framework/operator-sdk/pull/1916)
+- Added new `--list` flag to the `operator-sdk scorecard` command to support listing scorecard tests that would be executed based on selector filters. ([#1916](https://github.com/operator-framework/operator-sdk/pull/1916)
 
 ### Changed
 

cmd/operator-sdk/scorecard/cmd.go

@@ -39,6 +39,7 @@ func NewCmd() *cobra.Command {
 	scorecardCmd.Flags().StringP(scorecard.OutputFormatOpt, "o", scorecard.TextOutputFormat, fmt.Sprintf("Output format for results. Valid values: %s, %s", scorecard.TextOutputFormat, scorecard.JSONOutputFormat))
 	scorecardCmd.Flags().String(schelpers.VersionOpt, schelpers.DefaultScorecardVersion, "scorecard version. Valid values: v1alpha1, v1alpha2")
 	scorecardCmd.Flags().StringP(scorecard.SelectorOpt, "l", "", "selector (label query) to filter tests on (only valid when version is v1alpha2)")
+	scorecardCmd.Flags().BoolP(scorecard.ListOpt, "L", false, "If true, only print the test names that would be run based on selector filtering (only valid when version is v1alpha2)")
 
 	// TODO: make config file global and make this a top level flag
 	viper.BindPFlag(scorecard.ConfigOpt, scorecardCmd.Flags().Lookup(scorecard.ConfigOpt))
@@ -47,6 +48,7 @@ func NewCmd() *cobra.Command {
 	viper.BindPFlag("scorecard."+scorecard.OutputFormatOpt, scorecardCmd.Flags().Lookup(scorecard.OutputFormatOpt))
 	viper.BindPFlag("scorecard."+schelpers.VersionOpt, scorecardCmd.Flags().Lookup(schelpers.VersionOpt))
 	viper.BindPFlag("scorecard."+scorecard.SelectorOpt, scorecardCmd.Flags().Lookup(scorecard.SelectorOpt))
+	viper.BindPFlag("scorecard."+scorecard.ListOpt, scorecardCmd.Flags().Lookup(scorecard.ListOpt))
 
 	return scorecardCmd
 }

doc/test-framework/scorecard.md

@@ -108,6 +108,7 @@ While most configuration is done via a config file, there are a few important ar
 | `--kubeconfig`, `-o`  | string |  path to kubeconfig. It sets the kubeconfig internally for internal plugins and sets the `KUBECONFIG` env var to the provided value for external plugins. If an external plugin specifically sets the `KUBECONFIG` env var, the kubeconfig from the specified env var will be used for that plugin instead. |
 | `--version`  | string |  The version of scorecard to run, v1alpha2 is the default, valid values are v1alpha and v1alpha2. |
 | `--selector`, `-l`  | string |  The label selector to filter tests on, only valid in version v1alpha2. |
+| `--list`, `-L`  | bool |  If true, only print the test names that would be run based on selector filtering, only valid in version v1alpha2. |
 
 ### Config File Options
 

hack/tests/subcommand-scorecard.sh

@@ -20,14 +20,34 @@ operator-sdk scorecard --version v1alpha2 --config "$CONFIG_PATH" |& grep '^.*er
 
 # test to see if v1alpha2 is used from the command line
 commandoutput="$(operator-sdk scorecard --version v1alpha2 --config "$CONFIG_PATH_V1ALPHA2" 2>&1)"
-failCount=`echo $commandoutput | grep -o ": fail" | wc -l`
+failCount=`echo $commandoutput | grep -o "fail" | wc -l`
 expectedFailCount=3
 if [ $failCount -ne $expectedFailCount ]
 then
 	echo "expected fail count $expectedFailCount, got $failCount"
 	exit 1
 fi
 
+# test to see if list flag work
+commandoutput="$(operator-sdk scorecard --version v1alpha2 --list --selector=suite=basic --config "$CONFIG_PATH_V1ALPHA2" 2>&1)"
+labelCount=`echo $commandoutput | grep -o "Label" | wc -l`
+expectedLabelCount=3
+if [ $labelCount -ne $expectedLabelCount ]
+then
+	echo "expected label count $expectedLabelCount, got $labelCount"
+	exit 1
+fi
+
+# test to see if selector flags work
+commandoutput="$(operator-sdk scorecard --version v1alpha2 --selector=suite=basic --config "$CONFIG_PATH_V1ALPHA2" 2>&1)"
+labelCount=`echo $commandoutput | grep -o "Label" | wc -l`
+expectedLabelCount=3
+if [ $labelCount -ne $expectedLabelCount ]
+then
+	echo "expected label count $expectedLabelCount, got $labelCount"
+	exit 1
+fi
+
 # test to see if version in config file allows v1alpha1 to be specified
 commandoutput="$(operator-sdk scorecard --config "$CONFIG_PATH_V1ALPHA1" 2>&1)"
 echo $commandoutput | grep "Total Score: 67%"

internal/scorecard/interfaces.go

@@ -30,6 +30,7 @@ import (
 )
 
 type Plugin interface {
+	List() scapiv1alpha1.ScorecardOutput
 	Run() scapiv1alpha1.ScorecardOutput
 }
 
@@ -42,6 +43,10 @@ type basicOrOLMPlugin struct {
 	config     scplugins.BasicAndOLMPluginConfig
 }
 
+func (p externalPlugin) List() scapiv1alpha1.ScorecardOutput {
+	return scapiv1alpha1.ScorecardOutput{}
+}
+
 func (p externalPlugin) Run() scapiv1alpha1.ScorecardOutput {
 	cmd := exec.Command(p.config.Command, p.config.Args...)
 	for _, env := range p.config.Env {
@@ -84,6 +89,15 @@ var olmTestsPlugin = basicOrOLMPlugin{
 	pluginType: scplugins.OLMIntegration,
 }
 
+func (p basicOrOLMPlugin) List() scapiv1alpha1.ScorecardOutput {
+	res, err := scplugins.ListInternalPlugin(p.pluginType, p.config)
+	if err != nil {
+		log.Errorf("%v", err)
+		return scapiv1alpha1.ScorecardOutput{}
+	}
+	return res
+}
+
 func (p basicOrOLMPlugin) Run() scapiv1alpha1.ScorecardOutput {
 	pluginLogs := &bytes.Buffer{}
 	res, err := scplugins.RunInternalPlugin(p.pluginType, p.config, pluginLogs)

internal/scorecard/plugins/config.go

@@ -54,6 +54,7 @@ type BasicAndOLMPluginConfig struct {
 	DeployDir          string          `mapstructure:"deploy-dir"`
 	Selector           labels.Selector `mapstructure:"selector"`
 	Version            string          `mapstructure:"version"`
+	ListOpt            bool            `mapstructure:"list"`
 }
 
 func validateScorecardPluginFlags(config BasicAndOLMPluginConfig, pluginType PluginType) error {

internal/scorecard/plugins/plugin_runner.go

@@ -38,6 +38,7 @@ import (
 	"github.com/sirupsen/logrus"
 	v1 "k8s.io/api/core/v1"
 	extscheme "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
@@ -267,6 +268,7 @@ func RunInternalPlugin(pluginType PluginType, config BasicAndOLMPluginConfig, lo
 		logReadWriter := &bytes.Buffer{}
 		log.SetOutput(logReadWriter)
 		log.Printf("Running for cr: %s", cr)
+		var obj *unstructured.Unstructured
 		if !config.OLMDeployed {
 			if err := createFromYAMLFile(config.Namespace, config.GlobalManifest, config.ProxyImage, config.ProxyPullPolicy); err != nil {
 				return scapiv1alpha1.ScorecardOutput{}, fmt.Errorf("failed to create global resources: %v", err)
@@ -278,7 +280,7 @@ func RunInternalPlugin(pluginType PluginType, config BasicAndOLMPluginConfig, lo
 		if err := createFromYAMLFile(config.Namespace, cr, config.ProxyImage, config.ProxyPullPolicy); err != nil {
 			return scapiv1alpha1.ScorecardOutput{}, fmt.Errorf("failed to create cr resource: %v", err)
 		}
-		obj, err := yamlToUnstructured(config.Namespace, cr)
+		obj, err = yamlToUnstructured(config.Namespace, cr)
 		if err != nil {
 			return scapiv1alpha1.ScorecardOutput{}, fmt.Errorf("failed to decode custom resource manifest into object: %s", err)
 		}
@@ -295,6 +297,7 @@ func RunInternalPlugin(pluginType PluginType, config BasicAndOLMPluginConfig, lo
 			if schelpers.IsV1alpha2(config.Version) {
 				basicTests.ApplySelector(config.Selector)
 			}
+
 			basicTests.Run(context.TODO())
 			logs, err := ioutil.ReadAll(logReadWriter)
 			if err != nil {
@@ -316,6 +319,7 @@ func RunInternalPlugin(pluginType PluginType, config BasicAndOLMPluginConfig, lo
 			if schelpers.IsV1alpha2(config.Version) {
 				olmTests.ApplySelector(config.Selector)
 			}
+
 			olmTests.Run(context.TODO())
 			logs, err := ioutil.ReadAll(logReadWriter)
 			if err != nil {
@@ -346,3 +350,50 @@ func RunInternalPlugin(pluginType PluginType, config BasicAndOLMPluginConfig, lo
 	}
 	return output, nil
 }
+
+func ListInternalPlugin(pluginType PluginType, config BasicAndOLMPluginConfig) (scapiv1alpha1.ScorecardOutput, error) {
+	var suites []schelpers.TestSuite
+
+	switch pluginType {
+	case BasicOperator:
+		conf := BasicTestConfig{}
+		basicTests := NewBasicTestSuite(conf)
+
+		if schelpers.IsV1alpha2(config.Version) {
+			basicTests.ApplySelector(config.Selector)
+		}
+
+		basicTests.TestResults = make([]schelpers.TestResult, 0)
+		for i := 0; i < len(basicTests.Tests); i++ {
+			result := schelpers.TestResult{}
+			result.Test = basicTests.Tests[i]
+			result.Suggestions = make([]string, 0)
+			result.Errors = make([]error, 0)
+			basicTests.TestResults = append(basicTests.TestResults, result)
+		}
+		suites = append(suites, *basicTests)
+	case OLMIntegration:
+		conf := OLMTestConfig{}
+		olmTests := NewOLMTestSuite(conf)
+
+		if schelpers.IsV1alpha2(config.Version) {
+			olmTests.ApplySelector(config.Selector)
+		}
+
+		olmTests.TestResults = make([]schelpers.TestResult, 0)
+		for i := 0; i < len(olmTests.Tests); i++ {
+			result := schelpers.TestResult{}
+			result.Test = olmTests.Tests[i]
+			result.Suggestions = make([]string, 0)
+			result.Errors = make([]error, 0)
+			olmTests.TestResults = append(olmTests.TestResults, result)
+		}
+		suites = append(suites, *olmTests)
+	}
+	suites, err := schelpers.MergeSuites(suites)
+	if err != nil {
+		return scapiv1alpha1.ScorecardOutput{}, fmt.Errorf("failed to merge test suite results: %v", err)
+	}
+	output := schelpers.TestSuitesToScorecardOutput(suites, "")
+	return output, nil
+}

internal/scorecard/scorecard.go

@@ -43,6 +43,7 @@ const (
 	JSONOutputFormat = "json"
 	TextOutputFormat = "text"
 	SelectorOpt      = "selector"
+	ListOpt          = "list"
 )
 
 // make a global logger for scorecard
@@ -64,6 +65,7 @@ func getPlugins(version string, selector labels.Selector) ([]Plugin, error) {
 	if scViper.IsSet(scplugins.KubeconfigOpt) {
 		kubeconfig = scViper.GetString(scplugins.KubeconfigOpt)
 	}
+
 	// Add plugins from config
 	var plugins []Plugin
 	configs := []pluginConfig{}
@@ -80,12 +82,14 @@ func getPlugins(version string, selector labels.Selector) ([]Plugin, error) {
 			pluginConfig := plugin.Basic
 			pluginConfig.Version = version
 			pluginConfig.Selector = selector
+			pluginConfig.ListOpt = scViper.GetBool(ListOpt)
 			setConfigDefaults(pluginConfig, kubeconfig)
 			newPlugin = basicOrOLMPlugin{pluginType: scplugins.BasicOperator, config: *pluginConfig}
 		} else if plugin.Olm != nil {
 			pluginConfig := plugin.Olm
 			pluginConfig.Version = version
 			pluginConfig.Selector = selector
+			pluginConfig.ListOpt = scViper.GetBool(ListOpt)
 			setConfigDefaults(pluginConfig, kubeconfig)
 			newPlugin = basicOrOLMPlugin{pluginType: scplugins.OLMIntegration, config: *pluginConfig}
 		} else {
@@ -122,7 +126,11 @@ func ScorecardTests(cmd *cobra.Command, args []string) error {
 
 	var pluginOutputs []scapiv1alpha1.ScorecardOutput
 	for _, plugin := range plugins {
-		pluginOutputs = append(pluginOutputs, plugin.Run())
+		if scViper.GetBool(ListOpt) {
+			pluginOutputs = append(pluginOutputs, plugin.List())
+		} else {
+			pluginOutputs = append(pluginOutputs, plugin.Run())
+		}
 	}
 
 	// Update the state for the tests
@@ -187,7 +195,17 @@ func validateScorecardConfig() error {
 		return fmt.Errorf("invalid output format (%s); valid values: %s, %s", outputFormat, TextOutputFormat, JSONOutputFormat)
 	}
 
-	return schelpers.ValidateVersion(scViper.GetString(schelpers.VersionOpt))
+	version := scViper.GetString(schelpers.VersionOpt)
+	err := schelpers.ValidateVersion(version)
+	if err != nil {
+		return err
+	}
+
+	if !schelpers.IsV1alpha2(version) && scViper.GetBool(ListOpt) {
+		return fmt.Errorf("list flag is not supported on v1alpha1")
+	}
+
+	return nil
 
 }
 
@@ -198,6 +216,7 @@ func makeSCViper() {
 	scViper.Set(scplugins.KubeconfigOpt, viper.GetString("scorecard."+scplugins.KubeconfigOpt))
 	scViper.Set(schelpers.VersionOpt, viper.GetString("scorecard."+schelpers.VersionOpt))
 	scViper.Set(SelectorOpt, viper.GetString("scorecard."+SelectorOpt))
+	scViper.Set(ListOpt, viper.GetString("scorecard."+ListOpt))
 
 }
 

internal/scorecard/scorecard_output.go

@@ -20,6 +20,7 @@ import (
 	schelpers "github.com/operator-framework/operator-sdk/internal/scorecard/helpers"
 	scapi "github.com/operator-framework/operator-sdk/pkg/apis/scorecard"
 	scapiv1alpha1 "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha1"
+	scapiv1alpha2 "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha2"
 	"io/ioutil"
 )
 
@@ -34,6 +35,12 @@ func printPluginOutputs(version string, pluginOutputs []scapiv1alpha1.ScorecardO
 
 	if schelpers.IsV1alpha2(version) {
 		list = scapi.ConvertScorecardOutputV1ToV2(list.(scapiv1alpha1.ScorecardOutput))
+		if scViper.GetBool(ListOpt) {
+			scorecardOutput := list.(scapiv1alpha2.ScorecardOutput)
+			for i := 0; i < len(scorecardOutput.Results); i++ {
+				scorecardOutput.Results[i].State = scapiv1alpha2.NotRunState
+			}
+		}
 	}
 
 	// produce text output

pkg/apis/scorecard/v1alpha2/formatter.go

@@ -30,8 +30,8 @@ const (
 func (s ScorecardOutput) MarshalText() (string, error) {
 	var sb strings.Builder
 
-	failColor := "\033[1;" + redColor + "m%s\033[0m\n"
-	passColor := "\033[1;" + greenColor + "m%s\033[0m\n"
+	failColor := ": \033[1;" + redColor + "m%s\033[0m\n"
+	passColor := ": \033[1;" + greenColor + "m%s\033[0m\n"
 
 	// turn off colorization if not in a terminal
 	if !isatty.IsTerminal(os.Stdout.Fd()) &&
@@ -46,27 +46,37 @@ func (s ScorecardOutput) MarshalText() (string, error) {
 			sb.WriteString(fmt.Sprintf("%s:\n", result.Labels["suite"]))
 			currentSuite = result.Labels["suite"]
 		}
-		sb.WriteString(fmt.Sprintf("\t%-35s: ", result.Name))
+		sb.WriteString(fmt.Sprintf("\t%-35s ", result.Name))
 
 		if result.State == PassState {
 			sb.WriteString(fmt.Sprintf(passColor, PassState))
-		} else {
+		} else if result.State == FailState {
 			sb.WriteString(fmt.Sprintf(failColor, FailState))
+		} else {
+			sb.WriteString(fmt.Sprintf("\n"))
 		}
-	}
 
-	for _, result := range s.Results {
-		for _, suggestion := range result.Suggestions {
+		sb.WriteString("\tLabels: \n")
+		for labelKey, labelValue := range result.Labels {
+			sb.WriteString(fmt.Sprintf("\t\t%q:%q\n", labelKey, labelValue))
+		}
+
+		if len(result.Suggestions) > 0 {
 			// 33 is yellow (specifically, the same shade of yellow that logrus uses for warnings)
-			sb.WriteString(fmt.Sprintf("\x1b[%dmSUGGESTION:\x1b[0m %s\n", 33, suggestion))
+			sb.WriteString(fmt.Sprintf("\t\x1b[%dmSuggestions:\x1b[0m \n", 33))
+		}
+		for _, suggestion := range result.Suggestions {
+			sb.WriteString(fmt.Sprintf("\t\t%s\n", suggestion))
 		}
-	}
 
-	for _, result := range s.Results {
-		for _, err := range result.Errors {
+		if len(result.Errors) > 0 {
 			// 31 is red (specifically, the same shade of red that logrus uses for errors)
-			sb.WriteString(fmt.Sprintf("\x1b[%dmERROR:\x1b[0m %s\n", 31, err))
+			sb.WriteString(fmt.Sprintf("\t\x1b[%dmErrors:\x1b[0m \n", 31))
+		}
+		for _, err := range result.Errors {
+			sb.WriteString(fmt.Sprintf("\t\t%s\n", err))
 		}
+		sb.WriteString(fmt.Sprintf("\n"))
 	}
 
 	return sb.String(), nil

pkg/apis/scorecard/v1alpha2/types.go

@@ -22,6 +22,8 @@ import (
 type State string
 
 const (
+	// NotRun occurs when a user specifies the --list flag
+	NotRunState State = ""
 	// PassState occurs when a Test's ExpectedPoints == MaximumPoints.
 	PassState State = "pass"
 	// FailState occurs when a Test's ExpectedPoints == 0.