Skip to content

Commit 794bcf5

Browse files
Jeff McCormickEric Stroczynskijoelanford
authored
Scorecard2 label processing (#2778)
* add initial test processor and label selection * make code pass linter * tweak comment * Update internal/scorecard/alpha/labels_test.go Co-Authored-By: Eric Stroczynski <[email protected]> * updates based on PR review * fix unit test * fix test and linter * add config path unit test * remove unused Log * Update internal/scorecard/alpha/scorecard.go Co-Authored-By: Joe Lanford <[email protected]> * changes from reviews * Update cmd/operator-sdk/alpha/scorecard/cmd.go Co-Authored-By: Joe Lanford <[email protected]> * change to RunE in command to be consistent with other commands * add TODO comments * comment out unused flags for now Co-authored-by: Eric Stroczynski <[email protected]> Co-authored-by: Joe Lanford <[email protected]>
1 parent bc442cc commit 794bcf5

File tree

5 files changed

+293
-14
lines changed

5 files changed

+293
-14
lines changed

‎cmd/operator-sdk/alpha/scorecard/cmd.go

Lines changed: 33 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -15,33 +15,54 @@
1515
package scorecard
1616

1717
import (
18-
log "github.com/sirupsen/logrus"
19-
"github.com/spf13/cobra"
20-
)
18+
"fmt"
19+
"log"
2120

22-
var (
23-
config string
24-
bundle string
25-
selector string
26-
listAll bool
21+
scorecard "github.com/operator-framework/operator-sdk/internal/scorecard/alpha"
22+
"github.com/spf13/cobra"
23+
"k8s.io/apimachinery/pkg/labels"
2724
)
2825

2926
func NewCmd() *cobra.Command {
27+
var (
28+
config string
29+
//bundle string // TODO - to be implemented
30+
selector string
31+
//list bool // TODO - to be implemented
32+
)
3033
scorecardCmd := &cobra.Command{
3134
Use: "scorecard",
3235
Short: "Runs scorecard",
3336
Long: `Has flags to configure dsl, bundle, and selector.`,
3437
Hidden: true,
35-
Run: func(cmd *cobra.Command, args []string) {
36-
log.Info("TODO")
38+
RunE: func(cmd *cobra.Command, args []string) error {
39+
40+
var err error
41+
o := scorecard.Options{}
42+
o.Config, err = scorecard.LoadConfig(config)
43+
if err != nil {
44+
return fmt.Errorf("could not find config file %s", err.Error())
45+
}
46+
47+
o.Selector, err = labels.Parse(selector)
48+
if err != nil {
49+
return fmt.Errorf("could not parse selector %s", err.Error())
50+
}
51+
52+
// TODO - process list and output formatting here?
53+
54+
if err := scorecard.RunTests(o); err != nil {
55+
log.Fatal(err)
56+
}
57+
return nil
3758
},
3859
}
3960

4061
scorecardCmd.Flags().StringVarP(&config, "config", "c", "",
4162
"path to a new to be defined DSL yaml formatted file that configures what tests get executed")
42-
scorecardCmd.Flags().StringVar(&bundle, "bundle", "", "path to the operator bundle contents on disk")
63+
// scorecardCmd.Flags().StringVar(&bundle, "bundle", "", "path to the operator bundle contents on disk")
4364
scorecardCmd.Flags().StringVarP(&selector, "selector", "l", "", "label selector to determine which tests are run")
44-
scorecardCmd.Flags().BoolVarP(&listAll, "list", "L", false, "option to enable listing which tests are run")
65+
// scorecardCmd.Flags().BoolVarP(&list, "list", "L", false, "option to enable listing which tests are run")
4566

4667
return scorecardCmd
4768
}

‎internal/scorecard/alpha/config.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@ type ScorecardTest struct {
2222
Description string `yaml:"description"` // User readable test description
2323
}
2424

25-
// ScorecardConfig represents the set of test configurations which scorecard
25+
// Config represents the set of test configurations which scorecard
2626
// would run based on user input
27-
type ScorecardConfig struct {
27+
type Config struct {
2828
Tests []ScorecardTest `yaml:"tests"`
2929
}

‎internal/scorecard/alpha/config_test.go

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
// Copyright 2020 The Operator-SDK Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package alpha
16+
17+
import (
18+
"testing"
19+
)
20+
21+
func TestInvalidConfigPath(t *testing.T) {
22+
23+
cases := []struct {
24+
configPathValue string
25+
wantError bool
26+
}{
27+
{"", true},
28+
}
29+
30+
for _, c := range cases {
31+
t.Run(c.configPathValue, func(t *testing.T) {
32+
_, err := LoadConfig(c.configPathValue)
33+
if err != nil && c.wantError {
34+
t.Logf("Wanted error and got error : %v", err)
35+
return
36+
} else if err != nil && !c.wantError {
37+
t.Errorf("Wanted result but got error: %v", err)
38+
return
39+
}
40+
41+
})
42+
43+
}
44+
}

‎internal/scorecard/alpha/labels_test.go

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
// Copyright 2020 The Operator-SDK Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package alpha
16+
17+
import (
18+
"testing"
19+
20+
"gopkg.in/yaml.v2"
21+
"k8s.io/apimachinery/pkg/labels"
22+
)
23+
24+
func TestEmptySelector(t *testing.T) {
25+
26+
cases := []struct {
27+
selectorValue string
28+
testsSelected int
29+
wantError bool
30+
}{
31+
{"", 7, false},
32+
{"suite in (kuttl)", 1, false},
33+
{"test=basic-check-spec-test", 1, false},
34+
{"testXwriteintocr", 0, false},
35+
{"test X writeintocr", 0, true},
36+
}
37+
38+
for _, c := range cases {
39+
t.Run(c.selectorValue, func(t *testing.T) {
40+
var scConfig Config
41+
42+
err := yaml.Unmarshal([]byte(testConfig), &scConfig)
43+
if err != nil {
44+
t.Log(err)
45+
return
46+
}
47+
48+
selector, err := labels.Parse(c.selectorValue)
49+
if err != nil && c.wantError {
50+
t.Logf("Wanted error and got error : %v", err)
51+
return
52+
} else if err != nil && !c.wantError {
53+
t.Errorf("Wanted result but got error: %v", err)
54+
return
55+
}
56+
57+
tests := selectTests(selector, scConfig.Tests)
58+
testsSelected := len(tests)
59+
if testsSelected != c.testsSelected {
60+
t.Errorf("Wanted testsSelected %d, got: %d", c.testsSelected, testsSelected)
61+
}
62+
})
63+
64+
}
65+
}
66+
67+
const testConfig = `tests:
68+
- name: "customtest1"
69+
image: quay.io/someuser/customtest1:v0.0.1
70+
labels:
71+
suite: custom
72+
test: customtest1
73+
description: an ISV custom test that does...
74+
- name: "customtest2"
75+
image: quay.io/someuser/customtest2:v0.0.1
76+
labels:
77+
suite: custom
78+
test: customtest2
79+
description: an ISV custom test that does...
80+
- name: "basic-check-spec"
81+
image: quay.io/redhat/basictests:v0.0.1
82+
entrypoint: basic-check-spec
83+
labels:
84+
suite: basic
85+
test: basic-check-spec-test
86+
description: check the spec test
87+
- name: "basic-check-status"
88+
image: quay.io/redhat/basictests:v0.0.1
89+
entrypoint: basic-check-status
90+
labels:
91+
suite: basic
92+
test: basic-check-status-test
93+
description: check the status test
94+
- name: "olm-bundle-validation"
95+
image: quay.io/redhat/olmtests:v0.0.1
96+
entrypoint: olm-bundle-validation
97+
labels:
98+
suite: olm
99+
test: olm-bundle-validation-test
100+
description: validate the bundle test
101+
- name: "olm-crds-have-validation"
102+
image: quay.io/redhat/olmtests:v0.0.1
103+
entrypoint: olm-crds-have-validation
104+
labels:
105+
suite: olm
106+
test: olm-crds-have-validation-test
107+
description: CRDs have validation
108+
- name: "kuttl-tests"
109+
image: quay.io/redhat/kuttltests:v0.0.1
110+
labels:
111+
suite: kuttl
112+
description: Kuttl tests
113+
`

‎internal/scorecard/alpha/scorecard.go

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
// Copyright 2020 The Operator-SDK Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package alpha
16+
17+
import (
18+
"errors"
19+
"fmt"
20+
"io/ioutil"
21+
"log"
22+
"strings"
23+
24+
"github.com/operator-framework/operator-sdk/version"
25+
"gopkg.in/yaml.v2"
26+
"k8s.io/apimachinery/pkg/labels"
27+
"k8s.io/client-go/kubernetes"
28+
)
29+
30+
type Options struct {
31+
Config Config
32+
Selector labels.Selector
33+
List bool
34+
OutputFormat string
35+
Client kubernetes.Interface
36+
}
37+
38+
// RunTests executes the scorecard tests as configured
39+
func RunTests(o Options) error {
40+
tests := selectTests(o.Selector, o.Config.Tests)
41+
42+
for i := 0; i < len(tests); i++ {
43+
if err := runTest(tests[i]); err != nil {
44+
return fmt.Errorf("test %s failed %s", tests[i].Name, err.Error())
45+
}
46+
}
47+
48+
return nil
49+
}
50+
51+
// LoadConfig will find and return the scorecard config, the config file
52+
// can be passed in via command line flag or from a bundle location or
53+
// bundle image
54+
func LoadConfig(configFilePath string) (Config, error) {
55+
c := Config{}
56+
57+
// TODO handle getting config from bundle (ondisk or image)
58+
yamlFile, err := ioutil.ReadFile(configFilePath)
59+
if err != nil {
60+
return c, err
61+
}
62+
63+
if err := yaml.Unmarshal(yamlFile, &c); err != nil {
64+
return c, err
65+
}
66+
67+
return c, nil
68+
}
69+
70+
// selectTests applies an optionally passed selector expression
71+
// against the configured set of tests, returning the selected tests
72+
func selectTests(selector labels.Selector, tests []ScorecardTest) []ScorecardTest {
73+
74+
selected := make([]ScorecardTest, 0)
75+
for i := 0; i < len(tests); i++ {
76+
if selector.String() == "" || selector.Matches(labels.Set(tests[i].Labels)) {
77+
// TODO olm manifests check
78+
selected = append(selected, tests[i])
79+
}
80+
}
81+
return selected
82+
}
83+
84+
// runTest executes a single test
85+
// TODO once tests exists, handle the test output
86+
func runTest(test ScorecardTest) error {
87+
if test.Name == "" {
88+
return errors.New("todo - remove later, only for linter")
89+
}
90+
log.Printf("running test %s labels %v", test.Name, test.Labels)
91+
return nil
92+
}
93+
94+
func ConfigDocLink() string {
95+
if strings.HasSuffix(version.Version, "+git") {
96+
return "https://github.com/operator-framework/operator-sdk/blob/master/doc/test-framework/scorecard.md"
97+
}
98+
return fmt.Sprintf(
99+
"https://github.com/operator-framework/operator-sdk/blob/%s/doc/test-framework/scorecard.md",
100+
version.Version)
101+
}

0 commit comments

Comments
 (0)