Skip to content

Commit f509fcd

Browse files
author
Eric Stroczynski
authored
*: refactor CRD/API getters (#1782)
* *: refactor CRD/API getters to internal/util/crd.go
1 parent 9d6ffdd commit f509fcd

File tree

6 files changed

+113
-78
lines changed

6 files changed

+113
-78
lines changed

‎cmd/operator-sdk/internal/genutil/genutil.go

Lines changed: 0 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -15,72 +15,14 @@
1515
package genutil
1616

1717
import (
18-
"fmt"
1918
"io/ioutil"
2019
"os"
21-
"path"
22-
"path/filepath"
2320

2421
"github.com/operator-framework/operator-sdk/internal/pkg/scaffold"
2522

2623
log "github.com/sirupsen/logrus"
2724
)
2825

29-
// ParseGroupVersions parses the layout of pkg/apis to return a map of
30-
// API groups to versions.
31-
func parseGroupVersions() (map[string][]string, error) {
32-
gvs := make(map[string][]string)
33-
groups, err := ioutil.ReadDir(scaffold.ApisDir)
34-
if err != nil {
35-
return nil, fmt.Errorf("could not read pkg/apis directory to find api Versions: %v", err)
36-
}
37-
38-
for _, g := range groups {
39-
if g.IsDir() {
40-
groupDir := filepath.Join(scaffold.ApisDir, g.Name())
41-
versions, err := ioutil.ReadDir(groupDir)
42-
if err != nil {
43-
return nil, fmt.Errorf("could not read %s directory to find api Versions: %v", groupDir, err)
44-
}
45-
46-
gvs[g.Name()] = make([]string, 0)
47-
for _, v := range versions {
48-
if v.IsDir() {
49-
// Ignore directories that do not contain any files, so generators
50-
// do not get empty directories as arguments.
51-
verDir := filepath.Join(groupDir, v.Name())
52-
files, err := ioutil.ReadDir(verDir)
53-
if err != nil {
54-
return nil, fmt.Errorf("could not read %s directory to find api Versions: %v", verDir, err)
55-
}
56-
for _, f := range files {
57-
if !f.IsDir() && filepath.Ext(f.Name()) == ".go" {
58-
gvs[g.Name()] = append(gvs[g.Name()], filepath.ToSlash(v.Name()))
59-
break
60-
}
61-
}
62-
}
63-
}
64-
}
65-
}
66-
67-
if len(gvs) == 0 {
68-
return nil, fmt.Errorf("no groups or versions found in %s", scaffold.ApisDir)
69-
}
70-
return gvs, nil
71-
}
72-
73-
// createFQAPIs return a slice of all fully qualified pkg + groups + versions
74-
// of pkg and gvs in the format "pkg/groupA/v1".
75-
func createFQAPIs(pkg string, gvs map[string][]string) (apis []string) {
76-
for g, vs := range gvs {
77-
for _, v := range vs {
78-
apis = append(apis, path.Join(pkg, g, v))
79-
}
80-
}
81-
return apis
82-
}
83-
8426
// generateWithHeaderFile runs f with a header file path as an arguemnt.
8527
// If there is no project boilerplate.go.txt file, an empty header file is
8628
// created and its path passed as the argument.

‎cmd/operator-sdk/internal/genutil/k8s.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"strings"
2323

2424
"github.com/operator-framework/operator-sdk/internal/pkg/scaffold"
25+
"github.com/operator-framework/operator-sdk/internal/util/k8sutil"
2526
"github.com/operator-framework/operator-sdk/internal/util/projutil"
2627

2728
"github.com/pkg/errors"
@@ -37,7 +38,7 @@ func K8sCodegen() error {
3738

3839
repoPkg := projutil.GetGoPkg()
3940

40-
gvMap, err := parseGroupVersions()
41+
gvMap, err := k8sutil.ParseGroupSubpackages(scaffold.ApisDir)
4142
if err != nil {
4243
return fmt.Errorf("failed to parse group versions: (%v)", err)
4344
}
@@ -49,7 +50,7 @@ func K8sCodegen() error {
4950
log.Infof("Running deepcopy code-generation for Custom Resource group versions: [%v]\n", gvb.String())
5051

5152
apisPkg := filepath.Join(repoPkg, scaffold.ApisDir)
52-
fqApis := createFQAPIs(apisPkg, gvMap)
53+
fqApis := k8sutil.CreateFQAPIs(apisPkg, gvMap)
5354
f := func(a string) error { return deepcopyGen(a, fqApis) }
5455
if err = generateWithHeaderFile(f); err != nil {
5556
return err

‎cmd/operator-sdk/internal/genutil/openapi.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ func OpenAPIGen() error {
3939
absProjectPath := projutil.MustGetwd()
4040
repoPkg := projutil.GetGoPkg()
4141

42-
gvMap, err := parseGroupVersions()
42+
gvMap, err := k8sutil.ParseGroupSubpackages(scaffold.ApisDir)
4343
if err != nil {
4444
return fmt.Errorf("failed to parse group versions: (%v)", err)
4545
}
@@ -51,7 +51,7 @@ func OpenAPIGen() error {
5151
log.Infof("Running OpenAPI code-generation for Custom Resource group versions: [%v]\n", gvb.String())
5252

5353
apisPkg := filepath.Join(repoPkg, scaffold.ApisDir)
54-
fqApis := createFQAPIs(apisPkg, gvMap)
54+
fqApis := k8sutil.CreateFQAPIs(apisPkg, gvMap)
5555
f := func(a string) error { return openAPIGen(a, fqApis) }
5656
if err = generateWithHeaderFile(f); err != nil {
5757
return err

‎internal/pkg/scaffold/olm-catalog/csv.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -336,16 +336,16 @@ func (s *CSV) updateCSVFromManifestFiles(cfg *CSVConfig, csv *olmapiv1alpha1.Clu
336336
scanner := yamlutil.NewYAMLScanner(yamlData)
337337
for scanner.Scan() {
338338
yamlSpec := scanner.Bytes()
339-
typemeta, err := k8sutil.GetTypeMetaFromBytes(yamlSpec)
339+
typeMeta, err := k8sutil.GetTypeMetaFromBytes(yamlSpec)
340340
if err != nil {
341341
return errors.Wrapf(err, "error getting type metadata from manifest %s", f)
342342
}
343-
found, err := store.AddToUpdater(yamlSpec, typemeta.Kind)
343+
found, err := store.AddToUpdater(yamlSpec, typeMeta.Kind)
344344
if err != nil {
345345
return errors.Wrapf(err, "error adding manifest %s to CSV updaters", f)
346346
}
347347
if !found {
348-
id := gvkID(typemeta.GroupVersionKind())
348+
id := gvkID(typeMeta.GroupVersionKind())
349349
if _, ok := otherSpecs[id]; !ok {
350350
otherSpecs[id] = make([][]byte, 0)
351351
}

‎internal/util/k8sutil/crd.go

Lines changed: 95 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,16 @@ import (
1818
"fmt"
1919
"io/ioutil"
2020
"os"
21+
"path"
2122
"path/filepath"
22-
"strings"
23+
"regexp"
2324

2425
yaml "github.com/ghodss/yaml"
26+
"github.com/pkg/errors"
2527
apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
2628
)
2729

30+
// GetCRDs parses all CRD manifests in the directory crdsDir and all of its subdirectories.
2831
func GetCRDs(crdsDir string) ([]*apiextv1beta1.CustomResourceDefinition, error) {
2932
manifests, err := GetCRDManifestPaths(crdsDir)
3033
if err != nil {
@@ -45,6 +48,7 @@ func GetCRDs(crdsDir string) ([]*apiextv1beta1.CustomResourceDefinition, error)
4548
return crds, nil
4649
}
4750

51+
// GetCRDManifestPaths gets all CRD manifest paths in crdsDir and subdirs.
4852
func GetCRDManifestPaths(crdsDir string) (crdPaths []string, err error) {
4953
err = filepath.Walk(crdsDir, func(path string, info os.FileInfo, werr error) error {
5054
if werr != nil {
@@ -53,10 +57,98 @@ func GetCRDManifestPaths(crdsDir string) (crdPaths []string, err error) {
5357
if info == nil {
5458
return nil
5559
}
56-
if !info.IsDir() && strings.HasSuffix(path, "_crd.yaml") {
57-
crdPaths = append(crdPaths, path)
60+
if !info.IsDir() {
61+
b, err := ioutil.ReadFile(path)
62+
if err != nil {
63+
return errors.Wrapf(err, "error reading manifest %s", path)
64+
}
65+
typeMeta, err := GetTypeMetaFromBytes(b)
66+
if err != nil {
67+
return errors.Wrapf(err, "error getting kind from manifest %s", path)
68+
}
69+
if typeMeta.Kind == "CustomResourceDefinition" {
70+
crdPaths = append(crdPaths, path)
71+
}
5872
}
5973
return nil
6074
})
6175
return crdPaths, err
6276
}
77+
78+
// ParseGroupSubpackages parses the apisDir directory tree and returns a map of
79+
// all found API groups to subpackages.
80+
func ParseGroupSubpackages(apisDir string) (map[string][]string, error) {
81+
return parseGroupSubdirs(apisDir, false)
82+
}
83+
84+
// ParseGroupVersions parses the apisDir directory tree and returns a map of
85+
// all found API groups to versions.
86+
func ParseGroupVersions(apisDir string) (map[string][]string, error) {
87+
return parseGroupSubdirs(apisDir, true)
88+
}
89+
90+
// versionRegexp defines a kube-like version:
91+
// https://kubernetes.io/docs/concepts/overview/kubernetes-api/#api-versioning
92+
var versionRegexp = regexp.MustCompile("^v[1-9][0-9]*((alpha|beta)[1-9][0-9]*)?$")
93+
94+
// parseGroupSubdirs searches apisDir for all groups and potential version
95+
// subdirs directly beneath each group dir, and returns a map of each group
96+
// dir name to all children version dir names. If strictVersionMatch is true,
97+
// all potential version dir names must strictly match versionRegexp. If
98+
// false, all subdir names are considered valid.
99+
func parseGroupSubdirs(apisDir string, strictVersionMatch bool) (map[string][]string, error) {
100+
gvs := make(map[string][]string)
101+
groups, err := ioutil.ReadDir(apisDir)
102+
if err != nil {
103+
return nil, errors.Wrapf(err, "error reading directory %q to find API groups", apisDir)
104+
}
105+
106+
for _, g := range groups {
107+
if g.IsDir() {
108+
groupDir := filepath.Join(apisDir, g.Name())
109+
versions, err := ioutil.ReadDir(groupDir)
110+
if err != nil {
111+
return nil, errors.Wrapf(err, "error reading directory %q to find API versions", groupDir)
112+
}
113+
114+
gvs[g.Name()] = make([]string, 0)
115+
for _, v := range versions {
116+
if v.IsDir() {
117+
// Ignore directories that do not contain any files, so generators
118+
// do not get empty directories as arguments.
119+
verDir := filepath.Join(groupDir, v.Name())
120+
files, err := ioutil.ReadDir(verDir)
121+
if err != nil {
122+
return nil, errors.Wrapf(err, "error reading directory %q to find API source files", verDir)
123+
}
124+
for _, f := range files {
125+
if !f.IsDir() && filepath.Ext(f.Name()) == ".go" {
126+
// If strictVersionMatch is true, strictly check if v.Name()
127+
// is a Kubernetes API version.
128+
if !strictVersionMatch || versionRegexp.MatchString(v.Name()) {
129+
gvs[g.Name()] = append(gvs[g.Name()], v.Name())
130+
}
131+
break
132+
}
133+
}
134+
}
135+
}
136+
}
137+
}
138+
139+
if len(gvs) == 0 {
140+
return nil, fmt.Errorf("no groups or versions found in %s", apisDir)
141+
}
142+
return gvs, nil
143+
}
144+
145+
// CreateFQAPIs return a slice of all fully qualified pkg + groups + versions
146+
// of pkg and gvs in the format "pkg/groupA/v1".
147+
func CreateFQAPIs(pkg string, gvs map[string][]string) (apis []string) {
148+
for g, vs := range gvs {
149+
for _, v := range vs {
150+
apis = append(apis, path.Join(pkg, g, v))
151+
}
152+
}
153+
return apis
154+
}

‎internal/util/yamlutil/manifest.go

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,13 @@ import (
2020
"io/ioutil"
2121
"os"
2222
"path/filepath"
23-
"strings"
2423

2524
"github.com/operator-framework/operator-sdk/internal/pkg/scaffold"
2625
"github.com/operator-framework/operator-sdk/internal/util/fileutil"
26+
"github.com/operator-framework/operator-sdk/internal/util/k8sutil"
2727

28+
"github.com/ghodss/yaml"
29+
"github.com/pkg/errors"
2830
log "github.com/sirupsen/logrus"
2931
)
3032

@@ -110,19 +112,17 @@ func GenerateCombinedGlobalManifest(crdsDir string) (*os.File, error) {
110112
}
111113
}()
112114

113-
files, err := ioutil.ReadDir(crdsDir)
115+
crds, err := k8sutil.GetCRDs(crdsDir)
114116
if err != nil {
115-
return nil, fmt.Errorf("could not read deploy directory: (%v)", err)
117+
return nil, errors.Wrapf(err, "error getting CRD's from %s", crdsDir)
116118
}
117119
combined := []byte{}
118-
for _, file := range files {
119-
if strings.HasSuffix(file.Name(), "crd.yaml") {
120-
fileBytes, err := ioutil.ReadFile(filepath.Join(crdsDir, file.Name()))
121-
if err != nil {
122-
return nil, fmt.Errorf("could not read file %s: (%v)", filepath.Join(crdsDir, file.Name()), err)
123-
}
124-
combined = CombineManifests(combined, fileBytes)
120+
for _, crd := range crds {
121+
b, err := yaml.Marshal(crd)
122+
if err != nil {
123+
return nil, errors.Wrapf(err, "error marshalling CRD %s bytes", crd.GetName())
125124
}
125+
combined = CombineManifests(combined, b)
126126
}
127127

128128
if err := file.Chmod(os.FileMode(fileutil.DefaultFileMode)); err != nil {

0 commit comments

Comments
 (0)