Skip to content

cmd/.../gen-csv: use --include instead of CSV config #2249

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
## Unreleased

### Added

- Support for vars in top level ansible watches. ([#2147](https://github.com/operator-framework/operator-sdk/pull/2147))
- Support for `"ansible.operator-sdk/verbosity"` annotation on Custom Resources watched by Ansible based operators to override verbosity on an individual resource. ([#2102](https://github.com/operator-framework/operator-sdk/pull/2102))
- Support for relative helm chart paths in the Helm operator's watches.yaml file. ([#2287](https://github.com/operator-framework/operator-sdk/pull/2287))
- Added the [`olm-catalog gen-csv --include`](doc/cli/operator-sdk_olm-catalog_gen-csv.md#options) option to include files as input to the CSV generator in lieu of a config. ([#2249](https://github.com/operator-framework/operator-sdk/pull/2249))

### Changed

- Upgrade minimal Ansible version in the init projects from `2.4` to `2.6`. ([#2107](https://github.com/operator-framework/operator-sdk/pull/2107))
- Upgrade Kubernetes version from `kubernetes-1.15.4` to `kubernetes-1.16.2`. ([#2145](https://github.com/operator-framework/operator-sdk/pull/2145))
- Upgrade Helm version from `v2.15.0` to `v2.16.1`. ([#2145](https://github.com/operator-framework/operator-sdk/pull/2145))
Expand All @@ -20,6 +23,8 @@

### Removed

- Removed CSV configuration file support in favor of including all files in the default dir `deploy` and using [`gen-csv --include`](doc/cli/operator-sdk_olm-catalog_gen-csv.md#options) to include files as input to generator. ([#2249](https://github.com/operator-framework/operator-sdk/pull/2249))

### Bug Fixes
- Fix issue faced in the Ansible based operators when `jmespath` queries are used because it was not installed. ([#2252](https://github.com/operator-framework/operator-sdk/pull/2252))
- Fix scorecard behavior such that a CSV file is read correctly when `olm-deployed` is set to `true`. ([#2274](https://github.com/operator-framework/operator-sdk/pull/2274))
Expand Down
62 changes: 29 additions & 33 deletions cmd/operator-sdk/olmcatalog/gen-csv.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,10 @@ package olmcatalog
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"

genutil "github.com/operator-framework/operator-sdk/internal/generate/util"
"github.com/operator-framework/operator-sdk/internal/scaffold"
"github.com/operator-framework/operator-sdk/internal/scaffold/input"
catalog "github.com/operator-framework/operator-sdk/internal/scaffold/olm-catalog"
Expand All @@ -35,7 +37,7 @@ var (
csvVersion string
csvChannel string
fromVersion string
csvConfigPath string
includePaths []string
operatorName string
updateCRDs bool
defaultChannel bool
Expand All @@ -50,9 +52,7 @@ for the operator. This file is used to publish the operator to the OLM Catalog.

A CSV semantic version is supplied via the --csv-version flag. If your operator
has already generated a CSV manifest you want to use as a base, supply its
version to --from-version. Otherwise the SDK will scaffold a new CSV manifest.

Configure CSV generation by writing a config file 'deploy/olm-catalog/csv-config.yaml`,
version to --from-version. Otherwise the SDK will scaffold a new CSV manifest.`,
RunE: genCSVFunc,
}

Expand All @@ -61,7 +61,7 @@ Configure CSV generation by writing a config file 'deploy/olm-catalog/csv-config
log.Fatalf("Failed to mark `csv-version` flag for `olm-catalog gen-csv` subcommand as required: %v", err)
}
genCSVCmd.Flags().StringVar(&fromVersion, "from-version", "", "Semantic version of an existing CSV to use as a base")
genCSVCmd.Flags().StringVar(&csvConfigPath, "csv-config", "", "Path to CSV config file. Defaults to deploy/olm-catalog/csv-config.yaml")
genCSVCmd.Flags().StringSliceVar(&includePaths, "include", []string{scaffold.DeployDir}, "Paths to include in CSV generation, ex. \"deploy/prod,deploy/test\". If this flag is set and you want to enable default behavior, you must include \"deploy/\" in the argument list")
genCSVCmd.Flags().BoolVar(&updateCRDs, "update-crds", false, "Update CRD manifests in deploy/{operator-name}/{csv-version} the using latest API's")
genCSVCmd.Flags().StringVar(&operatorName, "operator-name", "", "Operator name to use while generating CSV")
genCSVCmd.Flags().StringVar(&csvChannel, "csv-channel", "", "Channel the CSV should be registered under in the package manifest")
Expand Down Expand Up @@ -91,40 +91,36 @@ func genCSVFunc(cmd *cobra.Command, args []string) error {
log.Infof("Generating CSV manifest version %s", csvVersion)

if operatorName == "" {
operatorName = filepath.Base(absProjectPath)
operatorName = filepath.Base(projutil.MustGetwd())
}
gcfg := genutil.Config{
OperatorName: operatorName,
IncludeFuncs: genutil.MakeIncludeFuncs(includePaths...),
}

s := &scaffold.Scaffold{}
csv := &catalog.CSV{
CSVVersion: csvVersion,
FromVersion: fromVersion,
ConfigFilePath: csvConfigPath,
OperatorName: operatorName,
Config: gcfg,
CSVVersion: csvVersion,
FromVersion: fromVersion,
}
err := s.Execute(cfg,
csv,
&catalog.PackageManifest{
Config: gcfg,
CSVVersion: csvVersion,
Channel: csvChannel,
ChannelIsDefault: defaultChannel,
OperatorName: operatorName,
},
)
if err != nil {
return fmt.Errorf("catalog scaffold failed: (%v)", err)
}

// Write CRD's to the new or updated CSV package dir.
bundleDir := filepath.Join(catalog.OLMCatalogDir, operatorName, csvVersion)
if updateCRDs {
input, err := csv.GetInput()
if err != nil {
return err
}
cfg, err := catalog.GetCSVConfig(csvConfigPath)
if err != nil {
return err
}
err = writeCRDsToDir(cfg.CRDCRPaths, filepath.Dir(input.Path))
err = writeCRDsToDir(scaffold.CRDsDir, bundleDir, gcfg.IncludeFuncs)
if err != nil {
return err
}
Expand Down Expand Up @@ -166,25 +162,25 @@ func verifyCSVVersion(version string) error {
return nil
}

func writeCRDsToDir(crdPaths []string, toDir string) error {
for _, p := range crdPaths {
b, err := ioutil.ReadFile(p)
if err != nil {
func writeCRDsToDir(fromDir, toDir string, includeFuncs genutil.IncludeFuncs) error {
return filepath.Walk(fromDir, func(path string, info os.FileInfo, err error) error {
if err != nil || info.IsDir() {
return err
}
typeMeta, err := k8sutil.GetTypeMetaFromBytes(b)
if !includeFuncs.IsInclude(path) {
return nil
}
b, err := ioutil.ReadFile(path)
if err != nil {
return err
}
if typeMeta.Kind != "CustomResourceDefinition" {
continue
}

path := filepath.Join(toDir, filepath.Base(p))
err = ioutil.WriteFile(path, b, fileutil.DefaultFileMode)
typeMeta, err := k8sutil.GetTypeMetaFromBytes(b)
if err != nil {
return err
}
}
return nil
if typeMeta.Kind == "CustomResourceDefinition" {
return ioutil.WriteFile(path, b, fileutil.DefaultFileMode)
}
return nil
})
}
4 changes: 1 addition & 3 deletions doc/cli/operator-sdk_olm-catalog_gen-csv.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,6 @@ A CSV semantic version is supplied via the --csv-version flag. If your operator
has already generated a CSV manifest you want to use as a base, supply its
version to --from-version. Otherwise the SDK will scaffold a new CSV manifest.

Configure CSV generation by writing a config file 'deploy/olm-catalog/csv-config.yaml

```
operator-sdk olm-catalog gen-csv [flags]
```
Expand All @@ -21,11 +19,11 @@ operator-sdk olm-catalog gen-csv [flags]

```
--csv-channel string Channel the CSV should be registered under in the package manifest
--csv-config string Path to CSV config file. Defaults to deploy/olm-catalog/csv-config.yaml
--csv-version string Semantic version of the CSV
--default-channel Use the channel passed to --csv-channel as the package manifests' default channel. Only valid when --csv-channel is set
--from-version string Semantic version of an existing CSV to use as a base
-h, --help help for gen-csv
--include strings Paths to include in CSV generation, ex. "deploy/prod,deploy/test". If this flag is set and you want to enable default behavior, you must include "deploy/" in the argument list (default [deploy])
--operator-name string Operator name to use while generating CSV
--update-crds Update CRD manifests in deploy/{operator-name}/{csv-version} the using latest API's
```
Expand Down
28 changes: 4 additions & 24 deletions doc/user/olm-catalog/generating-a-csv.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,40 +11,20 @@ This document describes how to manage the following lifecycle for your Operator
Operator SDK projects have an expected [project layout][doc-project-layout]. In particular, a few manifests are expected to be present in the `deploy` directory:

* Roles: `role.yaml`
* ClusterRoles: `cluster_role.yaml`
* Deployments: `operator.yaml`
* Custom Resources (CR's): `crds/<full group>_<version>_<kind>_cr.yaml`
* Custom Resource Definitions (CRD's): `crds/<full group>_<resource>_crd.yaml`.
* CustomResourceDefinitions (CRD's): `crds/<full group>_<resource>_crd.yaml`.

`gen-csv` reads these files and adds their data to a CSV in an alternate form.

The following example config containing default values should be copied and written to `deploy/olm-catalog/csv-config.yaml`:

```yaml
crd-cr-paths:
- deploy/crds
operator-path: deploy/operator.yaml
role-paths:
- deploy/role.yaml
```

Explanation of all config fields:

- `crd-cr-paths`: list of strings - a list of CRD and CR manifest file/directory paths. Defaults to `[deploy/crds]`.
- `operator-path`: string - the operator `Deployment` manifest file path. Defaults to `deploy/operator.yaml`.
- `role-paths`: list of strings - Role and ClusterRole manifest file paths. Defaults to `[deploy/role.yaml]`.
- `operator-name`: string - the name used to create the CSV and manifest file names. Defaults to the project's name.

**Note**: The [design doc][doc-csv-design] has outdated field information which should not be referenced.

Fields in this config file can be modified to point towards alternate manifest locations, and passed to `gen-csv --csv-config=<path>` to configure CSV generation. For example, if I have one set of production CR/CRD manifests under `deploy/crds/production`, and a set of test manifests under `deploy/crds/test`, and I only want to include production manifests in my CSV, I can set `crd-cr-paths: [deploy/crds/production]`. `gen-csv` will then ignore `deploy/crds/test` when getting CR/CRD data.
`gen-csv` extracts manifests from files in `deploy/` by default that match the kinds above and adds them to the CSV. If your manifest files are not in `deploy/`, you can use the `--include=[list of paths]` option to instruct the command to extract manifests from files at those paths, ex. `--include="deploy/prod,deploy/test"`. Setting `--include` overrides default behavior; if you still want default behavior, you must append `deploy/` to the list of paths passed to `--include`.

## Versioning

CSV's are versioned in path, file name, and in their `metadata.name` field. For example, running `operator-sdk olm-catalog gen-csv --csv-version 0.0.1` will generate a CSV at `deploy/olm-catalog/<operator-name>/0.0.1/<operator-name>.v0.0.1.clusterserviceversion.yaml`. A versioned directory such as `deploy/olm-catalog/<operator-name>/0.0.1` is known as a [*bundle*][doc-bundle]. Versions allow the OLM to upgrade or downgrade your Operator at runtime, i.e. in a cluster. A valid semantic version is required.

`gen-csv` allows you to upgrade your CSV using the `--from-version` flag. If you have an existing CSV with version `0.0.1` and want to write a new version `0.0.2`, you can run `operator-sdk olm-catalog gen-csv --csv-version 0.0.2 --from-version 0.0.1`. This will write a new CSV manifest to `deploy/olm-catalog/<operator-name>/0.0.2/<operator-name>.v0.0.2.clusterserviceversion.yaml` containing user-defined data from `0.0.1` and any modifications you've made to `roles.yaml`, `operator.yaml`, CR's, or CRD's.

The SDK can manage CRD's in your Operator bundle as well. You can pass the `--update-crds` flag to `gen-csv` to add or update your CRD's in your bundle by copying manifests pointed to by `crd-cr-paths` in your config. CRD's in a bundle are not updated by default.
The SDK can manage CRD's in your Operator bundle as well. You can pass the `--update-crds` flag to `gen-csv` to add or update your CRD's in your bundle by copying manifests in `deploy/crds` to your bundle. CRD's in a bundle are not updated by default.

## First Generation

Expand Down
74 changes: 74 additions & 0 deletions internal/generate/util/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
// Copyright 2019 The Operator-SDK Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package genutil

import (
"path/filepath"
"strings"

"github.com/operator-framework/operator-sdk/internal/util/projutil"
)

// Config configures a generator with common operator project information.
type Config struct {
// OperatorName is the operator's name, ex. app-operator
OperatorName string
// InputDir is a dir containing relevant input files. If not set, a default
// is used on a per-generator basis.
InputDir string
// OutputDir is a dir in which to generate output files. If not set, a
// default is used on a per-generator basis.
OutputDir string
// IncludeFuncs contains a set of filters for paths that a generator
// may encounter while gathering data for generation. If any func returns
// true, that path will be included by the generator.
IncludeFuncs IncludeFuncs
}

// IncludeFuncs is a slice of filter funcs. A string passing any func in
// IncludeFuncs satisfies the filter.
type IncludeFuncs []func(string) bool

// MakeIncludeFuncs creates a set of closures around each path in paths
// to populate Config.IncludeFuncs. If the argument to the closure has
// a prefix of path, it returns true.
func MakeIncludeFuncs(paths ...string) (includes IncludeFuncs) {
pathSet := map[string]struct{}{}
for _, path := range paths {
pathSet[filepath.Clean(path)] = struct{}{}
}
wd := projutil.MustGetwd() + string(filepath.Separator)
for path := range pathSet {
// Copy the string for the closure.
pb := strings.Builder{}
pb.WriteString(path)
includes = append(includes, func(p string) bool {
// Handle absolute paths referencing the project directory.
p = strings.TrimPrefix(p, wd)
return strings.HasPrefix(filepath.Clean(p), pb.String())
})
}
return includes
}

// IsInclude checks if path passes any filter in funcs.
func (funcs IncludeFuncs) IsInclude(path string) bool {
for _, f := range funcs {
if f(path) {
return true
}
}
return false
}
Loading