Skip to content

Commit c57aa5a

Browse files
Implement File-Based Catalog support for run bundle command in SDK
Create File-Based Catalog by generating bundle, package and channel blobs for a default index image Generate an FBC if index image provided in the CLI is FBC - add bundle to index if bundle is not on index Infer the image type (SQLite/FBC) based on the image label and handle FBC scenarios accordingly Validate the generated FBC and convert to a string format Add a new Registry pod container creation command to support FBC images Seperate out the codepaths for SQLite based and FBC images Signed-off-by: rashmigottipati <[email protected]>
1 parent 4b01a1b commit c57aa5a

File tree

6 files changed

+548
-46
lines changed

6 files changed

+548
-46
lines changed

go.mod

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,9 @@ require (
103103
github.com/garyburd/redigo v1.6.0 // indirect
104104
github.com/ghodss/yaml v1.0.0 // indirect
105105
github.com/go-errors/errors v1.0.1 // indirect
106+
github.com/go-git/gcfg v1.5.0 // indirect
107+
github.com/go-git/go-billy/v5 v5.1.0 // indirect
108+
github.com/go-git/go-git/v5 v5.3.0 // indirect
106109
github.com/go-logr/zapr v1.2.0 // indirect
107110
github.com/go-openapi/jsonpointer v0.19.5 // indirect
108111
github.com/go-openapi/jsonreference v0.19.5 // indirect
@@ -112,6 +115,7 @@ require (
112115
github.com/gobwas/glob v0.2.3 // indirect
113116
github.com/gofrs/uuid v3.3.0+incompatible // indirect
114117
github.com/gogo/protobuf v1.3.2 // indirect
118+
github.com/golang-migrate/migrate/v4 v4.6.2 // indirect
115119
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
116120
github.com/golang/protobuf v1.5.2 // indirect
117121
github.com/google/btree v1.0.1 // indirect
@@ -134,7 +138,9 @@ require (
134138
github.com/huandu/xstrings v1.3.1 // indirect
135139
github.com/imdario/mergo v0.3.12 // indirect
136140
github.com/inconshreveable/mousetrap v1.0.0 // indirect
141+
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
137142
github.com/jmoiron/sqlx v1.3.1 // indirect
143+
github.com/joelanford/ignore v0.0.0-20210607151042-0d25dc18b62d // indirect
138144
github.com/joho/godotenv v1.3.0 // indirect
139145
github.com/josharian/intern v1.0.0 // indirect
140146
github.com/json-iterator/go v1.1.12 // indirect
@@ -149,10 +155,12 @@ require (
149155
github.com/mattn/go-colorable v0.1.12 // indirect
150156
github.com/mattn/go-isatty v0.0.14 // indirect
151157
github.com/mattn/go-runewidth v0.0.7 // indirect
158+
github.com/mattn/go-sqlite3 v1.14.10 // indirect
152159
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
153160
github.com/mitchellh/copystructure v1.1.1 // indirect
154161
github.com/mitchellh/go-homedir v1.1.0 // indirect
155162
github.com/mitchellh/go-wordwrap v1.0.0 // indirect
163+
github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
156164
github.com/mitchellh/mapstructure v1.4.3 // indirect
157165
github.com/mitchellh/reflectwalk v1.0.1 // indirect
158166
github.com/moby/spdystream v0.2.0 // indirect
@@ -228,6 +236,7 @@ require (
228236
gopkg.in/inf.v0 v0.9.1 // indirect
229237
gopkg.in/ini.v1 v1.66.2 // indirect
230238
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
239+
gopkg.in/warnings.v0 v0.1.2 // indirect
231240
gopkg.in/yaml.v2 v2.4.0 // indirect
232241
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
233242
k8s.io/apiserver v0.23.5 // indirect

internal/olm/operator/bundle/install.go

Lines changed: 225 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,17 +15,40 @@
1515
package bundle
1616

1717
import (
18+
"bytes"
1819
"context"
20+
"errors"
21+
"fmt"
22+
"io/ioutil"
23+
"os"
1924
"strings"
2025

21-
"github.com/operator-framework/api/pkg/operators/v1alpha1"
22-
registrybundle "github.com/operator-framework/operator-registry/pkg/lib/bundle"
26+
log "github.com/sirupsen/logrus"
2327
"github.com/spf13/pflag"
2428

29+
"github.com/operator-framework/api/pkg/operators/v1alpha1"
30+
"github.com/operator-framework/operator-registry/alpha/action"
31+
"github.com/operator-framework/operator-registry/alpha/declcfg"
32+
declarativeconfig "github.com/operator-framework/operator-registry/alpha/declcfg"
33+
"github.com/operator-framework/operator-registry/pkg/containertools"
34+
registrybundle "github.com/operator-framework/operator-registry/pkg/lib/bundle"
2535
"github.com/operator-framework/operator-sdk/internal/olm/operator"
2636
"github.com/operator-framework/operator-sdk/internal/olm/operator/registry"
37+
registryutil "github.com/operator-framework/operator-sdk/internal/registry"
38+
)
39+
40+
const (
41+
schemaChannel = "olm.channel"
42+
schemaPackage = "olm.package"
43+
defaultChannel = "operator-sdk-run"
2744
)
2845

46+
type bundleDeclcfg struct {
47+
Package declcfg.Package
48+
Channel declcfg.Channel
49+
Bundle declcfg.Bundle
50+
}
51+
2952
type Install struct {
3053
BundleImage string
3154

@@ -35,6 +58,13 @@ type Install struct {
3558
cfg *operator.Configuration
3659
}
3760

61+
type FBCContext struct {
62+
Package string
63+
ChannelName string
64+
Refs []string
65+
ChannelEntry declarativeconfig.ChannelEntry
66+
}
67+
3868
func NewInstall(cfg *operator.Configuration) Install {
3969
i := Install{
4070
OperatorInstaller: registry.NewOperatorInstaller(cfg),
@@ -87,6 +117,46 @@ func (i *Install) setup(ctx context.Context) error {
87117
return err
88118
}
89119

120+
// get index image labels.
121+
catalogLabels, err := registryutil.GetImageLabels(ctx, nil, i.IndexImageCatalogCreator.IndexImage, false)
122+
if err != nil {
123+
return fmt.Errorf("get index image labels: %v", err)
124+
}
125+
126+
// set the field to true if FBC label is on the image or for a default index image.
127+
if _, hasFBCLabel := catalogLabels[containertools.ConfigsLocationLabel]; hasFBCLabel || i.IndexImageCatalogCreator.IndexImage == registry.DefaultIndexImage {
128+
i.IndexImageCatalogCreator.HasFBCLabel = true
129+
} else {
130+
// index image is of the SQLite index format.
131+
deprecationMsg := fmt.Sprintf("%s is a SQLite index image. SQLite based index images are being deprecated and will be removed in a future release, please migrate your catalogs to the new File-Based Catalog format", i.IndexImageCatalogCreator.IndexImage)
132+
log.Warn(deprecationMsg)
133+
}
134+
135+
if i.IndexImageCatalogCreator.HasFBCLabel {
136+
// FBC variables
137+
f := &FBCContext{
138+
Package: labels[registrybundle.PackageLabel],
139+
Refs: []string{i.BundleImage},
140+
ChannelEntry: declarativeconfig.ChannelEntry{
141+
Name: csv.Name,
142+
},
143+
}
144+
145+
if _, hasChannelMetadata := labels[registrybundle.ChannelsLabel]; hasChannelMetadata {
146+
f.ChannelName = strings.Split(labels[registrybundle.ChannelsLabel], ",")[0]
147+
} else {
148+
f.ChannelName = defaultChannel
149+
}
150+
151+
// generate an fbc if an fbc specific label is found on the image or for a default index image.
152+
content, err := f.generateFBCContent(ctx, i.BundleImage, i.IndexImageCatalogCreator.IndexImage)
153+
if err != nil {
154+
return fmt.Errorf("error generating File-Based Catalog with bundle %q: %v", i.BundleImage, err)
155+
}
156+
157+
i.IndexImageCatalogCreator.FBCContent = content
158+
}
159+
90160
i.OperatorInstaller.PackageName = labels[registrybundle.PackageLabel]
91161
i.OperatorInstaller.CatalogSourceName = operator.CatalogNameForPackage(i.OperatorInstaller.PackageName)
92162
i.OperatorInstaller.StartingCSV = csv.Name
@@ -98,3 +168,156 @@ func (i *Install) setup(ctx context.Context) error {
98168

99169
return nil
100170
}
171+
172+
func (f *FBCContext) generateFBCContent(ctx context.Context, bundleImage, indexImage string) (string, error) {
173+
log.Infof("Creating a File-Based Catalog of the bundle %q", bundleImage)
174+
// generate a File-Based Catalog representation of the bundle image
175+
bundleDeclcfg, err := f.createFBC(ctx)
176+
if err != nil {
177+
return "", fmt.Errorf("error creating a File-Based Catalog with image %q: %v", bundleImage, err)
178+
}
179+
180+
declcfg := &declarativeconfig.DeclarativeConfig{
181+
Bundles: []declarativeconfig.Bundle{bundleDeclcfg.Bundle},
182+
Packages: []declarativeconfig.Package{bundleDeclcfg.Package},
183+
Channels: []declarativeconfig.Channel{bundleDeclcfg.Channel},
184+
}
185+
186+
if indexImage != registry.DefaultIndexImage { // non-default index image was specified.
187+
// since an index image is specified, the bundle image will be added to the index image.
188+
// addBundleToIndexImage will ensure that the bundle is not already present in the index image and error out if it does.
189+
declcfg, err = addBundleToIndexImage(ctx, indexImage, bundleDeclcfg)
190+
if err != nil {
191+
return "", fmt.Errorf("error adding bundle image %q to index image %q: %v", bundleImage, indexImage, err)
192+
}
193+
}
194+
195+
// validate the generated File-Based Catalog
196+
if err = validateFBC(declcfg); err != nil {
197+
return "", fmt.Errorf("error validating the generated FBC: %v", err)
198+
}
199+
200+
// convert declarative config to string
201+
content, err := stringifyDecConfig(declcfg)
202+
if err != nil {
203+
return "", fmt.Errorf("error converting the declarative config to string: %v", err)
204+
}
205+
206+
if content == "" {
207+
return "", errors.New("file based catalog contents cannot be empty")
208+
}
209+
210+
log.Infof("Generated a valid File-Based Catalog")
211+
212+
return content, nil
213+
}
214+
215+
/// addBundleToIndexImage adds the bundle to an existing index image if the bundle is not already present in the index image.
216+
func addBundleToIndexImage(ctx context.Context, indexImage string, bundleDeclConfig bundleDeclcfg) (*declarativeconfig.DeclarativeConfig, error) {
217+
log.Infof("Rendering a File-Based Catalog of the Index Image %q", indexImage)
218+
log.SetOutput(ioutil.Discard)
219+
render := action.Render{
220+
Refs: []string{indexImage},
221+
}
222+
223+
imageDeclConfig, err := render.Run(ctx)
224+
log.SetOutput(os.Stdout)
225+
if err != nil {
226+
return nil, fmt.Errorf("error rendering the index image %q: %v", indexImage, err)
227+
}
228+
229+
for _, bundle := range imageDeclConfig.Bundles {
230+
if bundle.Name == bundleDeclConfig.Bundle.Name && bundle.Package == bundleDeclConfig.Bundle.Package {
231+
return nil, fmt.Errorf("bundle %q already exists in the index image: %s", bundleDeclConfig.Bundle.Name, indexImage)
232+
}
233+
}
234+
235+
for _, channel := range imageDeclConfig.Channels {
236+
if channel.Name == bundleDeclConfig.Channel.Name && channel.Package == bundleDeclConfig.Bundle.Package {
237+
return nil, fmt.Errorf("channel %q already exists in the index image: %s", bundleDeclConfig.Channel.Name, indexImage)
238+
}
239+
}
240+
241+
var isPackagePresent bool
242+
for _, pkg := range imageDeclConfig.Packages {
243+
if pkg.Name == bundleDeclConfig.Package.Name {
244+
isPackagePresent = true
245+
break
246+
}
247+
}
248+
249+
imageDeclConfig.Bundles = append(imageDeclConfig.Bundles, bundleDeclConfig.Bundle)
250+
imageDeclConfig.Channels = append(imageDeclConfig.Channels, bundleDeclConfig.Channel)
251+
252+
if !isPackagePresent {
253+
imageDeclConfig.Packages = append(imageDeclConfig.Packages, bundleDeclConfig.Package)
254+
}
255+
256+
log.Infof("Inserted the new bundle %q into the index image: %s", bundleDeclConfig.Bundle.Name, indexImage)
257+
258+
return imageDeclConfig, nil
259+
}
260+
261+
// createFBC generates an FBC by creating bundle, package and channel blobs.
262+
func (f *FBCContext) createFBC(ctx context.Context) (bundleDeclcfg, error) {
263+
var bundleDC bundleDeclcfg
264+
// Rendering the bundle image into a declarative config format.
265+
log.SetOutput(ioutil.Discard)
266+
render := action.Render{
267+
Refs: f.Refs,
268+
}
269+
270+
// generate bundles by rendering the bundle objects.
271+
declcfg, err := render.Run(ctx)
272+
log.SetOutput(os.Stdout)
273+
if err != nil {
274+
return bundleDeclcfg{}, fmt.Errorf("error rendering the bundle image: %v", err)
275+
}
276+
277+
// Ensuring a valid bundle size.
278+
if len(declcfg.Bundles) != 1 {
279+
return bundleDeclcfg{}, fmt.Errorf("bundle image should contain exactly one bundle blob")
280+
}
281+
282+
bundleDC.Bundle = declcfg.Bundles[0]
283+
284+
// generate package.
285+
bundleDC.Package = declarativeconfig.Package{
286+
Schema: schemaPackage,
287+
Name: f.Package,
288+
DefaultChannel: f.ChannelName,
289+
}
290+
291+
// generate channel.
292+
bundleDC.Channel = declarativeconfig.Channel{
293+
Schema: schemaChannel,
294+
Name: f.ChannelName,
295+
Package: f.Package,
296+
Entries: []declarativeconfig.ChannelEntry{f.ChannelEntry},
297+
}
298+
299+
return bundleDC, nil
300+
}
301+
302+
// stringifyDecConfig writes the generated declarative config to a string.
303+
func stringifyDecConfig(declcfg *declarativeconfig.DeclarativeConfig) (string, error) {
304+
var buf bytes.Buffer
305+
err := declarativeconfig.WriteYAML(*declcfg, &buf)
306+
if err != nil {
307+
return "", fmt.Errorf("error writing generated declarative config to JSON encoder: %v", err)
308+
}
309+
310+
return buf.String(), nil
311+
}
312+
313+
// validateFBC converts the generated declarative config to a model and validates it.
314+
func validateFBC(declcfg *declarativeconfig.DeclarativeConfig) error {
315+
// validates and converts declarative config to model
316+
_, err := declarativeconfig.ConvertToModel(*declcfg)
317+
if err != nil {
318+
return fmt.Errorf("error converting the declarative config to model: %v", err)
319+
320+
}
321+
322+
return nil
323+
}

0 commit comments

Comments
 (0)