Add Catalog Source priority for dependency resolution

Signed-off-by: Harish <[email protected]>

Author: harishsurf

deploy/chart/crds/0000_50_olm_00-catalogsources.crd.yaml

@@ -90,6 +90,17 @@ spec:
                   a registry-server with. Only used when SourceType = SourceTypeGrpc.
                   If present, the address field is ignored.
                 type: string
+              priority:
+                description: 'Priority field assigns a weight to the catalog source
+                  to prioritize them so that it can be consumed by the dependency
+                  resolver. Usage: Higher weight indicates that this catalog source
+                  is preferred over lower weighted catalog sources during dependency
+                  resolution. The range of the priority value can go from positive
+                  to negative in the range of int32. The default value to a catalog
+                  source with unassigned priority would be 0. The catalog source with
+                  the same priority values will be ranked lexicographically based
+                  on its name.'
+                type: integer
               publisher:
                 type: string
               secrets:

deploy/chart/crds/0000_50_olm_00-clusterserviceversions.crd.yaml

@@ -662,13 +662,19 @@ spec:
                       deployments:
                         type: array
                         items:
-                          description: StrategyDeploymentSpec contains the name and
-                            spec for the deployment ALM should create
+                          description: StrategyDeploymentSpec contains the name, spec
+                            and labels for the deployment ALM should create
                           type: object
                           required:
                           - name
                           - spec
                           properties:
+                            label:
+                              description: Set is a map of label:value. It implements
+                                Labels.
+                              type: object
+                              additionalProperties:
+                                type: string
                             name:
                               type: string
                             spec:
@@ -8698,6 +8704,10 @@ spec:
                     containerPort:
                       type: integer
                       format: int32
+                    conversionCRDs:
+                      type: array
+                      items:
+                        type: string
                     deploymentName:
                       type: string
                     failurePolicy:

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/onsi/gomega v1.9.0
 	github.com/openshift/api v0.0.0-20200331152225-585af27e34fd
 	github.com/openshift/client-go v0.0.0-20200326155132-2a6cd50aedd0
-	github.com/operator-framework/api v0.3.7
+	github.com/operator-framework/api v0.3.9
 	github.com/operator-framework/operator-registry v1.13.3
 	github.com/otiai10/copy v1.2.0
 	github.com/pkg/errors v0.9.1

go.sum

@@ -585,8 +585,8 @@ github.com/openshift/client-go v0.0.0-20200326155132-2a6cd50aedd0 h1:kMiuiZXH1Gd
 github.com/openshift/client-go v0.0.0-20200326155132-2a6cd50aedd0/go.mod h1:uUQ4LClRO+fg5MF/P6QxjMCb1C9f7Oh4RKepftDnEJE=
 github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
 github.com/operator-framework/api v0.3.7-0.20200602203552-431198de9fc2/go.mod h1:Xbje9x0SHmh0nihE21kpesB38vk3cyxnE6JdDS8Jo1Q=
-github.com/operator-framework/api v0.3.7 h1:gTER+WQdjRAegxKdcoylNqHVGGnBRAAgC2zS3CZL78g=
-github.com/operator-framework/api v0.3.7/go.mod h1:Xbje9x0SHmh0nihE21kpesB38vk3cyxnE6JdDS8Jo1Q=
+github.com/operator-framework/api v0.3.9 h1:ggRW5C/7MLmA4cOJ2mjEZ5RqDMsIeMknd6vx15S7Xo4=
+github.com/operator-framework/api v0.3.9/go.mod h1:Xbje9x0SHmh0nihE21kpesB38vk3cyxnE6JdDS8Jo1Q=
 github.com/operator-framework/operator-registry v1.13.3 h1:SaZ1IKLKGizVgTtT8AlDgaMXFYOiIPxRf8KLve0/DXM=
 github.com/operator-framework/operator-registry v1.13.3/go.mod h1:YhnIzOVjRU2ZwZtzt+fjcjW8ujJaSFynBEu7QVKaSdU=
 github.com/otiai10/copy v1.2.0 h1:HvG945u96iNadPoG2/Ja2+AUJeW5YuFQMixq9yirC+k=

pkg/controller/operators/catalog/operator.go

@@ -865,7 +865,7 @@ func (o *Operator) syncResolvingNamespace(obj interface{}) error {
 	// resolve a set of steps to apply to a cluster, a set of subscriptions to create/update, and any errors
 	steps, bundleLookups, updatedSubs, err := o.resolver.ResolveSteps(namespace, querier)
 	if err != nil {
-		go o.recorder.Event(ns, corev1.EventTypeWarning,"ResolutionFailed", err.Error())
+		go o.recorder.Event(ns, corev1.EventTypeWarning, "ResolutionFailed", err.Error())
 		return err
 	}
 

pkg/controller/registry/resolver/cache.go

@@ -9,12 +9,12 @@ import (
 	"time"
 
 	"github.com/blang/semver"
+	v1alpha1listers "github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/listers/operators/v1alpha1"
+	"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/registry"
 	"github.com/operator-framework/operator-registry/pkg/api"
 	"github.com/operator-framework/operator-registry/pkg/client"
 	opregistry "github.com/operator-framework/operator-registry/pkg/registry"
 	"github.com/sirupsen/logrus"
-
-	"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/registry"
 )
 
 type RegistryClientProvider interface {
@@ -43,27 +43,31 @@ type OperatorCacheProvider interface {
 }
 
 type OperatorCache struct {
-	logger    logrus.FieldLogger
-	rcp       RegistryClientProvider
-	snapshots map[registry.CatalogKey]*CatalogSnapshot
-	ttl       time.Duration
-	sem       chan struct{}
-	m         sync.RWMutex
+	logger       logrus.FieldLogger
+	rcp          RegistryClientProvider
+	catsrcLister v1alpha1listers.CatalogSourceLister
+	snapshots    map[registry.CatalogKey]*CatalogSnapshot
+	ttl          time.Duration
+	sem          chan struct{}
+	m            sync.RWMutex
 }
 
+type catalogPriority int
+
 var _ OperatorCacheProvider = &OperatorCache{}
 
-func NewOperatorCache(rcp RegistryClientProvider, log logrus.FieldLogger) *OperatorCache {
+func NewOperatorCache(rcp RegistryClientProvider, log logrus.FieldLogger, catsrcLister v1alpha1listers.CatalogSourceLister) *OperatorCache {
 	const (
 		MaxConcurrentSnapshotUpdates = 4
 	)
 
 	return &OperatorCache{
-		logger:    log,
-		rcp:       rcp,
-		snapshots: make(map[registry.CatalogKey]*CatalogSnapshot),
-		ttl:       5 * time.Minute,
-		sem:       make(chan struct{}, MaxConcurrentSnapshotUpdates),
+		logger:       log,
+		rcp:          rcp,
+		catsrcLister: catsrcLister,
+		snapshots:    make(map[registry.CatalogKey]*CatalogSnapshot),
+		ttl:          5 * time.Minute,
+		sem:          make(chan struct{}, MaxConcurrentSnapshotUpdates),
 	}
 }
 
@@ -151,11 +155,20 @@ func (c *OperatorCache) Namespaced(namespaces ...string) MultiCatalogOperatorFin
 
 	for _, miss := range misses {
 		ctx, cancel := context.WithTimeout(context.Background(), CachePopulateTimeout)
+
+		catsrcPriority := 0
+		// Ignoring error and treat catsrc priority as 0 if not found.
+		catsrc, err := c.catsrcLister.CatalogSources(miss.Namespace).Get(miss.Name)
+		if err == nil {
+			catsrcPriority = catsrc.Spec.Priority
+		}
+
 		s := CatalogSnapshot{
-			logger: c.logger.WithField("catalog", miss),
-			key:    miss,
-			expiry: now.Add(c.ttl),
-			pop:    cancel,
+			logger:   c.logger.WithField("catalog", miss),
+			key:      miss,
+			expiry:   now.Add(c.ttl),
+			pop:      cancel,
+			priority: catalogPriority(catsrcPriority),
 		}
 		s.m.Lock()
 		c.snapshots[miss] = &s
@@ -222,13 +235,13 @@ func ensurePackageProperty(o *Operator, name, version string) {
 		PackageName: name,
 		Version:     version,
 	}
-	byte, err := json.Marshal(prop)
+	bytes, err := json.Marshal(prop)
 	if err != nil {
 		return
 	}
 	o.properties = append(o.properties, &api.Property{
 		Type:  opregistry.PackageType,
-		Value: string(byte),
+		Value: string(bytes),
 	})
 }
 
@@ -277,6 +290,7 @@ type CatalogSnapshot struct {
 	operators []*Operator
 	m         sync.RWMutex
 	pop       context.CancelFunc
+	priority  catalogPriority
 }
 
 func (s *CatalogSnapshot) Cancel() {
@@ -354,10 +368,14 @@ func (s SortableSnapshots) Less(i, j int) bool {
 		return false
 	}
 
-	// the rest are sorted first in namespace preference order, then by name
+	// the rest are sorted first on priority, namespace and then by name
+	if s.snapshots[i].priority != s.snapshots[j].priority {
+		return s.snapshots[i].priority > s.snapshots[j].priority
+	}
 	if s.snapshots[i].key.Namespace != s.snapshots[j].key.Namespace {
 		return s.namespaces[s.snapshots[i].key.Namespace] < s.namespaces[s.snapshots[j].key.Namespace]
 	}
+
 	return s.snapshots[i].key.Name < s.snapshots[j].key.Name
 }
 

pkg/controller/registry/resolver/cache_test.go

@@ -13,11 +13,11 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/registry"
+	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorlister"
 	"github.com/operator-framework/operator-registry/pkg/api"
 	"github.com/operator-framework/operator-registry/pkg/client"
 	opregistry "github.com/operator-framework/operator-registry/pkg/registry"
-
-	"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/registry"
 )
 
 type BundleStreamStub struct {
@@ -83,8 +83,8 @@ func TestOperatorCacheConcurrency(t *testing.T) {
 	const (
 		NWorkers = 64
 	)
-
 	rcp := RegistryClientProviderStub{}
+	catsrcLister := operatorlister.NewLister().OperatorsV1alpha1().CatalogSourceLister()
 	var keys []registry.CatalogKey
 	for i := 0; i < 128; i++ {
 		for j := 0; j < 8; j++ {
@@ -106,7 +106,7 @@ func TestOperatorCacheConcurrency(t *testing.T) {
 		}
 	}
 
-	c := NewOperatorCache(rcp, logrus.New())
+	c := NewOperatorCache(rcp, logrus.New(), catsrcLister)
 
 	errs := make(chan error)
 	for w := 0; w < NWorkers; w++ {
@@ -140,6 +140,7 @@ func TestOperatorCacheConcurrency(t *testing.T) {
 
 func TestOperatorCacheExpiration(t *testing.T) {
 	rcp := RegistryClientProviderStub{}
+	catsrcLister := operatorlister.NewLister().OperatorsV1alpha1().CatalogSourceLister()
 	key := registry.CatalogKey{Namespace: "dummynamespace", Name: "dummyname"}
 	rcp[key] = &RegistryClientStub{
 		BundleIterator: client.NewBundleIterator(&BundleStreamStub{
@@ -155,14 +156,15 @@ func TestOperatorCacheExpiration(t *testing.T) {
 		}),
 	}
 
-	c := NewOperatorCache(rcp, logrus.New())
+	c := NewOperatorCache(rcp, logrus.New(), catsrcLister)
 	c.ttl = 0 // instantly stale
 
 	require.Len(t, c.Namespaced("dummynamespace").Catalog(key).Find(WithCSVName("csvname")), 1)
 }
 
 func TestOperatorCacheReuse(t *testing.T) {
 	rcp := RegistryClientProviderStub{}
+	catsrcLister := operatorlister.NewLister().OperatorsV1alpha1().CatalogSourceLister()
 	key := registry.CatalogKey{Namespace: "dummynamespace", Name: "dummyname"}
 	rcp[key] = &RegistryClientStub{
 		BundleIterator: client.NewBundleIterator(&BundleStreamStub{
@@ -178,7 +180,7 @@ func TestOperatorCacheReuse(t *testing.T) {
 		}),
 	}
 
-	c := NewOperatorCache(rcp, logrus.New())
+	c := NewOperatorCache(rcp, logrus.New(), catsrcLister)
 
 	require.Len(t, c.Namespaced("dummynamespace").Catalog(key).Find(WithCSVName("csvname")), 1)
 }
@@ -290,6 +292,7 @@ func TestCatalogSnapshotFind(t *testing.T) {
 
 func TestStripPluralRequiredAndProvidedAPIKeys(t *testing.T) {
 	rcp := RegistryClientProviderStub{}
+	catsrcLister := operatorlister.NewLister().OperatorsV1alpha1().CatalogSourceLister()
 	key := registry.CatalogKey{Namespace: "testnamespace", Name: "testname"}
 	rcp[key] = &RegistryClientStub{
 		BundleIterator: client.NewBundleIterator(&BundleStreamStub{
@@ -327,7 +330,7 @@ func TestStripPluralRequiredAndProvidedAPIKeys(t *testing.T) {
 		}),
 	}
 
-	c := NewOperatorCache(rcp, logrus.New())
+	c := NewOperatorCache(rcp, logrus.New(), catsrcLister)
 
 	nc := c.Namespaced("testnamespace")
 	result, err := AtLeast(1, nc.Find(ProvidingAPI(opregistry.APIKey{Group: "g", Version: "v1", Kind: "K"})))

pkg/controller/registry/resolver/generation_test.go

@@ -9,7 +9,6 @@ import (
 	opregistry "github.com/operator-framework/operator-registry/pkg/registry"
 	"github.com/stretchr/testify/require"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
 )
 
 var NoVersion = semver.MustParse("0.0.0")

pkg/controller/registry/resolver/legacy_resolver.go

@@ -79,7 +79,6 @@ func (r *LegacyResolver) ResolveSteps(namespace string, sourceQuerier SourceQuer
 		return nil, nil, nil, err
 	}
 
-
 	// if there's no error, we were able to satisfy all constraints in the subscription set, so we calculate what
 	// changes to persist to the cluster and write them out as `steps`
 	steps := []*v1alpha1.Step{}

pkg/controller/registry/resolver/legacy_resolver_test.go

@@ -10,8 +10,6 @@ import (
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/lib/operatorlister"
 )
 
-
-
 func TestResolverLegacy(t *testing.T) {
 	namespace := "catsrc-namespace"
 	for _, tt := range SharedResolverSpecs() {

pkg/controller/registry/resolver/operators.go

@@ -258,7 +258,7 @@ func NewOperatorFromBundle(bundle *api.Bundle, startingCSV string, sourceKey reg
 
 	// legacy support - if the api doesn't contain properties/dependencies, build them from required/provided apis
 	properties := bundle.Properties
-	if properties == nil || len(properties) == 0{
+	if properties == nil || len(properties) == 0 {
 		properties, err = apisToProperties(provided)
 		if err != nil {
 			return nil, err
@@ -297,8 +297,6 @@ func NewOperatorFromBundle(bundle *api.Bundle, startingCSV string, sourceKey reg
 		return op, nil
 	}
 
-
-
 	return &Operator{
 		name:         bundle.CsvName,
 		replaces:     bundle.Replaces,

pkg/controller/registry/resolver/resolver.go

@@ -6,13 +6,14 @@ import (
 	"fmt"
 	"sort"
 
-	"github.com/operator-framework/api/pkg/operators/v1alpha1"
-	opregistry "github.com/operator-framework/operator-registry/pkg/registry"
 	"github.com/sirupsen/logrus"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 
+	"github.com/operator-framework/api/pkg/operators/v1alpha1"
+	v1alpha1listers "github.com/operator-framework/operator-lifecycle-manager/pkg/api/client/listers/operators/v1alpha1"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/registry"
 	"github.com/operator-framework/operator-lifecycle-manager/pkg/controller/registry/resolver/solver"
+	opregistry "github.com/operator-framework/operator-registry/pkg/registry"
 )
 
 type OperatorResolver interface {
@@ -24,9 +25,9 @@ type SatResolver struct {
 	log   logrus.FieldLogger
 }
 
-func NewDefaultSatResolver(rcp RegistryClientProvider, log logrus.FieldLogger) *SatResolver {
+func NewDefaultSatResolver(rcp RegistryClientProvider, catsrcLister v1alpha1listers.CatalogSourceLister, log logrus.FieldLogger) *SatResolver {
 	return &SatResolver{
-		cache: NewOperatorCache(rcp, log),
+		cache: NewOperatorCache(rcp, log, catsrcLister),
 		log:   log,
 	}
 }