Skip to content

Commit 0155dd4

Browse files
committed
✨ Allow configuring more granular cache filtering
This change implements most of the [cache options design][0], in particular it is now possible to: * Configure Namespaces per type * Configure filtering per namespace What is still missing is to allow configuring namespace-level default settings that will be used for all namespaces not explicitly configured. There is nothing in the way of doing that as a follow-up. The implementation slightly derivates from the design document in that the filter settings in `ByGVK` do not use the `Config` struct but instead are directly embedded. This allows us to not break compatibility and is more ergonomic to use. The implementation is based on a new and internal per-type delegating "meta cache". [0]: https://github.com/kubernetes-sigs/controller-runtime/blob/main/designs/cache_options.md
1 parent 21779fb commit 0155dd4

File tree

12 files changed

+968
-248
lines changed

12 files changed

+968
-248
lines changed

examples/scratch-env/go.sum

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1251,6 +1251,7 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0
12511251
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
12521252
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
12531253
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA=
1254+
golang.org/x/exp v0.0.0-20220827204233-334a2380cb91 h1:tnebWN09GYg9OLPss1KXj8txwZc6X6uMr6VFdcGNbHw=
12541255
golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
12551256
golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
12561257
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=

go.mod

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,14 @@ require (
99
github.com/go-logr/logr v1.2.4
1010
github.com/go-logr/zapr v1.2.4
1111
github.com/google/go-cmp v0.5.9
12+
github.com/google/gofuzz v1.2.0
1213
github.com/onsi/ginkgo/v2 v2.11.0
1314
github.com/onsi/gomega v1.27.10
1415
github.com/prometheus/client_golang v1.16.0
1516
github.com/prometheus/client_model v0.4.0
1617
go.uber.org/goleak v1.2.1
1718
go.uber.org/zap v1.24.0
19+
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e
1820
golang.org/x/sys v0.10.0
1921
gomodules.xyz/jsonpatch/v2 v2.3.0
2022
k8s.io/api v0.28.0-beta.0
@@ -40,7 +42,6 @@ require (
4042
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
4143
github.com/golang/protobuf v1.5.3 // indirect
4244
github.com/google/gnostic-models v0.6.8 // indirect
43-
github.com/google/gofuzz v1.2.0 // indirect
4445
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
4546
github.com/google/uuid v1.3.0 // indirect
4647
github.com/imdario/mergo v0.3.6 // indirect

go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,8 @@ go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
128128
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
129129
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
130130
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
131+
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA=
132+
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA=
131133
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
132134
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
133135
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=

pkg/cache/cache.go

Lines changed: 188 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -22,15 +22,16 @@ import (
2222
"net/http"
2323
"time"
2424

25+
corev1 "k8s.io/api/core/v1"
2526
"k8s.io/apimachinery/pkg/api/meta"
26-
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2727
"k8s.io/apimachinery/pkg/fields"
2828
"k8s.io/apimachinery/pkg/labels"
2929
"k8s.io/apimachinery/pkg/runtime"
3030
"k8s.io/apimachinery/pkg/runtime/schema"
3131
"k8s.io/client-go/kubernetes/scheme"
3232
"k8s.io/client-go/rest"
3333
toolscache "k8s.io/client-go/tools/cache"
34+
"k8s.io/utils/pointer"
3435

3536
"sigs.k8s.io/controller-runtime/pkg/cache/internal"
3637
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -144,9 +145,13 @@ type Options struct {
144145
// instead of `reconcile.Result{}`.
145146
SyncPeriod *time.Duration
146147

147-
// Namespaces restricts the cache's ListWatch to the desired namespaces
148-
// Per default ListWatch watches all namespaces
149-
Namespaces []string
148+
// DefaultNamespaces maps namespace names to cache configs. If set, only
149+
// the namespaces in here will be watched and it will by used to default
150+
// ByObject.Namespaces for all objects if that is nil.
151+
//
152+
// The options in the Config that are nil will be defaulted from
153+
// the respective Default* settings.
154+
DefaultNamespaces map[string]Config
150155

151156
// DefaultLabelSelector will be used as a label selector for all object types
152157
// unless they have a more specific selector set in ByObject.
@@ -160,20 +165,39 @@ type Options struct {
160165
// unless they have a more specific transform set in ByObject.
161166
DefaultTransform toolscache.TransformFunc
162167

163-
// UnsafeDisableDeepCopy indicates not to deep copy objects during get or
164-
// list objects for EVERY object.
168+
// DefaultUnsafeDisableDeepCopy is the default for UnsafeDisableDeepCopy
169+
// for everything that doesn't specify this.
170+
//
165171
// Be very careful with this, when enabled you must DeepCopy any object before mutating it,
166172
// otherwise you will mutate the object in the cache.
167173
//
168174
// This is a global setting for all objects, and can be overridden by the ByObject setting.
169-
UnsafeDisableDeepCopy *bool
175+
DefaultUnsafeDisableDeepCopy *bool
170176

171177
// ByObject restricts the cache's ListWatch to the desired fields per GVK at the specified object.
178+
// object, this will fall through to Default* settings.
172179
ByObject map[client.Object]ByObject
173180
}
174181

175182
// ByObject offers more fine-grained control over the cache's ListWatch by object.
176183
type ByObject struct {
184+
// Namespaces maps a namespace name to cache configs. If set, only the
185+
// namespaces in this map will be cached.
186+
//
187+
// Settings in the map value that are unset will be defaulted.
188+
// Use an empty value for the specific setting to prevent that.
189+
//
190+
// A nil map allows to default this to the cache's DefaultNamespaces setting.
191+
// An empty map prevents this and means that all namespaces will be cached.
192+
//
193+
// The defaulting follows the following precedence order:
194+
// 1. ByObject
195+
// 2. DefaultNamespaces[namespace]
196+
// 3. Default*
197+
//
198+
// This must be unset for cluster-scoped objects.
199+
Namespaces map[string]Config
200+
177201
// Label represents a label selector for the object.
178202
Label labels.Selector
179203

@@ -194,48 +218,118 @@ type ByObject struct {
194218
UnsafeDisableDeepCopy *bool
195219
}
196220

221+
// Config describes all potential options for a given watch.
222+
type Config struct {
223+
// LabelSelector specifies a label selector. A nil value allows to
224+
// default this.
225+
//
226+
// Set to labels.Everything() if you don't want this defaulted.
227+
LabelSelector labels.Selector
228+
229+
// FieldSelector specifics a field selector. A nil value allows to
230+
// default this.
231+
//
232+
// Set to fields.Everything() if you don't want this defaulted.
233+
FieldSelector fields.Selector
234+
235+
// Transform specifies a transform func. A nil value allows to default
236+
// this.
237+
//
238+
// Set to an empty func to prevent this:
239+
// func(in interface{}) (interface{}, error) { return in, nil }
240+
Transform toolscache.TransformFunc
241+
242+
// UnsafeDisableDeepCopy specifies if List and Get requests against the
243+
// cache should not DeepCopy. A nil value allows to default this.
244+
UnsafeDisableDeepCopy *bool
245+
}
246+
197247
// NewCacheFunc - Function for creating a new cache from the options and a rest config.
198248
type NewCacheFunc func(config *rest.Config, opts Options) (Cache, error)
199249

200250
// New initializes and returns a new Cache.
201-
func New(config *rest.Config, opts Options) (Cache, error) {
202-
if len(opts.Namespaces) == 0 {
203-
opts.Namespaces = []string{metav1.NamespaceAll}
251+
func New(cfg *rest.Config, opts Options) (Cache, error) {
252+
opts, err := defaultOpts(cfg, opts)
253+
if err != nil {
254+
return nil, err
204255
}
205-
if len(opts.Namespaces) > 1 {
206-
return newMultiNamespaceCache(config, opts)
256+
257+
newCacheFunc := newCache(cfg, opts)
258+
259+
var defaultCache Cache
260+
if len(opts.DefaultNamespaces) > 0 {
261+
defaultConfig := optionDefaultsToConfig(&opts)
262+
defaultCache = newMultiNamespaceCache(newCacheFunc, opts.Scheme, opts.Mapper, opts.DefaultNamespaces, &defaultConfig)
263+
} else {
264+
defaultCache = newCacheFunc(optionDefaultsToConfig(&opts), corev1.NamespaceAll)
207265
}
208266

209-
opts, err := defaultOpts(config, opts)
210-
if err != nil {
211-
return nil, err
267+
if len(opts.ByObject) == 0 {
268+
return defaultCache, nil
212269
}
213270

214-
byGVK, err := convertToInformerOptsByGVK(opts.ByObject, opts.Scheme)
215-
if err != nil {
216-
return nil, err
271+
delegating := &delegatingByTypeCache{
272+
scheme: opts.Scheme,
273+
caches: make(map[schema.GroupVersionKind]Cache, len(opts.ByObject)),
274+
defaultCache: defaultCache,
275+
}
276+
277+
for obj, config := range opts.ByObject {
278+
gvk, err := apiutil.GVKForObject(obj, opts.Scheme)
279+
if err != nil {
280+
return nil, fmt.Errorf("failed to get GVK for type %T: %w", obj, err)
281+
}
282+
var cache Cache
283+
if len(config.Namespaces) > 0 {
284+
cache = newMultiNamespaceCache(newCacheFunc, opts.Scheme, opts.Mapper, config.Namespaces, nil)
285+
} else {
286+
cache = newCacheFunc(byObjectToConfig(config), corev1.NamespaceAll)
287+
}
288+
delegating.caches[gvk] = cache
217289
}
218-
// Set the default selector and transform.
219-
byGVK[schema.GroupVersionKind{}] = internal.InformersOptsByGVK{
220-
Selector: internal.Selector{
221-
Label: opts.DefaultLabelSelector,
222-
Field: opts.DefaultFieldSelector,
223-
},
290+
291+
return delegating, nil
292+
}
293+
294+
func optionDefaultsToConfig(opts *Options) Config {
295+
return Config{
296+
LabelSelector: opts.DefaultLabelSelector,
297+
FieldSelector: opts.DefaultFieldSelector,
224298
Transform: opts.DefaultTransform,
225-
UnsafeDisableDeepCopy: opts.UnsafeDisableDeepCopy,
299+
UnsafeDisableDeepCopy: opts.DefaultUnsafeDisableDeepCopy,
300+
}
301+
}
302+
303+
func byObjectToConfig(byObject ByObject) Config {
304+
return Config{
305+
LabelSelector: byObject.Label,
306+
FieldSelector: byObject.Field,
307+
Transform: byObject.Transform,
308+
UnsafeDisableDeepCopy: byObject.UnsafeDisableDeepCopy,
226309
}
310+
}
227311

228-
return &informerCache{
229-
scheme: opts.Scheme,
230-
Informers: internal.NewInformers(config, &internal.InformersOpts{
231-
HTTPClient: opts.HTTPClient,
232-
Scheme: opts.Scheme,
233-
Mapper: opts.Mapper,
234-
ResyncPeriod: *opts.SyncPeriod,
235-
Namespace: opts.Namespaces[0],
236-
ByGVK: byGVK,
237-
}),
238-
}, nil
312+
type newCacheFunc func(config Config, namespace string) Cache
313+
314+
func newCache(restConfig *rest.Config, opts Options) newCacheFunc {
315+
return func(config Config, namespace string) Cache {
316+
return &informerCache{
317+
scheme: opts.Scheme,
318+
Informers: internal.NewInformers(restConfig, &internal.InformersOpts{
319+
HTTPClient: opts.HTTPClient,
320+
Scheme: opts.Scheme,
321+
Mapper: opts.Mapper,
322+
ResyncPeriod: *opts.SyncPeriod,
323+
Namespace: namespace,
324+
Selector: internal.Selector{
325+
Label: config.LabelSelector,
326+
Field: config.FieldSelector,
327+
},
328+
Transform: config.Transform,
329+
UnsafeDisableDeepCopy: pointer.BoolDeref(config.UnsafeDisableDeepCopy, false),
330+
}),
331+
}
332+
}
239333
}
240334

241335
func defaultOpts(config *rest.Config, opts Options) (Options, error) {
@@ -262,31 +356,70 @@ func defaultOpts(config *rest.Config, opts Options) (Options, error) {
262356
}
263357
}
264358

359+
for namespace, cfg := range opts.DefaultNamespaces {
360+
cfg = defaultConfig(cfg, optionDefaultsToConfig(&opts))
361+
opts.DefaultNamespaces[namespace] = cfg
362+
}
363+
364+
for obj, byObject := range opts.ByObject {
365+
isNamespaced, err := apiutil.IsObjectNamespaced(obj, opts.Scheme, opts.Mapper)
366+
if err != nil {
367+
return opts, fmt.Errorf("failed to determine if %T is namespaced: %w", obj, err)
368+
}
369+
if !isNamespaced && byObject.Namespaces != nil {
370+
return opts, fmt.Errorf("type %T is not namespaced, but its ByObject.Namespaces setting is not nil", obj)
371+
}
372+
373+
// Default the namespace-level configs first, because they need to use the undefaulted type-level config.
374+
for namespace, config := range byObject.Namespaces {
375+
// 1. Default from the undefaulted type-level config
376+
config = defaultConfig(config, byObjectToConfig(byObject))
377+
378+
// 2. Default from the namespace-level config. This was defaulted from the global default config earlier, but
379+
// might not have an entry for the current namespace.
380+
if defaultNamespaceSettings, hasDefaultNamespace := opts.DefaultNamespaces[namespace]; hasDefaultNamespace {
381+
config = defaultConfig(config, defaultNamespaceSettings)
382+
}
383+
384+
// 3. Default from the global defaults
385+
config = defaultConfig(config, optionDefaultsToConfig(&opts))
386+
387+
byObject.Namespaces[namespace] = config
388+
}
389+
390+
defaultedConfig := defaultConfig(byObjectToConfig(byObject), optionDefaultsToConfig(&opts))
391+
byObject.Label = defaultedConfig.LabelSelector
392+
byObject.Field = defaultedConfig.FieldSelector
393+
byObject.Transform = defaultedConfig.Transform
394+
byObject.UnsafeDisableDeepCopy = defaultedConfig.UnsafeDisableDeepCopy
395+
396+
if byObject.Namespaces == nil {
397+
byObject.Namespaces = opts.DefaultNamespaces
398+
}
399+
400+
opts.ByObject[obj] = byObject
401+
}
402+
265403
// Default the resync period to 10 hours if unset
266404
if opts.SyncPeriod == nil {
267405
opts.SyncPeriod = &defaultSyncPeriod
268406
}
269407
return opts, nil
270408
}
271409

272-
func convertToInformerOptsByGVK(in map[client.Object]ByObject, scheme *runtime.Scheme) (map[schema.GroupVersionKind]internal.InformersOptsByGVK, error) {
273-
out := map[schema.GroupVersionKind]internal.InformersOptsByGVK{}
274-
for object, byObject := range in {
275-
gvk, err := apiutil.GVKForObject(object, scheme)
276-
if err != nil {
277-
return nil, err
278-
}
279-
if _, ok := out[gvk]; ok {
280-
return nil, fmt.Errorf("duplicate cache options for GVK %v, cache.Options.ByObject has multiple types with the same GroupVersionKind", gvk)
281-
}
282-
out[gvk] = internal.InformersOptsByGVK{
283-
Selector: internal.Selector{
284-
Field: byObject.Field,
285-
Label: byObject.Label,
286-
},
287-
Transform: byObject.Transform,
288-
UnsafeDisableDeepCopy: byObject.UnsafeDisableDeepCopy,
289-
}
410+
func defaultConfig(toDefault, defaultFrom Config) Config {
411+
if toDefault.LabelSelector == nil {
412+
toDefault.LabelSelector = defaultFrom.LabelSelector
290413
}
291-
return out, nil
414+
if toDefault.FieldSelector == nil {
415+
toDefault.FieldSelector = defaultFrom.FieldSelector
416+
}
417+
if toDefault.Transform == nil {
418+
toDefault.Transform = defaultFrom.Transform
419+
}
420+
if toDefault.UnsafeDisableDeepCopy == nil {
421+
toDefault.UnsafeDisableDeepCopy = defaultFrom.UnsafeDisableDeepCopy
422+
}
423+
424+
return toDefault
292425
}

0 commit comments

Comments
 (0)