Skip to content

Commit af1dc80

Browse files
joelanfordjoelanford
committed
multiNamespaceCache: support custom newCache funcs per namespace
Signed-off-by: Joe Lanford <[email protected]>
1 parent 6182d4b commit af1dc80

File tree

2 files changed

+261
-16
lines changed

2 files changed

+261
-16
lines changed

‎pkg/cache/cache_test.go

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,192 @@ var _ = Describe("Multi-Namespace Informer Cache", func() {
122122
var _ = Describe("Informer Cache without DeepCopy", func() {
123123
CacheTest(cache.New, cache.Options{UnsafeDisableDeepCopyByObject: cache.DisableDeepCopyByObject{cache.ObjectAll{}: true}})
124124
})
125+
var _ = Describe("ByNamespace Cache", func() {
126+
defer GinkgoRecover()
127+
var (
128+
informerCache cache.Cache
129+
informerCacheCtx context.Context
130+
informerCacheCancel context.CancelFunc
131+
pod1a client.Object
132+
pod1b client.Object
133+
pod2a client.Object
134+
pod2b client.Object
135+
pod2c client.Object
136+
pod3a client.Object
137+
pod3b client.Object
138+
)
139+
BeforeEach(func() {
140+
informerCacheCtx, informerCacheCancel = context.WithCancel(context.Background())
141+
Expect(cfg).NotTo(BeNil())
142+
cl, err := client.New(cfg, client.Options{})
143+
Expect(err).NotTo(HaveOccurred())
144+
err = ensureNamespace(testNamespaceOne, cl)
145+
Expect(err).NotTo(HaveOccurred())
146+
err = ensureNamespace(testNamespaceTwo, cl)
147+
Expect(err).NotTo(HaveOccurred())
148+
err = ensureNamespace(testNamespaceThree, cl)
149+
Expect(err).NotTo(HaveOccurred())
150+
err = ensureNode(testNodeOne, cl)
151+
Expect(err).NotTo(HaveOccurred())
152+
// namespace 1 stuff
153+
pod1a = createPod("pod-1a", testNamespaceOne, corev1.RestartPolicyNever) // matches (everything matches)
154+
pod1b = createPodWithLabels("pod-1b", testNamespaceOne, corev1.RestartPolicyNever, map[string]string{"other-match": "true"}) // matches (everything matches)
155+
// namespace 2 stuff
156+
pod2a = createPodWithLabels("pod-2a", testNamespaceTwo, corev1.RestartPolicyNever, map[string]string{"ns2-match": "false"}) // no match (does not match ns2 label selector)
157+
pod2b = createPodWithLabels("pod-2b", testNamespaceTwo, corev1.RestartPolicyNever, map[string]string{"ns2-match": "true"}) // matches (matches ns2 label selector)
158+
pod2c = createPodWithLabels("pod-2c", testNamespaceTwo, corev1.RestartPolicyNever, map[string]string{"other-match": "true"}) // no match (does not match ns2 label selector)
159+
// namespace 3 stuff
160+
pod3a = createPodWithLabels("pod-3a", testNamespaceThree, corev1.RestartPolicyNever, map[string]string{"other-match": "false"}) // no match (does not match default cache label selector)
161+
pod3b = createPodWithLabels("pod-3b", testNamespaceThree, corev1.RestartPolicyNever, map[string]string{"other-match": "true"}) // matches (matches default cache label selector)
162+
By("creating the informer cache")
163+
informerCache, err = cache.BuilderByNamespace(cache.ByNamespaceOptions{
164+
NewNamespaceCaches: map[string]cache.NewCacheFunc{
165+
// Everything in ns1
166+
testNamespaceOne: cache.New,
167+
// Only things in ns2 with label "ns2-match"="true"
168+
testNamespaceTwo: cache.BuilderWithOptions(cache.Options{
169+
DefaultSelector: cache.ObjectSelector{
170+
Label: labels.Set{"ns2-match": "true"}.AsSelector(),
171+
},
172+
}),
173+
},
174+
// For all other namespaces, match "other-match"="true"
175+
NewDefaultNamespaceCache: cache.BuilderWithOptions(cache.Options{
176+
DefaultSelector: cache.ObjectSelector{
177+
Label: labels.Set{"other-match": "true"}.AsSelector(),
178+
},
179+
}),
180+
// For cluster-scoped objects, only match metadata.name = "test-node-1"
181+
NewClusterCache: cache.BuilderWithOptions(cache.Options{
182+
DefaultSelector: cache.ObjectSelector{
183+
Field: fields.OneTermEqualSelector("metadata.name", testNodeOne),
184+
},
185+
}),
186+
})(cfg, cache.Options{})
187+
Expect(err).NotTo(HaveOccurred())
188+
By("running the cache and waiting for it to sync")
189+
// pass as an arg so that we don't race between close and re-assign
190+
go func(ctx context.Context) {
191+
defer GinkgoRecover()
192+
Expect(informerCache.Start(ctx)).To(Succeed())
193+
}(informerCacheCtx)
194+
Expect(informerCache.WaitForCacheSync(informerCacheCtx)).To(BeTrue())
195+
})
196+
Describe("Get", func() {
197+
It("should get an item from a namespace cache", func() {
198+
pod := &corev1.Pod{}
199+
err := informerCache.Get(informerCacheCtx, client.ObjectKeyFromObject(pod1a), pod)
200+
Expect(err).NotTo(HaveOccurred())
201+
})
202+
It("should get an item from the default namespace cache", func() {
203+
pod := &corev1.Pod{}
204+
err := informerCache.Get(informerCacheCtx, client.ObjectKeyFromObject(pod3b), pod)
205+
Expect(err).NotTo(HaveOccurred())
206+
})
207+
It("should get a cluster-scoped item", func() {
208+
node := &corev1.Node{}
209+
err := informerCache.Get(informerCacheCtx, client.ObjectKey{Name: testNodeOne}, node)
210+
Expect(err).NotTo(HaveOccurred())
211+
})
212+
It("should not find an item from a namespace-specific cache if it is not matched", func() {
213+
pod := &corev1.Pod{}
214+
err := informerCache.Get(informerCacheCtx, client.ObjectKeyFromObject(pod2a), pod)
215+
Expect(apierrors.IsNotFound(err)).To(BeTrue())
216+
})
217+
It("should not find an item from the default namespace cache if it is not matched", func() {
218+
pod := &corev1.Pod{}
219+
err := informerCache.Get(informerCacheCtx, client.ObjectKeyFromObject(pod3a), pod)
220+
Expect(apierrors.IsNotFound(err)).To(BeTrue())
221+
})
222+
It("should not find an item at the cluster-scope if it is not matched", func() {
223+
ns := &corev1.Namespace{}
224+
err := informerCache.Get(informerCacheCtx, client.ObjectKey{Name: testNamespaceOne}, ns)
225+
Expect(apierrors.IsNotFound(err)).To(BeTrue())
226+
})
227+
})
228+
Describe("List", func() {
229+
When("Request is cluster-scoped", func() {
230+
It("Should list all pods and find exactly four", func() {
231+
var pods corev1.PodList
232+
err := informerCache.List(informerCacheCtx, &pods)
233+
Expect(err).NotTo(HaveOccurred())
234+
sort.Slice(pods.Items, func(i, j int) bool {
235+
if pods.Items[i].Namespace != pods.Items[j].Namespace {
236+
return pods.Items[i].Namespace < pods.Items[j].Namespace
237+
}
238+
return pods.Items[i].Name < pods.Items[j].Name
239+
})
240+
Expect(pods.Items).To(HaveLen(4))
241+
Expect(pods.Items[0].Namespace).To(Equal(testNamespaceOne))
242+
Expect(pods.Items[0].Name).To(Equal("pod-1a"))
243+
Expect(pods.Items[1].Namespace).To(Equal(testNamespaceOne))
244+
Expect(pods.Items[1].Name).To(Equal("pod-1b"))
245+
Expect(pods.Items[2].Namespace).To(Equal(testNamespaceTwo))
246+
Expect(pods.Items[2].Name).To(Equal("pod-2b"))
247+
Expect(pods.Items[3].Namespace).To(Equal(testNamespaceThree))
248+
Expect(pods.Items[3].Name).To(Equal("pod-3b"))
249+
})
250+
It("Should list nodes and find exactly one", func() {
251+
var nodes corev1.NodeList
252+
err := informerCache.List(informerCacheCtx, &nodes)
253+
Expect(err).NotTo(HaveOccurred())
254+
Expect(nodes.Items).To(HaveLen(1))
255+
Expect(nodes.Items[0].Namespace).To(Equal(""))
256+
Expect(nodes.Items[0].Name).To(Equal(testNodeOne))
257+
})
258+
It("Should list namespaces and find none", func() {
259+
var namespaces corev1.NamespaceList
260+
err := informerCache.List(informerCacheCtx, &namespaces)
261+
Expect(err).NotTo(HaveOccurred())
262+
Expect(namespaces.Items).To(HaveLen(0))
263+
})
264+
})
265+
When("Request is namespace-scoped", func() {
266+
It("Should list pods in namespace one", func() {
267+
var pods corev1.PodList
268+
err := informerCache.List(informerCacheCtx, &pods, client.InNamespace(testNamespaceOne))
269+
Expect(err).NotTo(HaveOccurred())
270+
sort.Slice(pods.Items, func(i, j int) bool {
271+
if pods.Items[i].Namespace != pods.Items[j].Namespace {
272+
return pods.Items[i].Namespace < pods.Items[j].Namespace
273+
}
274+
return pods.Items[i].Name < pods.Items[j].Name
275+
})
276+
Expect(pods.Items).To(HaveLen(2))
277+
Expect(pods.Items[0].Namespace).To(Equal(testNamespaceOne))
278+
Expect(pods.Items[0].Name).To(Equal("pod-1a"))
279+
Expect(pods.Items[1].Namespace).To(Equal(testNamespaceOne))
280+
Expect(pods.Items[1].Name).To(Equal("pod-1b"))
281+
})
282+
It("Should list pods in namespace two", func() {
283+
var pods corev1.PodList
284+
err := informerCache.List(informerCacheCtx, &pods, client.InNamespace(testNamespaceTwo))
285+
Expect(err).NotTo(HaveOccurred())
286+
Expect(pods.Items).To(HaveLen(1))
287+
Expect(pods.Items[0].Namespace).To(Equal(testNamespaceTwo))
288+
Expect(pods.Items[0].Name).To(Equal("pod-2b"))
289+
})
290+
It("Should list pods in namespace three", func() {
291+
var pods corev1.PodList
292+
err := informerCache.List(informerCacheCtx, &pods, client.InNamespace(testNamespaceThree))
293+
Expect(err).NotTo(HaveOccurred())
294+
Expect(pods.Items).To(HaveLen(1))
295+
Expect(pods.Items[0].Namespace).To(Equal(testNamespaceThree))
296+
Expect(pods.Items[0].Name).To(Equal("pod-3b"))
297+
})
298+
})
299+
})
300+
AfterEach(func() {
301+
deletePod(pod1a)
302+
deletePod(pod1b)
303+
deletePod(pod2a)
304+
deletePod(pod2b)
305+
deletePod(pod2c)
306+
deletePod(pod3a)
307+
deletePod(pod3b)
308+
informerCacheCancel()
309+
})
310+
})
125311

126312
var _ = Describe("Cache with transformers", func() {
127313
var (

‎pkg/cache/multi_namespace_cache.go

Lines changed: 75 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,12 @@ import (
2323

2424
corev1 "k8s.io/api/core/v1"
2525
apimeta "k8s.io/apimachinery/pkg/api/meta"
26+
"k8s.io/apimachinery/pkg/fields"
2627
"k8s.io/apimachinery/pkg/runtime"
2728
"k8s.io/apimachinery/pkg/runtime/schema"
2829
"k8s.io/client-go/rest"
2930
toolscache "k8s.io/client-go/tools/cache"
31+
3032
"sigs.k8s.io/controller-runtime/pkg/client"
3133
"sigs.k8s.io/controller-runtime/pkg/internal/objectutil"
3234
)
@@ -37,39 +39,88 @@ type NewCacheFunc func(config *rest.Config, opts Options) (Cache, error)
3739
// a new global namespaced cache to handle cluster scoped resources.
3840
const globalCache = "_cluster-scope"
3941

40-
// MultiNamespacedCacheBuilder - Builder function to create a new multi-namespaced cache.
41-
// This will scope the cache to a list of namespaces. Listing for all namespaces
42-
// will list for all the namespaces that this knows about. By default this will create
43-
// a global cache for cluster scoped resource. Note that this is not intended
44-
// to be used for excluding namespaces, this is better done via a Predicate. Also note that
45-
// you may face performance issues when using this with a high number of namespaces.
46-
func MultiNamespacedCacheBuilder(namespaces []string) NewCacheFunc {
42+
// ByNamespaceOptions is used to configure the functions used to create caches
43+
// on a per-namespace basis.
44+
type ByNamespaceOptions struct {
45+
NewNamespaceCaches map[string]NewCacheFunc
46+
NewClusterCache NewCacheFunc
47+
NewDefaultNamespaceCache NewCacheFunc
48+
}
49+
50+
// BuilderByNamespace builds a composite cache that delegates to per-namespace
51+
// caches built according to the passed ByNamespaceOptions. If NewDefaultNamespaceCache
52+
// is defined, it will be used as a catch-all for objects not in the namespaces defined in
53+
// NewNamespaceCaches. The default namespace cache is automatically configured with extra
54+
// field selectors to avoid duplicate caching of objects between namespace-specific caches
55+
// and this catch-all cache. NewClusterCache is used to create a cache for cluster-scoped
56+
// objects. If it is undefined, a default cache will be created using the New function.
57+
func BuilderByNamespace(byNamespaceOpts ByNamespaceOptions) NewCacheFunc {
4758
return func(config *rest.Config, opts Options) (Cache, error) {
4859
opts, err := defaultOpts(config, opts)
4960
if err != nil {
5061
return nil, err
5162
}
5263

53-
caches := map[string]Cache{}
54-
55-
// create a cache for cluster scoped resources
56-
gCache, err := New(config, opts)
64+
if byNamespaceOpts.NewClusterCache == nil {
65+
byNamespaceOpts.NewClusterCache = New
66+
}
67+
clusterCache, err := byNamespaceOpts.NewClusterCache(config, opts)
5768
if err != nil {
58-
return nil, fmt.Errorf("error creating global cache: %w", err)
69+
return nil, err
5970
}
6071

61-
for _, ns := range namespaces {
72+
nsToCache := map[string]Cache{}
73+
if byNamespaceOpts.NewDefaultNamespaceCache != nil {
74+
defaultNamespaceCache, err := byNamespaceOpts.NewDefaultNamespaceCache(config, ignoreNamespaces(opts, byNamespaceOpts.NewNamespaceCaches))
75+
if err != nil {
76+
return nil, err
77+
}
78+
nsToCache[corev1.NamespaceAll] = defaultNamespaceCache
79+
}
80+
81+
for ns, newCacheFunc := range byNamespaceOpts.NewNamespaceCaches {
6282
opts.Namespace = ns
63-
c, err := New(config, opts)
83+
nsToCache[ns], err = newCacheFunc(config, opts)
6484
if err != nil {
6585
return nil, err
6686
}
67-
caches[ns] = c
6887
}
69-
return &multiNamespaceCache{namespaceToCache: caches, Scheme: opts.Scheme, RESTMapper: opts.Mapper, clusterCache: gCache}, nil
88+
89+
return &multiNamespaceCache{
90+
namespaceToCache: nsToCache,
91+
clusterCache: clusterCache,
92+
RESTMapper: opts.Mapper,
93+
Scheme: opts.Scheme,
94+
}, nil
7095
}
7196
}
7297

98+
func ignoreNamespaces(opts Options, newObjectCaches map[string]NewCacheFunc) Options {
99+
fieldSelectors := []fields.Selector{}
100+
if opts.DefaultSelector.Field != nil {
101+
fieldSelectors = append(fieldSelectors, opts.DefaultSelector.Field)
102+
}
103+
for ns := range newObjectCaches {
104+
fieldSelectors = append(fieldSelectors, fields.OneTermNotEqualSelector("metadata.namespace", ns))
105+
}
106+
opts.DefaultSelector.Field = fields.AndSelectors(fieldSelectors...)
107+
return opts
108+
}
109+
110+
// MultiNamespacedCacheBuilder - Builder function to create a new multi-namespaced cache.
111+
// This will scope the cache to a list of namespaces. Listing for all namespaces
112+
// will list for all the namespaces that this knows about. By default this will create
113+
// a global cache for cluster scoped resource. Note that this is not intended
114+
// to be used for excluding namespaces, this is better done via a Predicate. Also note that
115+
// you may face performance issues when using this with a high number of namespaces.
116+
func MultiNamespacedCacheBuilder(namespaces []string) NewCacheFunc {
117+
byNamespaceOpts := ByNamespaceOptions{NewNamespaceCaches: map[string]NewCacheFunc{}}
118+
for _, ns := range namespaces {
119+
byNamespaceOpts.NewNamespaceCaches[ns] = New
120+
}
121+
return BuilderByNamespace(byNamespaceOpts)
122+
}
123+
73124
// multiNamespaceCache knows how to handle multiple namespaced caches
74125
// Use this feature when scoping permissions for your
75126
// operator to a list of namespaces instead of watching every namespace
@@ -212,6 +263,10 @@ func (c *multiNamespaceCache) Get(ctx context.Context, key client.ObjectKey, obj
212263
}
213264

214265
cache, ok := c.namespaceToCache[key.Namespace]
266+
if !ok {
267+
// Use the default/catch-all namespace cache if we have one.
268+
cache, ok = c.namespaceToCache[corev1.NamespaceAll]
269+
}
215270
if !ok {
216271
return fmt.Errorf("unable to get: %v because of unknown namespace for the cache", key)
217272
}
@@ -235,6 +290,10 @@ func (c *multiNamespaceCache) List(ctx context.Context, list client.ObjectList,
235290

236291
if listOpts.Namespace != corev1.NamespaceAll {
237292
cache, ok := c.namespaceToCache[listOpts.Namespace]
293+
if !ok {
294+
// Use the default/catch-all namespace cache if we have one.
295+
cache, ok = c.namespaceToCache[corev1.NamespaceAll]
296+
}
238297
if !ok {
239298
return fmt.Errorf("unable to get: %v because of unknown namespace for the cache", listOpts.Namespace)
240299
}

0 commit comments

Comments
 (0)