Skip to content

Commit 5317c41

Browse files
authored
cherry-pick: Support AWS RGT APIs with feature flag (#3186) (#3193)
* Add support for RGT APIs with feature flag * revert change in iam policies and refactor tagging manager * list resrouces by arn in RGT * handle potential replication in returned RGT resources
1 parent ff8c13d commit 5317c41

File tree

10 files changed

+169
-16
lines changed

10 files changed

+169
-16
lines changed

controllers/ingress/group_controller.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ func NewGroupReconciler(cloud aws.Cloud, k8sClient client.Client, eventRecorder
5454
enhancedBackendBuilder := ingress.NewDefaultEnhancedBackendBuilder(k8sClient, annotationParser, authConfigBuilder)
5555
referenceIndexer := ingress.NewDefaultReferenceIndexer(enhancedBackendBuilder, authConfigBuilder, logger)
5656
trackingProvider := tracking.NewDefaultProvider(ingressTagPrefix, controllerConfig.ClusterName)
57-
elbv2TaggingManager := elbv2deploy.NewDefaultTaggingManager(cloud.ELBV2(), cloud.VpcID(), controllerConfig.FeatureGates, logger)
57+
elbv2TaggingManager := elbv2deploy.NewDefaultTaggingManager(cloud.ELBV2(), cloud.VpcID(), controllerConfig.FeatureGates, cloud.RGT(), logger)
5858
modelBuilder := ingress.NewDefaultModelBuilder(k8sClient, eventRecorder,
5959
cloud.EC2(), cloud.ACM(),
6060
annotationParser, subnetsResolver,

controllers/service/service_controller.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ func NewServiceReconciler(cloud aws.Cloud, k8sClient client.Client, eventRecorde
4141

4242
annotationParser := annotations.NewSuffixAnnotationParser(serviceAnnotationPrefix)
4343
trackingProvider := tracking.NewDefaultProvider(serviceTagPrefix, controllerConfig.ClusterName)
44-
elbv2TaggingManager := elbv2.NewDefaultTaggingManager(cloud.ELBV2(), cloud.VpcID(), controllerConfig.FeatureGates, logger)
44+
elbv2TaggingManager := elbv2.NewDefaultTaggingManager(cloud.ELBV2(), cloud.VpcID(), controllerConfig.FeatureGates, cloud.RGT(), logger)
4545
serviceUtils := service.NewServiceUtils(annotationParser, serviceFinalizer, controllerConfig.ServiceConfig.LoadBalancerClass, controllerConfig.FeatureGates)
4646
modelBuilder := service.NewDefaultModelBuilder(annotationParser, subnetsResolver, vpcInfoProvider, cloud.VpcID(), trackingProvider,
4747
elbv2TaggingManager, controllerConfig.FeatureGates, controllerConfig.ClusterName, controllerConfig.DefaultTags, controllerConfig.ExternalManagedTags, controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), serviceUtils)

docs/deploy/configurations.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,5 +158,6 @@ They are a set of kye=value pairs that describe AWS load balance controller feat
158158
| EndpointsFailOpen | string | true | Enable or disable allowing endpoints with `ready:unknown` state in the target groups. |
159159
| EnableServiceController | string | true | Toggles support for `Service` type resources. |
160160
| EnableIPTargetType | string | true | Used to toggle support for target-type `ip` across `Ingress` and `Service` type resources. |
161+
| EnableRGTAPI | string | false | If enabled, the tagging manager will describe resource tags via RGT APIs, otherwise via ELB APIs. In order to enable RGT API, `tag:GetResources` is needed in controller IAM policy. |
161162
| SubnetsClusterTagCheck | string | true | Enable or disable the check for `kubernetes.io/cluster/${cluster-name}` during subnet auto-discovery |
162163
| NLBHealthCheckAdvancedConfiguration | string | true | Enable or disable advanced health check configuration for NLB, for example health check timeout |

go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ go 1.20
44

55
require (
66
github.com/aws/aws-sdk-go v1.44.184
7+
github.com/aws/aws-sdk-go-v2 v0.18.0
78
github.com/evanphx/json-patch v5.6.0+incompatible
89
github.com/gavv/httpexpect/v2 v2.9.0
910
github.com/go-logr/logr v1.2.3

go.sum

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,9 @@ github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 h1:4daAzAu0
7979
github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
8080
github.com/aws/aws-sdk-go v1.44.184 h1:/MggyE66rOImXJKl1HqhLQITvWvqIV7w1Q4MaG6FHUo=
8181
github.com/aws/aws-sdk-go v1.44.184/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
82+
github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
83+
github.com/aws/aws-sdk-go-v2 v0.18.0 h1:qZ+woO4SamnH/eEbjM2IDLhRNwIwND/RQyVlBLp3Jqg=
84+
github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
8285
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
8386
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
8487
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=

pkg/aws/services/rgt.go

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,22 @@
11
package services
22

33
import (
4+
"context"
5+
"github.com/aws/aws-sdk-go/aws"
46
"github.com/aws/aws-sdk-go/aws/session"
57
"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
68
"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface"
79
)
810

11+
const (
12+
ResourceTypeELBTargetGroup = "elasticloadbalancing:targetgroup"
13+
ResourceTypeELBLoadBalancer = "elasticloadbalancing:loadbalancer"
14+
)
15+
916
type RGT interface {
1017
resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI
18+
19+
GetResourcesAsList(ctx context.Context, input *resourcegroupstaggingapi.GetResourcesInput) ([]*resourcegroupstaggingapi.ResourceTagMapping, error)
1120
}
1221

1322
// NewRGT constructs new RGT implementation.
@@ -17,6 +26,29 @@ func NewRGT(session *session.Session) RGT {
1726
}
1827
}
1928

29+
var _ RGT = (*defaultRGT)(nil)
30+
2031
type defaultRGT struct {
2132
resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI
2233
}
34+
35+
func (c *defaultRGT) GetResourcesAsList(ctx context.Context, input *resourcegroupstaggingapi.GetResourcesInput) ([]*resourcegroupstaggingapi.ResourceTagMapping, error) {
36+
var result []*resourcegroupstaggingapi.ResourceTagMapping
37+
if err := c.GetResourcesPagesWithContext(ctx, input, func(output *resourcegroupstaggingapi.GetResourcesOutput, _ bool) bool {
38+
for _, i := range output.ResourceTagMappingList {
39+
result = append(result, i)
40+
}
41+
return true
42+
}); err != nil {
43+
return nil, err
44+
}
45+
return result, nil
46+
}
47+
48+
func ParseRGTTags(tags []*resourcegroupstaggingapi.Tag) map[string]string {
49+
result := make(map[string]string, len(tags))
50+
for _, tag := range tags {
51+
result[aws.StringValue(tag.Key)] = aws.StringValue(tag.Value)
52+
}
53+
return result
54+
}

pkg/config/feature_gates.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ const (
1717
EndpointsFailOpen Feature = "EndpointsFailOpen"
1818
EnableServiceController Feature = "EnableServiceController"
1919
EnableIPTargetType Feature = "EnableIPTargetType"
20+
EnableRGTAPI Feature = "EnableRGTAPI"
2021
SubnetsClusterTagCheck Feature = "SubnetsClusterTagCheck"
2122
NLBHealthCheckAdvancedConfig Feature = "NLBHealthCheckAdvancedConfig"
2223
)
@@ -52,6 +53,7 @@ func NewFeatureGates() FeatureGates {
5253
EndpointsFailOpen: true,
5354
EnableServiceController: true,
5455
EnableIPTargetType: true,
56+
EnableRGTAPI: false,
5557
SubnetsClusterTagCheck: true,
5658
NLBHealthCheckAdvancedConfig: true,
5759
},

pkg/deploy/elbv2/tagging_manager.go

Lines changed: 118 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,12 @@ package elbv2
22

33
import (
44
"context"
5+
"github.com/aws/aws-sdk-go-v2/aws"
56
awssdk "github.com/aws/aws-sdk-go/aws"
67
elbv2sdk "github.com/aws/aws-sdk-go/service/elbv2"
8+
"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
79
"github.com/go-logr/logr"
10+
"github.com/pkg/errors"
811
"k8s.io/apimachinery/pkg/util/sets"
912
"sigs.k8s.io/aws-load-balancer-controller/pkg/algorithm"
1013
"sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services"
@@ -93,26 +96,28 @@ type TaggingManager interface {
9396
}
9497

9598
// NewDefaultTaggingManager constructs default TaggingManager.
96-
func NewDefaultTaggingManager(elbv2Client services.ELBV2, vpcID string, featureGates config.FeatureGates, logger logr.Logger) *defaultTaggingManager {
99+
func NewDefaultTaggingManager(elbv2Client services.ELBV2, vpcID string, featureGates config.FeatureGates, rgt services.RGT, logger logr.Logger) *defaultTaggingManager {
97100
return &defaultTaggingManager{
98101
elbv2Client: elbv2Client,
99102
vpcID: vpcID,
100103
featureGates: featureGates,
101104
logger: logger,
102105
describeTagsChunkSize: defaultDescribeTagsChunkSize,
106+
rgt: rgt,
103107
}
104108
}
105109

106110
var _ TaggingManager = &defaultTaggingManager{}
107111

108112
// default implementation for TaggingManager
109-
// @TODO: use AWS Resource Groups Tagging API to optimize this implementation once it have PrivateLink support.
113+
// @TODO: deprecate ELB API and only use AWS Resource Groups Tagging API to optimize this implementation once RGT has PrivateLink support.
110114
type defaultTaggingManager struct {
111115
elbv2Client services.ELBV2
112116
vpcID string
113117
featureGates config.FeatureGates
114118
logger logr.Logger
115119
describeTagsChunkSize int
120+
rgt services.RGT
116121
}
117122

118123
func (m *defaultTaggingManager) ReconcileTags(ctx context.Context, arn string, desiredTags map[string]string, opts ...ReconcileTagsOption) error {
@@ -123,7 +128,7 @@ func (m *defaultTaggingManager) ReconcileTags(ctx context.Context, arn string, d
123128
reconcileOpts.ApplyOptions(opts)
124129
currentTags := reconcileOpts.CurrentTags
125130
if currentTags == nil {
126-
tagsByARN, err := m.describeResourceTags(ctx, []string{arn})
131+
tagsByARN, err := m.describeResourceTagsNative(ctx, []string{arn})
127132
if err != nil {
128133
return err
129134
}
@@ -188,7 +193,7 @@ func (m *defaultTaggingManager) ListListeners(ctx context.Context, lbARN string)
188193
}
189194
var tagsByARN map[string]map[string]string
190195
if m.featureGates.Enabled(config.ListenerRulesTagging) {
191-
tagsByARN, err = m.describeResourceTags(ctx, lsARNs)
196+
tagsByARN, err = m.describeResourceTagsNative(ctx, lsARNs)
192197
if err != nil {
193198
return nil, err
194199
}
@@ -221,7 +226,7 @@ func (m *defaultTaggingManager) ListListenerRules(ctx context.Context, lsARN str
221226
}
222227
var tagsByARN map[string]map[string]string
223228
if m.featureGates.Enabled(config.ListenerRulesTagging) {
224-
tagsByARN, err = m.describeResourceTags(ctx, lrARNs)
229+
tagsByARN, err = m.describeResourceTagsNative(ctx, lrARNs)
225230
if err != nil {
226231
return nil, err
227232
}
@@ -238,12 +243,64 @@ func (m *defaultTaggingManager) ListListenerRules(ctx context.Context, lsARN str
238243
}
239244

240245
func (m *defaultTaggingManager) ListLoadBalancers(ctx context.Context, tagFilters ...tracking.TagFilter) ([]LoadBalancerWithTags, error) {
246+
if m.featureGates.Enabled(config.EnableRGTAPI) {
247+
return m.listLoadBalancersRGT(ctx, tagFilters)
248+
}
249+
return m.listLoadBalancersNative(ctx, tagFilters)
250+
}
251+
252+
func (m *defaultTaggingManager) ListTargetGroups(ctx context.Context, tagFilters ...tracking.TagFilter) ([]TargetGroupWithTags, error) {
253+
if m.featureGates.Enabled(config.EnableRGTAPI) {
254+
return m.listTargetGroupsRGT(ctx, tagFilters)
255+
}
256+
return m.listTargetGroupsNative(ctx, tagFilters)
257+
258+
}
259+
260+
func (m *defaultTaggingManager) listLoadBalancersRGT(ctx context.Context, tagFilters []tracking.TagFilter) ([]LoadBalancerWithTags, error) {
261+
// use a map to avoid potential duplication in returned resources
262+
resourceTagsByARN := make(map[string][]*resourcegroupstaggingapi.Tag)
263+
for _, tagFilter := range tagFilters {
264+
req := &resourcegroupstaggingapi.GetResourcesInput{
265+
TagFilters: convertTagFiltersToRGTTagFilters(tagFilter),
266+
ResourceTypeFilters: aws.StringSlice([]string{services.ResourceTypeELBLoadBalancer}),
267+
}
268+
resources, err := m.rgt.GetResourcesAsList(ctx, req)
269+
if err != nil {
270+
return nil, err
271+
}
272+
for _, resource := range resources {
273+
if _, exists := resourceTagsByARN[aws.StringValue(resource.ResourceARN)]; !exists {
274+
resourceTagsByARN[aws.StringValue(resource.ResourceARN)] = resource.Tags
275+
}
276+
}
277+
}
278+
var matchedLBs []LoadBalancerWithTags
279+
for resourceARN, resourceTags := range resourceTagsByARN {
280+
elbv2Req := &elbv2sdk.DescribeLoadBalancersInput{
281+
LoadBalancerArns: []*string{&resourceARN},
282+
}
283+
elbv2Resp, err := m.elbv2Client.DescribeLoadBalancersAsList(ctx, elbv2Req)
284+
if err != nil {
285+
return nil, err
286+
}
287+
if len(elbv2Resp) == 0 {
288+
return nil, errors.Errorf("no load balancer found for the arn: %v", resourceARN)
289+
}
290+
matchedLBs = append(matchedLBs, LoadBalancerWithTags{
291+
LoadBalancer: elbv2Resp[0],
292+
Tags: services.ParseRGTTags(resourceTags),
293+
})
294+
}
295+
return matchedLBs, nil
296+
}
297+
298+
func (m *defaultTaggingManager) listLoadBalancersNative(ctx context.Context, tagFilters []tracking.TagFilter) ([]LoadBalancerWithTags, error) {
241299
req := &elbv2sdk.DescribeLoadBalancersInput{}
242300
lbs, err := m.elbv2Client.DescribeLoadBalancersAsList(ctx, req)
243301
if err != nil {
244302
return nil, err
245303
}
246-
247304
lbARNsWithinVPC := make([]string, 0, len(lbs))
248305
lbByARNWithinVPC := make(map[string]*elbv2sdk.LoadBalancer, len(lbs))
249306
for _, lb := range lbs {
@@ -254,7 +311,7 @@ func (m *defaultTaggingManager) ListLoadBalancers(ctx context.Context, tagFilter
254311
lbARNsWithinVPC = append(lbARNsWithinVPC, lbARN)
255312
lbByARNWithinVPC[lbARN] = lb
256313
}
257-
tagsByARN, err := m.describeResourceTags(ctx, lbARNsWithinVPC)
314+
tagsByARN, err := m.describeResourceTagsNative(ctx, lbARNsWithinVPC)
258315
if err != nil {
259316
return nil, err
260317
}
@@ -279,7 +336,45 @@ func (m *defaultTaggingManager) ListLoadBalancers(ctx context.Context, tagFilter
279336
return matchedLBs, nil
280337
}
281338

282-
func (m *defaultTaggingManager) ListTargetGroups(ctx context.Context, tagFilters ...tracking.TagFilter) ([]TargetGroupWithTags, error) {
339+
func (m *defaultTaggingManager) listTargetGroupsRGT(ctx context.Context, tagFilters []tracking.TagFilter) ([]TargetGroupWithTags, error) {
340+
// use a map to avoid potential duplication in returned resources
341+
resourceTagsByARN := make(map[string][]*resourcegroupstaggingapi.Tag)
342+
for _, tagFilter := range tagFilters {
343+
req := &resourcegroupstaggingapi.GetResourcesInput{
344+
TagFilters: convertTagFiltersToRGTTagFilters(tagFilter),
345+
ResourceTypeFilters: aws.StringSlice([]string{services.ResourceTypeELBTargetGroup}),
346+
}
347+
resources, err := m.rgt.GetResourcesAsList(ctx, req)
348+
if err != nil {
349+
return nil, err
350+
}
351+
for _, resource := range resources {
352+
if _, exists := resourceTagsByARN[aws.StringValue(resource.ResourceARN)]; !exists {
353+
resourceTagsByARN[aws.StringValue(resource.ResourceARN)] = resource.Tags
354+
}
355+
}
356+
}
357+
var matchedTGs []TargetGroupWithTags
358+
for resourceARN, resourceTags := range resourceTagsByARN {
359+
elbv2Req := &elbv2sdk.DescribeTargetGroupsInput{
360+
TargetGroupArns: []*string{&resourceARN},
361+
}
362+
elbv2Resp, err := m.elbv2Client.DescribeTargetGroupsAsList(ctx, elbv2Req)
363+
if err != nil {
364+
return nil, err
365+
}
366+
if len(elbv2Resp) == 0 {
367+
return nil, errors.Errorf("no target group found for the arn: %v", resourceARN)
368+
}
369+
matchedTGs = append(matchedTGs, TargetGroupWithTags{
370+
TargetGroup: elbv2Resp[0],
371+
Tags: services.ParseRGTTags(resourceTags),
372+
})
373+
}
374+
return matchedTGs, nil
375+
}
376+
377+
func (m *defaultTaggingManager) listTargetGroupsNative(ctx context.Context, tagFilters []tracking.TagFilter) ([]TargetGroupWithTags, error) {
283378
req := &elbv2sdk.DescribeTargetGroupsInput{}
284379
tgs, err := m.elbv2Client.DescribeTargetGroupsAsList(ctx, req)
285380
if err != nil {
@@ -296,7 +391,7 @@ func (m *defaultTaggingManager) ListTargetGroups(ctx context.Context, tagFilters
296391
tgARNsWithinVPC = append(tgARNsWithinVPC, tgARN)
297392
tgByARNWithinVPC[tgARN] = tg
298393
}
299-
tagsByARN, err := m.describeResourceTags(ctx, tgARNsWithinVPC)
394+
tagsByARN, err := m.describeResourceTagsNative(ctx, tgARNsWithinVPC)
300395
if err != nil {
301396
return nil, err
302397
}
@@ -321,9 +416,9 @@ func (m *defaultTaggingManager) ListTargetGroups(ctx context.Context, tagFilters
321416
return matchedTGs, nil
322417
}
323418

324-
// describeResourceTags describes tags for elbv2 resources.
419+
// describeResourceTagsNative describes tags for elbv2 resources.
325420
// returns tags indexed by resource ARN.
326-
func (m *defaultTaggingManager) describeResourceTags(ctx context.Context, arns []string) (map[string]map[string]string, error) {
421+
func (m *defaultTaggingManager) describeResourceTagsNative(ctx context.Context, arns []string) (map[string]map[string]string, error) {
327422
tagsByARN := make(map[string]map[string]string, len(arns))
328423
arnsChunks := algorithm.ChunkStrings(arns, m.describeTagsChunkSize)
329424
for _, arnsChunk := range arnsChunks {
@@ -365,3 +460,15 @@ func convertSDKTagsToTags(sdkTags []*elbv2sdk.Tag) map[string]string {
365460
}
366461
return tags
367462
}
463+
464+
// convert tagFilters to RGTTagFilters
465+
func convertTagFiltersToRGTTagFilters(tagFilter tracking.TagFilter) []*resourcegroupstaggingapi.TagFilter {
466+
var RGTTagFilters []*resourcegroupstaggingapi.TagFilter
467+
for k, v := range tagFilter {
468+
RGTTagFilters = append(RGTTagFilters, &resourcegroupstaggingapi.TagFilter{
469+
Key: aws.String(k),
470+
Values: aws.StringSlice(v),
471+
})
472+
}
473+
return RGTTagFilters
474+
}

0 commit comments

Comments
 (0)