Skip to content

Commit 1d584ab

Browse files
kishorjkishorj
authored
Ability to reconfigure NLB target group health check (#2967)
* reconfigure NLB target group health check without deleting * use default matcher 200-399 for http health checks
1 parent 73f1dc0 commit 1d584ab

File tree

11 files changed

+130
-64
lines changed

11 files changed

+130
-64
lines changed

‎docs/deploy/configurations.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -158,4 +158,4 @@ They are a set of kye=value pairs that describe AWS load balance controller feat
158158
| EnableServiceController | string | true | Toggles support for `Service` type resources. |
159159
| EnableIPTargetType | string | true | Used to toggle support for target-type `ip` across `Ingress` and `Service` type resources. |
160160
| SubnetsClusterTagCheck | string | true | Enable or disable the check for `kubernetes.io/cluster/${cluster-name}` during subnet auto-discovery |
161-
| NLBHealthCheckTimeout | string | true | Enable or disable the use of `service.beta.kubernetes.io/aws-load-balancer-healthcheck-timeout` for `Service` type resources (NLB) |
161+
| NLBHealthCheckAdvancedConfiguration | string | true | Enable or disable advanced health check configuration for NLB, for example health check timeout |

‎docs/guide/service/annotations.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
| [service.beta.kubernetes.io/aws-load-balancer-healthcheck-unhealthy-threshold](#healthcheck-unhealthy-threshold) | integer | 3 | |
4040
| [service.beta.kubernetes.io/aws-load-balancer-healthcheck-timeout](#healthcheck-timeout) | integer | 10 | |
4141
| [service.beta.kubernetes.io/aws-load-balancer-healthcheck-interval](#healthcheck-interval) | integer | 10 | |
42+
| [service.beta.kubernetes.io/aws-load-balancer-healthcheck-success-codes](#healthcheck-success-codes) | string | 200-399 | |
4243
| [service.beta.kubernetes.io/aws-load-balancer-eip-allocations](#eip-allocations) | stringList | | internet-facing lb only. Length must match the number of subnets|
4344
| [service.beta.kubernetes.io/aws-load-balancer-private-ipv4-addresses](#private-ipv4-addresses) | stringList | | internal lb only. Length must match the number of subnets |
4445
| [service.beta.kubernetes.io/aws-load-balancer-ipv6-addresses](#ipv6-addresses) | stringList | | dualstack lb only. Length must match the number of subnets |
@@ -341,6 +342,12 @@ Health check on target groups can be configured with following annotations:
341342
service.beta.kubernetes.io/aws-load-balancer-healthcheck-interval: "10"
342343
```
343344

345+
- <a name="healthcheck-success-codes">`service.beta.kubernetes.io/aws-load-balancer-healthcheck-success-codes`</a> specifies the http success codes for the health check in case of http/https protocol.
346+
347+
!!!example
348+
```
349+
service.beta.kubernetes.io/aws-load-balancer-healthcheck-success-codes: "200-399"
350+
```
344351

345352
- <a name="healthcheck-timeout">`service.beta.kubernetes.io/aws-load-balancer-healthcheck-timeout`</a> specifies the target group health check timeout. The target has to respond within the timeout for a successful health check.
346353

‎pkg/annotations/constants.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ const (
7373
SvcLBSuffixHCProtocol = "aws-load-balancer-healthcheck-protocol"
7474
SvcLBSuffixHCPort = "aws-load-balancer-healthcheck-port"
7575
SvcLBSuffixHCPath = "aws-load-balancer-healthcheck-path"
76+
SvcLBSuffixHCSuccessCodes = "aws-load-balancer-healthcheck-success-codes"
7677
SvcLBSuffixTargetGroupAttributes = "aws-load-balancer-target-group-attributes"
7778
SvcLBSuffixSubnets = "aws-load-balancer-subnets"
7879
SvcLBSuffixEIPAllocations = "aws-load-balancer-eip-allocations"

‎pkg/config/feature_gates.go

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,14 @@ import (
1111
type Feature string
1212

1313
const (
14-
ListenerRulesTagging Feature = "ListenerRulesTagging"
15-
WeightedTargetGroups Feature = "WeightedTargetGroups"
16-
ServiceTypeLoadBalancerOnly Feature = "ServiceTypeLoadBalancerOnly"
17-
EndpointsFailOpen Feature = "EndpointsFailOpen"
18-
EnableServiceController Feature = "EnableServiceController"
19-
EnableIPTargetType Feature = "EnableIPTargetType"
20-
SubnetsClusterTagCheck Feature = "SubnetsClusterTagCheck"
21-
NLBHealthCheckTimeout Feature = "NLBHealthCheckTimeout"
14+
ListenerRulesTagging Feature = "ListenerRulesTagging"
15+
WeightedTargetGroups Feature = "WeightedTargetGroups"
16+
ServiceTypeLoadBalancerOnly Feature = "ServiceTypeLoadBalancerOnly"
17+
EndpointsFailOpen Feature = "EndpointsFailOpen"
18+
EnableServiceController Feature = "EnableServiceController"
19+
EnableIPTargetType Feature = "EnableIPTargetType"
20+
SubnetsClusterTagCheck Feature = "SubnetsClusterTagCheck"
21+
NLBHealthCheckAdvancedConfig Feature = "NLBHealthCheckAdvancedConfig"
2222
)
2323

2424
type FeatureGates interface {
@@ -46,14 +46,14 @@ type defaultFeatureGates struct {
4646
func NewFeatureGates() FeatureGates {
4747
return &defaultFeatureGates{
4848
featureState: map[Feature]bool{
49-
ListenerRulesTagging: true,
50-
WeightedTargetGroups: true,
51-
ServiceTypeLoadBalancerOnly: false,
52-
EndpointsFailOpen: false,
53-
EnableServiceController: true,
54-
EnableIPTargetType: true,
55-
SubnetsClusterTagCheck: true,
56-
NLBHealthCheckTimeout: true,
49+
ListenerRulesTagging: true,
50+
WeightedTargetGroups: true,
51+
ServiceTypeLoadBalancerOnly: false,
52+
EndpointsFailOpen: false,
53+
EnableServiceController: true,
54+
EnableIPTargetType: true,
55+
SubnetsClusterTagCheck: true,
56+
NLBHealthCheckAdvancedConfig: true,
5757
},
5858
}
5959
}

‎pkg/deploy/elbv2/target_group_synthesizer.go

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,19 +7,21 @@ import (
77
"github.com/pkg/errors"
88
"k8s.io/apimachinery/pkg/util/sets"
99
"sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services"
10+
"sigs.k8s.io/aws-load-balancer-controller/pkg/config"
1011
"sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/tracking"
1112
"sigs.k8s.io/aws-load-balancer-controller/pkg/model/core"
1213
elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2"
1314
)
1415

1516
// NewTargetGroupSynthesizer constructs targetGroupSynthesizer
1617
func NewTargetGroupSynthesizer(elbv2Client services.ELBV2, trackingProvider tracking.Provider, taggingManager TaggingManager,
17-
tgManager TargetGroupManager, logger logr.Logger, stack core.Stack) *targetGroupSynthesizer {
18+
tgManager TargetGroupManager, logger logr.Logger, featureGates config.FeatureGates, stack core.Stack) *targetGroupSynthesizer {
1819
return &targetGroupSynthesizer{
1920
elbv2Client: elbv2Client,
2021
trackingProvider: trackingProvider,
2122
taggingManager: taggingManager,
2223
tgManager: tgManager,
24+
featureGates: featureGates,
2325
logger: logger,
2426
stack: stack,
2527
unmatchedSDKTGs: nil,
@@ -32,6 +34,7 @@ type targetGroupSynthesizer struct {
3234
trackingProvider tracking.Provider
3335
taggingManager TaggingManager
3436
tgManager TargetGroupManager
37+
featureGates config.FeatureGates
3538
logger logr.Logger
3639

3740
stack core.Stack
@@ -45,7 +48,8 @@ func (s *targetGroupSynthesizer) Synthesize(ctx context.Context) error {
4548
if err != nil {
4649
return err
4750
}
48-
matchedResAndSDKTGs, unmatchedResTGs, unmatchedSDKTGs, err := matchResAndSDKTargetGroups(resTGs, sdkTGs, s.trackingProvider.ResourceIDTagKey())
51+
matchedResAndSDKTGs, unmatchedResTGs, unmatchedSDKTGs, err := matchResAndSDKTargetGroups(resTGs, sdkTGs,
52+
s.trackingProvider.ResourceIDTagKey(), s.featureGates)
4953
if err != nil {
5054
return err
5155
}
@@ -95,7 +99,7 @@ type resAndSDKTargetGroupPair struct {
9599
}
96100

97101
func matchResAndSDKTargetGroups(resTGs []*elbv2model.TargetGroup, sdkTGs []TargetGroupWithTags,
98-
resourceIDTagKey string) ([]resAndSDKTargetGroupPair, []*elbv2model.TargetGroup, []TargetGroupWithTags, error) {
102+
resourceIDTagKey string, featureGates config.FeatureGates) ([]resAndSDKTargetGroupPair, []*elbv2model.TargetGroup, []TargetGroupWithTags, error) {
99103
var matchedResAndSDKTGs []resAndSDKTargetGroupPair
100104
var unmatchedResTGs []*elbv2model.TargetGroup
101105
var unmatchedSDKTGs []TargetGroupWithTags
@@ -113,7 +117,7 @@ func matchResAndSDKTargetGroups(resTGs []*elbv2model.TargetGroup, sdkTGs []Targe
113117
sdkTGs := sdkTGsByID[resID]
114118
foundMatch := false
115119
for _, sdkTG := range sdkTGs {
116-
if isSDKTargetGroupRequiresReplacement(sdkTG, resTG) {
120+
if isSDKTargetGroupRequiresReplacement(sdkTG, resTG, featureGates) {
117121
unmatchedSDKTGs = append(unmatchedSDKTGs, sdkTG)
118122
continue
119123
}
@@ -158,7 +162,7 @@ func mapSDKTargetGroupByResourceID(sdkTGs []TargetGroupWithTags, resourceIDTagKe
158162
}
159163

160164
// isSDKTargetGroupRequiresReplacement checks whether a sdk TargetGroup requires replacement to fulfill a TargetGroup resource.
161-
func isSDKTargetGroupRequiresReplacement(sdkTG TargetGroupWithTags, resTG *elbv2model.TargetGroup) bool {
165+
func isSDKTargetGroupRequiresReplacement(sdkTG TargetGroupWithTags, resTG *elbv2model.TargetGroup, featureGates config.FeatureGates) bool {
162166
if string(resTG.Spec.TargetType) != awssdk.StringValue(sdkTG.TargetGroup.TargetType) {
163167
return true
164168
}
@@ -171,12 +175,12 @@ func isSDKTargetGroupRequiresReplacement(sdkTG TargetGroupWithTags, resTG *elbv2
171175
}
172176
}
173177

174-
return isSDKTargetGroupRequiresReplacementDueToNLBHealthCheck(sdkTG, resTG)
178+
return isSDKTargetGroupRequiresReplacementDueToNLBHealthCheck(sdkTG, resTG, featureGates)
175179
}
176180

177181
// most of the healthCheck settings for NLB targetGroups cannot be changed for now.
178-
func isSDKTargetGroupRequiresReplacementDueToNLBHealthCheck(sdkTG TargetGroupWithTags, resTG *elbv2model.TargetGroup) bool {
179-
if resTG.Spec.HealthCheckConfig == nil {
182+
func isSDKTargetGroupRequiresReplacementDueToNLBHealthCheck(sdkTG TargetGroupWithTags, resTG *elbv2model.TargetGroup, featureGates config.FeatureGates) bool {
183+
if resTG.Spec.HealthCheckConfig == nil || featureGates.Enabled(config.NLBHealthCheckAdvancedConfig) {
180184
return false
181185
}
182186
if resTG.Spec.Protocol != elbv2model.ProtocolTCP && resTG.Spec.Protocol != elbv2model.ProtocolUDP &&

‎pkg/deploy/elbv2/target_group_synthesizer_test.go

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"github.com/pkg/errors"
77
"github.com/stretchr/testify/assert"
88
"k8s.io/apimachinery/pkg/util/intstr"
9+
"sigs.k8s.io/aws-load-balancer-controller/pkg/config"
910
coremodel "sigs.k8s.io/aws-load-balancer-controller/pkg/model/core"
1011
elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2"
1112
"testing"
@@ -283,7 +284,8 @@ func Test_matchResAndSDKTargetGroups(t *testing.T) {
283284
}
284285
for _, tt := range tests {
285286
t.Run(tt.name, func(t *testing.T) {
286-
got, got1, got2, err := matchResAndSDKTargetGroups(tt.args.resTGs, tt.args.sdkTGs, tt.args.resourceIDTagKey)
287+
featureGates := config.NewFeatureGates()
288+
got, got1, got2, err := matchResAndSDKTargetGroups(tt.args.resTGs, tt.args.sdkTGs, tt.args.resourceIDTagKey, featureGates)
287289
if tt.wantErr != nil {
288290
assert.EqualError(t, err, tt.wantErr.Error())
289291
} else {
@@ -619,7 +621,7 @@ func Test_isSDKTargetGroupRequiresReplacement(t *testing.T) {
619621
want: true,
620622
},
621623
{
622-
name: "healthCheck change need replacement",
624+
name: "healthCheck change needs no replacement for protocol change",
623625
args: args{
624626
sdkTG: TargetGroupWithTags{
625627
TargetGroup: &elbv2sdk.TargetGroup{
@@ -653,12 +655,13 @@ func Test_isSDKTargetGroupRequiresReplacement(t *testing.T) {
653655
},
654656
},
655657
},
656-
want: true,
658+
want: false,
657659
},
658660
}
659661
for _, tt := range tests {
660662
t.Run(tt.name, func(t *testing.T) {
661-
got := isSDKTargetGroupRequiresReplacement(tt.args.sdkTG, tt.args.resTG)
663+
featureGates := config.NewFeatureGates()
664+
got := isSDKTargetGroupRequiresReplacement(tt.args.sdkTG, tt.args.resTG, featureGates)
662665
assert.Equal(t, tt.want, got)
663666
})
664667
}
@@ -668,8 +671,9 @@ func Test_isSDKTargetGroupRequiresReplacementDueToNLBHealthCheck(t *testing.T) {
668671
port8080 := intstr.FromInt(8080)
669672
protocolHTTP := elbv2model.ProtocolHTTP
670673
type args struct {
671-
sdkTG TargetGroupWithTags
672-
resTG *elbv2model.TargetGroup
674+
sdkTG TargetGroupWithTags
675+
resTG *elbv2model.TargetGroup
676+
disableAdvancedNLBHealthCheckConfig bool
673677
}
674678
tests := []struct {
675679
name string
@@ -714,7 +718,7 @@ func Test_isSDKTargetGroupRequiresReplacementDueToNLBHealthCheck(t *testing.T) {
714718
want: false,
715719
},
716720
{
717-
name: "NLB TargetGroup healthCheck cannot change protocol",
721+
name: "NLB TargetGroup healthCheck cannot change protocol without advanced config",
718722
args: args{
719723
sdkTG: TargetGroupWithTags{
720724
TargetGroup: &elbv2sdk.TargetGroup{
@@ -747,6 +751,7 @@ func Test_isSDKTargetGroupRequiresReplacementDueToNLBHealthCheck(t *testing.T) {
747751
},
748752
},
749753
},
754+
disableAdvancedNLBHealthCheckConfig: true,
750755
},
751756
want: true,
752757
},
@@ -784,6 +789,7 @@ func Test_isSDKTargetGroupRequiresReplacementDueToNLBHealthCheck(t *testing.T) {
784789
},
785790
},
786791
},
792+
disableAdvancedNLBHealthCheckConfig: true,
787793
},
788794
want: true,
789795
},
@@ -821,6 +827,7 @@ func Test_isSDKTargetGroupRequiresReplacementDueToNLBHealthCheck(t *testing.T) {
821827
},
822828
},
823829
},
830+
disableAdvancedNLBHealthCheckConfig: true,
824831
},
825832
want: true,
826833
},
@@ -858,6 +865,7 @@ func Test_isSDKTargetGroupRequiresReplacementDueToNLBHealthCheck(t *testing.T) {
858865
},
859866
},
860867
},
868+
disableAdvancedNLBHealthCheckConfig: true,
861869
},
862870
want: true,
863871
},
@@ -975,7 +983,11 @@ func Test_isSDKTargetGroupRequiresReplacementDueToNLBHealthCheck(t *testing.T) {
975983
}
976984
for _, tt := range tests {
977985
t.Run(tt.name, func(t *testing.T) {
978-
got := isSDKTargetGroupRequiresReplacementDueToNLBHealthCheck(tt.args.sdkTG, tt.args.resTG)
986+
featureGates := config.NewFeatureGates()
987+
if tt.args.disableAdvancedNLBHealthCheckConfig {
988+
featureGates.Disable(config.NLBHealthCheckAdvancedConfig)
989+
}
990+
got := isSDKTargetGroupRequiresReplacementDueToNLBHealthCheck(tt.args.sdkTG, tt.args.resTG, featureGates)
979991
assert.Equal(t, tt.want, got)
980992
})
981993
}

‎pkg/deploy/stack_deployer.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ func NewDefaultStackDeployer(cloud aws.Cloud, k8sClient client.Client,
4747
wafv2WebACLAssociationManager: wafv2.NewDefaultWebACLAssociationManager(cloud.WAFv2(), logger),
4848
wafRegionalWebACLAssociationManager: wafregional.NewDefaultWebACLAssociationManager(cloud.WAFRegional(), logger),
4949
shieldProtectionManager: shield.NewDefaultProtectionManager(cloud.Shield(), logger),
50+
featureGates: config.FeatureGates,
5051
vpcID: cloud.VpcID(),
5152
logger: logger,
5253
}
@@ -71,6 +72,7 @@ type defaultStackDeployer struct {
7172
wafv2WebACLAssociationManager wafv2.WebACLAssociationManager
7273
wafRegionalWebACLAssociationManager wafregional.WebACLAssociationManager
7374
shieldProtectionManager shield.ProtectionManager
75+
featureGates config.FeatureGates
7476
vpcID string
7577

7678
logger logr.Logger
@@ -85,7 +87,7 @@ type ResourceSynthesizer interface {
8587
func (d *defaultStackDeployer) Deploy(ctx context.Context, stack core.Stack) error {
8688
synthesizers := []ResourceSynthesizer{
8789
ec2.NewSecurityGroupSynthesizer(d.cloud.EC2(), d.trackingProvider, d.ec2TaggingManager, d.ec2SGManager, d.vpcID, d.logger, stack),
88-
elbv2.NewTargetGroupSynthesizer(d.cloud.ELBV2(), d.trackingProvider, d.elbv2TaggingManager, d.elbv2TGManager, d.logger, stack),
90+
elbv2.NewTargetGroupSynthesizer(d.cloud.ELBV2(), d.trackingProvider, d.elbv2TaggingManager, d.elbv2TGManager, d.logger, d.featureGates, stack),
8991
elbv2.NewLoadBalancerSynthesizer(d.cloud.ELBV2(), d.trackingProvider, d.elbv2TaggingManager, d.elbv2LBManager, d.logger, stack),
9092
elbv2.NewListenerSynthesizer(d.cloud.ELBV2(), d.elbv2TaggingManager, d.elbv2LSManager, d.logger, stack),
9193
elbv2.NewListenerRuleSynthesizer(d.cloud.ELBV2(), d.elbv2TaggingManager, d.elbv2LRManager, d.logger, stack),

0 commit comments

Comments
 (0)