Skip to content

Commit 789d7a8

Browse files
kishorjkishorj
committed
add an option to disable SG rules management for NLB
1 parent dce29ad commit 789d7a8

File tree

5 files changed

+184
-6
lines changed

5 files changed

+184
-6
lines changed

‎docs/guide/service/annotations.md

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
| [service.beta.kubernetes.io/aws-load-balancer-alpn-policy](#alpn-policy) | stringList | | |
4747
| [service.beta.kubernetes.io/aws-load-balancer-target-node-labels](#target-node-labels) | stringMap | | |
4848
| [service.beta.kubernetes.io/aws-load-balancer-attributes](#load-balancer-attributes) | stringMap | | |
49+
| [service.beta.kubernetes.io/aws-load-balancer-manage-backend-security-group-rules](#manage-backend-sg-rules) | boolean | false | |
4950

5051
## Traffic Routing
5152
Traffic Routing can be controlled with following annotations:
@@ -415,6 +416,16 @@ Load balancer access can be controlled via following annotations:
415416
service.beta.kubernetes.io/aws-load-balancer-internal: "true"
416417
```
417418

419+
- <a name="manage-backend-sg-rules">`service.beta.kubernetes.io/aws-load-balancer-manage-backend-security-group-rules`</a> specifies whether the controller should automatically add the ingress rules to the instance/ENI security group.
420+
421+
!!!warning ""
422+
If you disable the automatic management of security group rules for an NLB, you will need to manually add appropriate ingress rules to your EC2 instance or ENI security groups to allow access to the traffic and health check ports.
423+
424+
!!!example
425+
```
426+
service.beta.kubernetes.io/aws-load-balancer-manage-backend-security-group-rules: "false"
427+
```
428+
418429
## Legacy Cloud Provider
419430
The AWS Load Balancer Controller manages Kubernetes Services in a compatible way with the legacy aws cloud provider. The annotation `service.beta.kubernetes.io/aws-load-balancer-type` is used to determine which controller reconciles the service. If the annotation value is `nlb-ip` or `external`, legacy cloud provider ignores the service resource (provided it has the correct patch) so that the AWS Load Balancer controller can take over. For all other values of the annotation, the legacy cloud provider will handle the service. Note that this annotation should be specified during service creation and not edited later.
420431

‎pkg/annotations/constants.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,4 +80,5 @@ const (
8080
SvcLBSuffixALPNPolicy = "aws-load-balancer-alpn-policy"
8181
SvcLBSuffixTargetNodeLabels = "aws-load-balancer-target-node-labels"
8282
SvcLBSuffixLoadBalancerAttributes = "aws-load-balancer-attributes"
83+
SvcLBSuffixManageSGRules = "aws-load-balancer-manage-backend-security-group-rules"
8384
)

‎pkg/service/model_build_target_group.go

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -383,7 +383,10 @@ func (t *defaultModelBuildTask) buildTargetGroupBindingSpec(ctx context.Context,
383383
if err != nil {
384384
return elbv2model.TargetGroupBindingResourceSpec{}, err
385385
}
386-
tgbNetworking := t.buildTargetGroupBindingNetworking(ctx, targetPort, preserveClientIP, *hc.Port, port, defaultSourceRanges, *targetGroup.Spec.IPAddressType)
386+
tgbNetworking, err := t.buildTargetGroupBindingNetworking(ctx, targetPort, preserveClientIP, *hc.Port, port, defaultSourceRanges, *targetGroup.Spec.IPAddressType)
387+
if err != nil {
388+
return elbv2model.TargetGroupBindingResourceSpec{}, err
389+
}
387390
return elbv2model.TargetGroupBindingResourceSpec{
388391
Template: elbv2model.TargetGroupBindingTemplate{
389392
ObjectMeta: metav1.ObjectMeta{
@@ -430,7 +433,14 @@ func (t *defaultModelBuildTask) buildPeersFromSourceRangesConfiguration(_ contex
430433
}
431434

432435
func (t *defaultModelBuildTask) buildTargetGroupBindingNetworking(ctx context.Context, tgPort intstr.IntOrString, preserveClientIP bool,
433-
hcPort intstr.IntOrString, port corev1.ServicePort, defaultSourceRanges []string, targetGroupIPAddressType elbv2model.TargetGroupIPAddressType) *elbv2model.TargetGroupBindingNetworking {
436+
hcPort intstr.IntOrString, port corev1.ServicePort, defaultSourceRanges []string, targetGroupIPAddressType elbv2model.TargetGroupIPAddressType) (*elbv2model.TargetGroupBindingNetworking, error) {
437+
manageBackendSGRules, err := t.buildManageSecurityGroupRulesFlag(ctx)
438+
if err != nil {
439+
return nil, err
440+
}
441+
if !manageBackendSGRules {
442+
return nil, nil
443+
}
434444
tgProtocol := port.Protocol
435445
loadBalancerSubnetsSourceRanges := t.getLoadBalancerSubnetsSourceRanges(targetGroupIPAddressType)
436446
networkingProtocol := elbv2api.NetworkingProtocolTCP
@@ -459,7 +469,7 @@ func (t *defaultModelBuildTask) buildTargetGroupBindingNetworking(ctx context.Co
459469
preserveClientIP, customSourceRangesConfigured); len(hcIngressRules) > 0 {
460470
tgbNetworking.Ingress = append(tgbNetworking.Ingress, hcIngressRules...)
461471
}
462-
return tgbNetworking
472+
return tgbNetworking, nil
463473
}
464474

465475
func (t *defaultModelBuildTask) getDefaultIPSourceRanges(ctx context.Context, targetGroupIPAddressType elbv2model.TargetGroupIPAddressType,
@@ -570,3 +580,15 @@ func (t *defaultModelBuildTask) buildHealthCheckNetworkingIngressRules(trafficSo
570580
Ports: healthCheckPorts,
571581
}}
572582
}
583+
584+
func (t *defaultModelBuildTask) buildManageSecurityGroupRulesFlag(_ context.Context) (bool, error) {
585+
var rawEnabled bool
586+
exists, err := t.annotationParser.ParseBoolAnnotation(annotations.SvcLBSuffixManageSGRules, &rawEnabled, t.service.Annotations)
587+
if err != nil {
588+
return true, err
589+
}
590+
if exists {
591+
return rawEnabled, nil
592+
}
593+
return true, nil
594+
}

‎pkg/service/model_build_target_group_test.go

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1031,6 +1031,25 @@ func Test_defaultModelBuilderTask_buildTargetGroupBindingNetworking(t *testing.T
10311031
},
10321032
},
10331033
},
1034+
{
1035+
name: "with manage backend SG disabled via annotation",
1036+
svc: &corev1.Service{
1037+
ObjectMeta: metav1.ObjectMeta{
1038+
Annotations: map[string]string{
1039+
"service.beta.kubernetes.io/aws-load-balancer-manage-backend-security-group-rules": "false",
1040+
},
1041+
},
1042+
},
1043+
tgPort: port80,
1044+
hcPort: port808,
1045+
subnets: []*ec2.Subnet{{
1046+
CidrBlock: aws.String("172.16.0.0/19"),
1047+
SubnetId: aws.String("az-1"),
1048+
}},
1049+
tgProtocol: corev1.ProtocolTCP,
1050+
ipAddressType: elbv2.TargetGroupIPAddressTypeIPv4,
1051+
want: nil,
1052+
},
10341053
}
10351054
for _, tt := range tests {
10361055
t.Run(tt.name, func(t *testing.T) {
@@ -1039,7 +1058,7 @@ func Test_defaultModelBuilderTask_buildTargetGroupBindingNetworking(t *testing.T
10391058
port := corev1.ServicePort{
10401059
Protocol: tt.tgProtocol,
10411060
}
1042-
got := builder.buildTargetGroupBindingNetworking(context.Background(), tt.tgPort, tt.preserveClientIP, tt.hcPort, port, tt.defaultSourceRanges, tt.ipAddressType)
1061+
got, _ := builder.buildTargetGroupBindingNetworking(context.Background(), tt.tgPort, tt.preserveClientIP, tt.hcPort, port, tt.defaultSourceRanges, tt.ipAddressType)
10431062
assert.Equal(t, tt.want, got)
10441063
})
10451064
}

‎pkg/service/model_builder_test.go

Lines changed: 127 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,20 +2,20 @@ package service
22

33
import (
44
"context"
5-
"github.com/pkg/errors"
6-
"sigs.k8s.io/aws-load-balancer-controller/pkg/config"
75
"testing"
86
"time"
97

108
"github.com/aws/aws-sdk-go/aws"
119
"github.com/aws/aws-sdk-go/service/ec2"
1210
elbv2sdk "github.com/aws/aws-sdk-go/service/elbv2"
1311
"github.com/golang/mock/gomock"
12+
"github.com/pkg/errors"
1413
"github.com/stretchr/testify/assert"
1514
corev1 "k8s.io/api/core/v1"
1615
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1716
"k8s.io/apimachinery/pkg/util/intstr"
1817
"sigs.k8s.io/aws-load-balancer-controller/pkg/annotations"
18+
"sigs.k8s.io/aws-load-balancer-controller/pkg/config"
1919
"sigs.k8s.io/aws-load-balancer-controller/pkg/deploy"
2020
"sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/elbv2"
2121
"sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/tracking"
@@ -2578,6 +2578,131 @@ func Test_defaultModelBuilderTask_Build(t *testing.T) {
25782578
}
25792579
}`,
25802580
},
2581+
{
2582+
testName: "with backend SG rule management disabled",
2583+
svc: &corev1.Service{
2584+
ObjectMeta: metav1.ObjectMeta{
2585+
Name: "manual-sg-rule",
2586+
Namespace: "default",
2587+
UID: "c93458ad-9ef5-4c4c-bc0b-b31599ff585b",
2588+
Annotations: map[string]string{
2589+
"service.beta.kubernetes.io/aws-load-balancer-type": "external",
2590+
"service.beta.kubernetes.io/aws-load-balancer-nlb-target-type": "ip",
2591+
"service.beta.kubernetes.io/aws-load-balancer-manage-backend-security-group-rules": "false",
2592+
},
2593+
},
2594+
Spec: corev1.ServiceSpec{
2595+
Type: corev1.ServiceTypeLoadBalancer,
2596+
Selector: map[string]string{"app": "hello"},
2597+
Ports: []corev1.ServicePort{
2598+
{
2599+
Port: 80,
2600+
TargetPort: intstr.FromInt(80),
2601+
Protocol: corev1.ProtocolTCP,
2602+
},
2603+
},
2604+
},
2605+
},
2606+
resolveViaDiscoveryCalls: []resolveViaDiscoveryCall{resolveViaDiscoveryCallForOneSubnet},
2607+
listLoadBalancerCalls: []listLoadBalancerCall{listLoadBalancerCallForEmptyLB},
2608+
wantError: false,
2609+
wantValue: `
2610+
{
2611+
"id":"default/manual-sg-rule",
2612+
"resources":{
2613+
"AWS::ElasticLoadBalancingV2::Listener":{
2614+
"80":{
2615+
"spec":{
2616+
"loadBalancerARN":{
2617+
"$ref":"#/resources/AWS::ElasticLoadBalancingV2::LoadBalancer/LoadBalancer/status/loadBalancerARN"
2618+
},
2619+
"port":80,
2620+
"protocol":"TCP",
2621+
"defaultActions":[
2622+
{
2623+
"type":"forward",
2624+
"forwardConfig":{
2625+
"targetGroups":[
2626+
{
2627+
"targetGroupARN":{
2628+
"$ref":"#/resources/AWS::ElasticLoadBalancingV2::TargetGroup/default/manual-sg-rule:80/status/targetGroupARN"
2629+
}
2630+
}
2631+
]
2632+
}
2633+
}
2634+
]
2635+
}
2636+
}
2637+
},
2638+
"AWS::ElasticLoadBalancingV2::LoadBalancer":{
2639+
"LoadBalancer":{
2640+
"spec":{
2641+
"name":"k8s-default-manualsg-7af4592f28",
2642+
"type":"network",
2643+
"scheme":"internal",
2644+
"ipAddressType":"ipv4",
2645+
"subnetMapping":[
2646+
{
2647+
"subnetID":"subnet-1"
2648+
}
2649+
]
2650+
}
2651+
}
2652+
},
2653+
"AWS::ElasticLoadBalancingV2::TargetGroup":{
2654+
"default/manual-sg-rule:80":{
2655+
"spec":{
2656+
"name":"k8s-default-manualsg-4f421e4c8d",
2657+
"targetType":"ip",
2658+
"ipAddressType":"ipv4",
2659+
"port":80,
2660+
"protocol":"TCP",
2661+
"healthCheckConfig":{
2662+
"port":"traffic-port",
2663+
"protocol":"TCP",
2664+
"intervalSeconds":10,
2665+
"healthyThresholdCount":3,
2666+
"unhealthyThresholdCount":3
2667+
},
2668+
"targetGroupAttributes":[
2669+
{
2670+
"key":"proxy_protocol_v2.enabled",
2671+
"value":"false"
2672+
}
2673+
]
2674+
}
2675+
}
2676+
},
2677+
"K8S::ElasticLoadBalancingV2::TargetGroupBinding":{
2678+
"default/manual-sg-rule:80":{
2679+
"spec":{
2680+
"template":{
2681+
"metadata":{
2682+
"name":"k8s-default-manualsg-4f421e4c8d",
2683+
"namespace":"default",
2684+
"creationTimestamp":null
2685+
},
2686+
"spec":{
2687+
"targetGroupARN":{
2688+
"$ref":"#/resources/AWS::ElasticLoadBalancingV2::TargetGroup/default/manual-sg-rule:80/status/targetGroupARN"
2689+
},
2690+
"targetType":"ip",
2691+
"ipAddressType":"ipv4",
2692+
"serviceRef":{
2693+
"name":"manual-sg-rule",
2694+
"port":80
2695+
}
2696+
}
2697+
}
2698+
}
2699+
}
2700+
}
2701+
}
2702+
}
2703+
`,
2704+
wantNumResources: 4,
2705+
},
25812706
}
25822707

25832708
for _, tt := range tests {

0 commit comments

Comments
 (0)