Skip to content

Commit bfff95f

Browse files
wweiwei-liwweiwei-li
committed
Add support for optionally enforcing NLB security groups on PrivateLink
1 parent c2515fe commit bfff95f

File tree

8 files changed

+162
-59
lines changed

8 files changed

+162
-59
lines changed

‎controllers/service/service_controller.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ func NewServiceReconciler(cloud aws.Cloud, k8sClient client.Client, eventRecorde
4747
modelBuilder := service.NewDefaultModelBuilder(annotationParser, subnetsResolver, vpcInfoProvider, cloud.VpcID(), trackingProvider,
4848
elbv2TaggingManager, cloud.EC2(), controllerConfig.FeatureGates, controllerConfig.ClusterName, controllerConfig.DefaultTags, controllerConfig.ExternalManagedTags,
4949
controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), serviceUtils,
50-
backendSGProvider, sgResolver, controllerConfig.EnableBackendSecurityGroup, controllerConfig.DisableRestrictedSGRules)
50+
backendSGProvider, sgResolver, controllerConfig.EnableBackendSecurityGroup, controllerConfig.DisableRestrictedSGRules, logger)
5151
stackMarshaller := deploy.NewDefaultStackMarshaller()
5252
stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingSGManager, networkingSGReconciler, elbv2TaggingManager, controllerConfig, serviceTagPrefix, logger)
5353
return &serviceReconciler{

‎docs/guide/service/annotations.md

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
| [service.beta.kubernetes.io/aws-load-balancer-attributes](#load-balancer-attributes) | stringMap | | |
5151
| [service.beta.kubernetes.io/aws-load-balancer-security-groups](#security-groups) | stringList | | |
5252
| [service.beta.kubernetes.io/aws-load-balancer-manage-backend-security-group-rules](#manage-backend-sg-rules) | boolean | true | |
53+
| [service.beta.kubernetes.io/aws-load-balancer-inbound-sg-rules-on-private-link-traffic](#update-security-settings) | string | |
5354

5455
## Traffic Routing
5556
Traffic Routing can be controlled with following annotations:
@@ -488,6 +489,14 @@ Load balancer access can be controlled via following annotations:
488489
service.beta.kubernetes.io/aws-load-balancer-manage-backend-security-group-rules: "false"
489490
```
490491

492+
- <a name="update-security-settings">`service.beta.kubernetes.io/aws-load-balancer-inbound-sg-rules-on-private-link-traffic`</a> specifies whether to apply security group rules to traffic sent to the load balancer through AWS PrivateLink.
493+
494+
!!!example
495+
```
496+
service.beta.kubernetes.io/aws-load-balancer-inbound-sg-rules-on-private-link-traffic: "off"
497+
```
498+
499+
491500
## Legacy Cloud Provider
492501
The AWS Load Balancer Controller manages Kubernetes Services in a compatible way with the AWS cloud provider's legacy service controller.
493502

‎pkg/annotations/constants.go

Lines changed: 36 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -50,39 +50,40 @@ const (
5050

5151
// NLB annotation suffixes
5252
// prefixes service.beta.kubernetes.io, service.kubernetes.io
53-
SvcLBSuffixSourceRanges = "load-balancer-source-ranges"
54-
SvcLBSuffixLoadBalancerType = "aws-load-balancer-type"
55-
SvcLBSuffixTargetType = "aws-load-balancer-nlb-target-type"
56-
SvcLBSuffixLoadBalancerName = "aws-load-balancer-name"
57-
SvcLBSuffixScheme = "aws-load-balancer-scheme"
58-
SvcLBSuffixInternal = "aws-load-balancer-internal"
59-
SvcLBSuffixProxyProtocol = "aws-load-balancer-proxy-protocol"
60-
SvcLBSuffixIPAddressType = "aws-load-balancer-ip-address-type"
61-
SvcLBSuffixAccessLogEnabled = "aws-load-balancer-access-log-enabled"
62-
SvcLBSuffixAccessLogS3BucketName = "aws-load-balancer-access-log-s3-bucket-name"
63-
SvcLBSuffixAccessLogS3BucketPrefix = "aws-load-balancer-access-log-s3-bucket-prefix"
64-
SvcLBSuffixCrossZoneLoadBalancingEnabled = "aws-load-balancer-cross-zone-load-balancing-enabled"
65-
SvcLBSuffixSSLCertificate = "aws-load-balancer-ssl-cert"
66-
SvcLBSuffixSSLPorts = "aws-load-balancer-ssl-ports"
67-
SvcLBSuffixSSLNegotiationPolicy = "aws-load-balancer-ssl-negotiation-policy"
68-
SvcLBSuffixBEProtocol = "aws-load-balancer-backend-protocol"
69-
SvcLBSuffixAdditionalTags = "aws-load-balancer-additional-resource-tags"
70-
SvcLBSuffixHCHealthyThreshold = "aws-load-balancer-healthcheck-healthy-threshold"
71-
SvcLBSuffixHCUnhealthyThreshold = "aws-load-balancer-healthcheck-unhealthy-threshold"
72-
SvcLBSuffixHCTimeout = "aws-load-balancer-healthcheck-timeout"
73-
SvcLBSuffixHCInterval = "aws-load-balancer-healthcheck-interval"
74-
SvcLBSuffixHCProtocol = "aws-load-balancer-healthcheck-protocol"
75-
SvcLBSuffixHCPort = "aws-load-balancer-healthcheck-port"
76-
SvcLBSuffixHCPath = "aws-load-balancer-healthcheck-path"
77-
SvcLBSuffixHCSuccessCodes = "aws-load-balancer-healthcheck-success-codes"
78-
SvcLBSuffixTargetGroupAttributes = "aws-load-balancer-target-group-attributes"
79-
SvcLBSuffixSubnets = "aws-load-balancer-subnets"
80-
SvcLBSuffixEIPAllocations = "aws-load-balancer-eip-allocations"
81-
SvcLBSuffixPrivateIpv4Addresses = "aws-load-balancer-private-ipv4-addresses"
82-
SvcLBSuffixIpv6Addresses = "aws-load-balancer-ipv6-addresses"
83-
SvcLBSuffixALPNPolicy = "aws-load-balancer-alpn-policy"
84-
SvcLBSuffixTargetNodeLabels = "aws-load-balancer-target-node-labels"
85-
SvcLBSuffixLoadBalancerAttributes = "aws-load-balancer-attributes"
86-
SvcLBSuffixLoadBalancerSecurityGroups = "aws-load-balancer-security-groups"
87-
SvcLBSuffixManageSGRules = "aws-load-balancer-manage-backend-security-group-rules"
53+
SvcLBSuffixSourceRanges = "load-balancer-source-ranges"
54+
SvcLBSuffixLoadBalancerType = "aws-load-balancer-type"
55+
SvcLBSuffixTargetType = "aws-load-balancer-nlb-target-type"
56+
SvcLBSuffixLoadBalancerName = "aws-load-balancer-name"
57+
SvcLBSuffixScheme = "aws-load-balancer-scheme"
58+
SvcLBSuffixInternal = "aws-load-balancer-internal"
59+
SvcLBSuffixProxyProtocol = "aws-load-balancer-proxy-protocol"
60+
SvcLBSuffixIPAddressType = "aws-load-balancer-ip-address-type"
61+
SvcLBSuffixAccessLogEnabled = "aws-load-balancer-access-log-enabled"
62+
SvcLBSuffixAccessLogS3BucketName = "aws-load-balancer-access-log-s3-bucket-name"
63+
SvcLBSuffixAccessLogS3BucketPrefix = "aws-load-balancer-access-log-s3-bucket-prefix"
64+
SvcLBSuffixCrossZoneLoadBalancingEnabled = "aws-load-balancer-cross-zone-load-balancing-enabled"
65+
SvcLBSuffixSSLCertificate = "aws-load-balancer-ssl-cert"
66+
SvcLBSuffixSSLPorts = "aws-load-balancer-ssl-ports"
67+
SvcLBSuffixSSLNegotiationPolicy = "aws-load-balancer-ssl-negotiation-policy"
68+
SvcLBSuffixBEProtocol = "aws-load-balancer-backend-protocol"
69+
SvcLBSuffixAdditionalTags = "aws-load-balancer-additional-resource-tags"
70+
SvcLBSuffixHCHealthyThreshold = "aws-load-balancer-healthcheck-healthy-threshold"
71+
SvcLBSuffixHCUnhealthyThreshold = "aws-load-balancer-healthcheck-unhealthy-threshold"
72+
SvcLBSuffixHCTimeout = "aws-load-balancer-healthcheck-timeout"
73+
SvcLBSuffixHCInterval = "aws-load-balancer-healthcheck-interval"
74+
SvcLBSuffixHCProtocol = "aws-load-balancer-healthcheck-protocol"
75+
SvcLBSuffixHCPort = "aws-load-balancer-healthcheck-port"
76+
SvcLBSuffixHCPath = "aws-load-balancer-healthcheck-path"
77+
SvcLBSuffixHCSuccessCodes = "aws-load-balancer-healthcheck-success-codes"
78+
SvcLBSuffixTargetGroupAttributes = "aws-load-balancer-target-group-attributes"
79+
SvcLBSuffixSubnets = "aws-load-balancer-subnets"
80+
SvcLBSuffixEIPAllocations = "aws-load-balancer-eip-allocations"
81+
SvcLBSuffixPrivateIpv4Addresses = "aws-load-balancer-private-ipv4-addresses"
82+
SvcLBSuffixIpv6Addresses = "aws-load-balancer-ipv6-addresses"
83+
SvcLBSuffixALPNPolicy = "aws-load-balancer-alpn-policy"
84+
SvcLBSuffixTargetNodeLabels = "aws-load-balancer-target-node-labels"
85+
SvcLBSuffixLoadBalancerAttributes = "aws-load-balancer-attributes"
86+
SvcLBSuffixLoadBalancerSecurityGroups = "aws-load-balancer-security-groups"
87+
SvcLBSuffixManageSGRules = "aws-load-balancer-manage-backend-security-group-rules"
88+
SvcLBSuffixEnforceSGInboundRulesOnPrivateLinkTraffic = "aws-load-balancer-inbound-sg-rules-on-private-link-traffic"
8889
)

‎pkg/deploy/elbv2/load_balancer_manager.go

Lines changed: 49 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package elbv2
33
import (
44
"context"
55
"fmt"
6+
67
awssdk "github.com/aws/aws-sdk-go/aws"
78
elbv2sdk "github.com/aws/aws-sdk-go/service/elbv2"
89
"github.com/go-logr/logr"
@@ -75,6 +76,12 @@ func (m *defaultLoadBalancerManager) Create(ctx context.Context, resLB *elbv2mod
7576
return elbv2model.LoadBalancerStatus{}, err
7677
}
7778

79+
if resLB.Spec.Type == elbv2model.LoadBalancerTypeNetwork && resLB.Spec.SecurityGroupsInboundRulesOnPrivateLink != nil {
80+
if err := m.updateSDKLoadBalancerWithSecurityGroups(ctx, resLB, sdkLB); err != nil {
81+
return elbv2model.LoadBalancerStatus{}, err
82+
}
83+
}
84+
7885
return buildResLoadBalancerStatus(sdkLB), nil
7986
}
8087

@@ -186,27 +193,39 @@ func (m *defaultLoadBalancerManager) updateSDKLoadBalancerWithSecurityGroups(ctx
186193
}
187194
desiredSecurityGroups := sets.NewString(awssdk.StringValueSlice(securityGroups)...)
188195
currentSecurityGroups := sets.NewString(awssdk.StringValueSlice(sdkLB.LoadBalancer.SecurityGroups)...)
189-
if desiredSecurityGroups.Equal(currentSecurityGroups) {
196+
197+
isEnforceSGInboundRulesOnPrivateLinkUpdated, currentEnforceSecurityGroupInboundRulesOnPrivateLinkTraffic, desiredEnforceSecurityGroupInboundRulesOnPrivateLinkTraffic := isEnforceSGInboundRulesOnPrivateLinkUpdated(resLB, sdkLB)
198+
if desiredSecurityGroups.Equal(currentSecurityGroups) && !isEnforceSGInboundRulesOnPrivateLinkUpdated {
190199
return nil
191200
}
192201

202+
var changeDescriptions []string
203+
204+
if !desiredSecurityGroups.Equal(currentSecurityGroups) {
205+
changeSecurityGroupsDesc := fmt.Sprintf("%v => %v", currentSecurityGroups.List(), desiredSecurityGroups.List())
206+
changeDescriptions = append(changeDescriptions, "changeSecurityGroups", changeSecurityGroupsDesc)
207+
}
208+
193209
req := &elbv2sdk.SetSecurityGroupsInput{
194210
LoadBalancerArn: sdkLB.LoadBalancer.LoadBalancerArn,
195211
SecurityGroups: securityGroups,
196212
}
197-
changeDesc := fmt.Sprintf("%v => %v", currentSecurityGroups.List(), desiredSecurityGroups.List())
198-
m.logger.Info("modifying loadBalancer securityGroups",
199-
"stackID", resLB.Stack().StackID(),
200-
"resourceID", resLB.ID(),
201-
"arn", awssdk.StringValue(sdkLB.LoadBalancer.LoadBalancerArn),
202-
"change", changeDesc)
213+
214+
if isEnforceSGInboundRulesOnPrivateLinkUpdated {
215+
changeEnforceSecurityGroupInboundRulesOnPrivateLinkTrafficDesc := fmt.Sprintf("%v => %v", currentEnforceSecurityGroupInboundRulesOnPrivateLinkTraffic, desiredEnforceSecurityGroupInboundRulesOnPrivateLinkTraffic)
216+
changeDescriptions = append(changeDescriptions, "changeEnforceSecurityGroupInboundRulesOnPrivateLinkTraffic", changeEnforceSecurityGroupInboundRulesOnPrivateLinkTrafficDesc)
217+
req.EnforceSecurityGroupInboundRulesOnPrivateLinkTraffic = &desiredEnforceSecurityGroupInboundRulesOnPrivateLinkTraffic
218+
}
219+
203220
if _, err := m.elbv2Client.SetSecurityGroupsWithContext(ctx, req); err != nil {
204221
return err
205222
}
206223
m.logger.Info("modified loadBalancer securityGroups",
207224
"stackID", resLB.Stack().StackID(),
208225
"resourceID", resLB.ID(),
209-
"arn", awssdk.StringValue(sdkLB.LoadBalancer.LoadBalancerArn))
226+
"arn", awssdk.StringValue(sdkLB.LoadBalancer.LoadBalancerArn),
227+
"changeSecurityGroups", changeDescriptions,
228+
)
210229

211230
return nil
212231
}
@@ -298,3 +317,25 @@ func buildResLoadBalancerStatus(sdkLB LoadBalancerWithTags) elbv2model.LoadBalan
298317
DNSName: awssdk.StringValue(sdkLB.LoadBalancer.DNSName),
299318
}
300319
}
320+
321+
func isEnforceSGInboundRulesOnPrivateLinkUpdated(resLB *elbv2model.LoadBalancer, sdkLB LoadBalancerWithTags) (bool, string, string) {
322+
323+
if resLB.Spec.Type != elbv2model.LoadBalancerTypeNetwork || resLB.Spec.SecurityGroupsInboundRulesOnPrivateLink == nil {
324+
return false, "", ""
325+
}
326+
327+
desiredEnforceSecurityGroupInboundRulesOnPrivateLinkTraffic := string(*resLB.Spec.SecurityGroupsInboundRulesOnPrivateLink)
328+
329+
var currentEnforceSecurityGroupInboundRulesOnPrivateLinkTraffic string
330+
331+
if sdkLB.LoadBalancer.EnforceSecurityGroupInboundRulesOnPrivateLinkTraffic != nil {
332+
currentEnforceSecurityGroupInboundRulesOnPrivateLinkTraffic = awssdk.StringValue(sdkLB.LoadBalancer.EnforceSecurityGroupInboundRulesOnPrivateLinkTraffic)
333+
}
334+
335+
if desiredEnforceSecurityGroupInboundRulesOnPrivateLinkTraffic == currentEnforceSecurityGroupInboundRulesOnPrivateLinkTraffic {
336+
return false, "", ""
337+
}
338+
339+
return true, currentEnforceSecurityGroupInboundRulesOnPrivateLinkTraffic, desiredEnforceSecurityGroupInboundRulesOnPrivateLinkTraffic
340+
341+
}

‎pkg/model/elbv2/load_balancer.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package elbv2
22

33
import (
44
"context"
5+
56
"github.com/pkg/errors"
67
"sigs.k8s.io/aws-load-balancer-controller/pkg/model/core"
78
)
@@ -86,6 +87,13 @@ const (
8687
IPAddressTypeDualStack IPAddressType = "dualstack"
8788
)
8889

90+
type SecurityGroupsInboundRulesOnPrivateLinkStatus string
91+
92+
const (
93+
SecurityGroupsInboundRulesOnPrivateLinkOn SecurityGroupsInboundRulesOnPrivateLinkStatus = "on"
94+
SecurityGroupsInboundRulesOnPrivateLinkOff SecurityGroupsInboundRulesOnPrivateLinkStatus = "off"
95+
)
96+
8997
type LoadBalancerScheme string
9098

9199
const (
@@ -143,6 +151,10 @@ type LoadBalancerSpec struct {
143151
// +optional
144152
SecurityGroups []core.StringToken `json:"securityGroups,omitempty"`
145153

154+
// [Network Load Balancers] The status of the security groups inbound rules on private link.
155+
// +optional
156+
SecurityGroupsInboundRulesOnPrivateLink *SecurityGroupsInboundRulesOnPrivateLinkStatus `json:"securityGroupsInboundRulesOnPrivateLink,omitempty"`
157+
146158
// [Application Load Balancers on Outposts] The ID of the customer-owned address pool (CoIP pool).
147159
// +optional
148160
CustomerOwnedIPv4Pool *string `json:"customerOwnedIPv4Pool,omitempty"`

‎pkg/service/model_build_load_balancer.go

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,11 @@ func (t *defaultModelBuildTask) buildLoadBalancerSpec(ctx context.Context, schem
7777
if err != nil {
7878
return elbv2model.LoadBalancerSpec{}, err
7979
}
80+
securityGroupsInboundRulesOnPrivateLink, err := t.buildSecurityGroupsInboundRulesOnPrivateLink(ctx)
81+
if err != nil {
82+
return elbv2model.LoadBalancerSpec{}, err
83+
}
84+
8085
spec := elbv2model.LoadBalancerSpec{
8186
Name: name,
8287
Type: elbv2model.LoadBalancerTypeNetwork,
@@ -87,6 +92,11 @@ func (t *defaultModelBuildTask) buildLoadBalancerSpec(ctx context.Context, schem
8792
LoadBalancerAttributes: lbAttributes,
8893
Tags: tags,
8994
}
95+
96+
if securityGroupsInboundRulesOnPrivateLink != nil {
97+
spec.SecurityGroupsInboundRulesOnPrivateLink = securityGroupsInboundRulesOnPrivateLink
98+
}
99+
90100
return spec, nil
91101
}
92102

@@ -177,6 +187,23 @@ func (t *defaultModelBuildTask) buildLoadBalancerIPAddressType(_ context.Context
177187
}
178188
}
179189

190+
func (t *defaultModelBuildTask) buildSecurityGroupsInboundRulesOnPrivateLink(_ context.Context) (*elbv2model.SecurityGroupsInboundRulesOnPrivateLinkStatus, error) {
191+
var rawSecurityGroupsInboundRulesOnPrivateLink string
192+
if exists := t.annotationParser.ParseStringAnnotation(annotations.SvcLBSuffixEnforceSGInboundRulesOnPrivateLinkTraffic, &rawSecurityGroupsInboundRulesOnPrivateLink, t.service.Annotations); !exists {
193+
return nil, nil
194+
}
195+
securityGroupsInboundRulesOnPrivateLink := elbv2model.SecurityGroupsInboundRulesOnPrivateLinkStatus(rawSecurityGroupsInboundRulesOnPrivateLink)
196+
197+
switch securityGroupsInboundRulesOnPrivateLink {
198+
case elbv2model.SecurityGroupsInboundRulesOnPrivateLinkOn:
199+
return &securityGroupsInboundRulesOnPrivateLink, nil
200+
case elbv2model.SecurityGroupsInboundRulesOnPrivateLinkOff:
201+
return &securityGroupsInboundRulesOnPrivateLink, nil
202+
default:
203+
return nil, errors.Errorf("Invalid value for securityGroupsInboundRulesOnPrivateLink status: %v, value must be one of [%v, %v]", securityGroupsInboundRulesOnPrivateLink, string(elbv2model.SecurityGroupsInboundRulesOnPrivateLinkOn), string(elbv2model.SecurityGroupsInboundRulesOnPrivateLinkOff))
204+
}
205+
}
206+
180207
func (t *defaultModelBuildTask) buildLoadBalancerScheme(ctx context.Context) (elbv2model.LoadBalancerScheme, error) {
181208
scheme, explicitSchemeSpecified, err := t.buildLoadBalancerSchemeViaAnnotation(ctx)
182209
if err != nil {

‎pkg/service/model_builder.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"sync"
77

88
"github.com/aws/aws-sdk-go/service/ec2"
9+
"github.com/go-logr/logr"
910
"github.com/pkg/errors"
1011
corev1 "k8s.io/api/core/v1"
1112
"k8s.io/apimachinery/pkg/util/sets"
@@ -40,7 +41,7 @@ func NewDefaultModelBuilder(annotationParser annotations.Parser, subnetsResolver
4041
elbv2TaggingManager elbv2deploy.TaggingManager, ec2Client services.EC2, featureGates config.FeatureGates, clusterName string, defaultTags map[string]string,
4142
externalManagedTags []string, defaultSSLPolicy string, defaultTargetType string, enableIPTargetType bool, serviceUtils ServiceUtils,
4243
backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, enableBackendSG bool,
43-
disableRestrictedSGRules bool) *defaultModelBuilder {
44+
disableRestrictedSGRules bool, logger logr.Logger) *defaultModelBuilder {
4445
return &defaultModelBuilder{
4546
annotationParser: annotationParser,
4647
subnetsResolver: subnetsResolver,
@@ -61,6 +62,7 @@ func NewDefaultModelBuilder(annotationParser annotations.Parser, subnetsResolver
6162
ec2Client: ec2Client,
6263
enableBackendSG: enableBackendSG,
6364
disableRestrictedSGRules: disableRestrictedSGRules,
65+
logger: logger,
6466
}
6567
}
6668

@@ -87,6 +89,7 @@ type defaultModelBuilder struct {
8789
defaultSSLPolicy string
8890
defaultTargetType elbv2model.TargetType
8991
enableIPTargetType bool
92+
logger logr.Logger
9093
}
9194

9295
func (b *defaultModelBuilder) Build(ctx context.Context, service *corev1.Service) (core.Stack, *elbv2model.LoadBalancer, bool, error) {
@@ -107,6 +110,7 @@ func (b *defaultModelBuilder) Build(ctx context.Context, service *corev1.Service
107110
ec2Client: b.ec2Client,
108111
enableBackendSG: b.enableBackendSG,
109112
disableRestrictedSGRules: b.disableRestrictedSGRules,
113+
logger: b.logger,
110114

111115
service: service,
112116
stack: stack,
@@ -162,6 +166,7 @@ type defaultModelBuildTask struct {
162166
serviceUtils ServiceUtils
163167
enableIPTargetType bool
164168
ec2Client services.EC2
169+
logger logr.Logger
165170

166171
service *corev1.Service
167172

0 commit comments

Comments
 (0)