Skip to content

Commit fedae1a

Browse files
wweiwei-liwweiwei-li
committed
Add support for optionally enforcing NLB security groups on PrivateLink traffic
1 parent b63a294 commit fedae1a

File tree

7 files changed

+177
-66
lines changed

7 files changed

+177
-66
lines changed

‎controllers/service/service_controller.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ func NewServiceReconciler(cloud aws.Cloud, k8sClient client.Client, eventRecorde
4747
modelBuilder := service.NewDefaultModelBuilder(annotationParser, subnetsResolver, vpcInfoProvider, cloud.VpcID(), trackingProvider,
4848
elbv2TaggingManager, cloud.EC2(), controllerConfig.FeatureGates, controllerConfig.ClusterName, controllerConfig.DefaultTags, controllerConfig.ExternalManagedTags,
4949
controllerConfig.DefaultSSLPolicy, controllerConfig.DefaultTargetType, controllerConfig.FeatureGates.Enabled(config.EnableIPTargetType), serviceUtils,
50-
backendSGProvider, sgResolver, controllerConfig.EnableBackendSecurityGroup, controllerConfig.DisableRestrictedSGRules)
50+
backendSGProvider, sgResolver, controllerConfig.EnableBackendSecurityGroup, controllerConfig.DisableRestrictedSGRules, logger)
5151
stackMarshaller := deploy.NewDefaultStackMarshaller()
5252
stackDeployer := deploy.NewDefaultStackDeployer(cloud, k8sClient, networkingSGManager, networkingSGReconciler, elbv2TaggingManager, controllerConfig, serviceTagPrefix, logger)
5353
return &serviceReconciler{

‎pkg/annotations/constants.go

Lines changed: 36 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -50,39 +50,40 @@ const (
5050

5151
// NLB annotation suffixes
5252
// prefixes service.beta.kubernetes.io, service.kubernetes.io
53-
SvcLBSuffixSourceRanges = "load-balancer-source-ranges"
54-
SvcLBSuffixLoadBalancerType = "aws-load-balancer-type"
55-
SvcLBSuffixTargetType = "aws-load-balancer-nlb-target-type"
56-
SvcLBSuffixLoadBalancerName = "aws-load-balancer-name"
57-
SvcLBSuffixScheme = "aws-load-balancer-scheme"
58-
SvcLBSuffixInternal = "aws-load-balancer-internal"
59-
SvcLBSuffixProxyProtocol = "aws-load-balancer-proxy-protocol"
60-
SvcLBSuffixIPAddressType = "aws-load-balancer-ip-address-type"
61-
SvcLBSuffixAccessLogEnabled = "aws-load-balancer-access-log-enabled"
62-
SvcLBSuffixAccessLogS3BucketName = "aws-load-balancer-access-log-s3-bucket-name"
63-
SvcLBSuffixAccessLogS3BucketPrefix = "aws-load-balancer-access-log-s3-bucket-prefix"
64-
SvcLBSuffixCrossZoneLoadBalancingEnabled = "aws-load-balancer-cross-zone-load-balancing-enabled"
65-
SvcLBSuffixSSLCertificate = "aws-load-balancer-ssl-cert"
66-
SvcLBSuffixSSLPorts = "aws-load-balancer-ssl-ports"
67-
SvcLBSuffixSSLNegotiationPolicy = "aws-load-balancer-ssl-negotiation-policy"
68-
SvcLBSuffixBEProtocol = "aws-load-balancer-backend-protocol"
69-
SvcLBSuffixAdditionalTags = "aws-load-balancer-additional-resource-tags"
70-
SvcLBSuffixHCHealthyThreshold = "aws-load-balancer-healthcheck-healthy-threshold"
71-
SvcLBSuffixHCUnhealthyThreshold = "aws-load-balancer-healthcheck-unhealthy-threshold"
72-
SvcLBSuffixHCTimeout = "aws-load-balancer-healthcheck-timeout"
73-
SvcLBSuffixHCInterval = "aws-load-balancer-healthcheck-interval"
74-
SvcLBSuffixHCProtocol = "aws-load-balancer-healthcheck-protocol"
75-
SvcLBSuffixHCPort = "aws-load-balancer-healthcheck-port"
76-
SvcLBSuffixHCPath = "aws-load-balancer-healthcheck-path"
77-
SvcLBSuffixHCSuccessCodes = "aws-load-balancer-healthcheck-success-codes"
78-
SvcLBSuffixTargetGroupAttributes = "aws-load-balancer-target-group-attributes"
79-
SvcLBSuffixSubnets = "aws-load-balancer-subnets"
80-
SvcLBSuffixEIPAllocations = "aws-load-balancer-eip-allocations"
81-
SvcLBSuffixPrivateIpv4Addresses = "aws-load-balancer-private-ipv4-addresses"
82-
SvcLBSuffixIpv6Addresses = "aws-load-balancer-ipv6-addresses"
83-
SvcLBSuffixALPNPolicy = "aws-load-balancer-alpn-policy"
84-
SvcLBSuffixTargetNodeLabels = "aws-load-balancer-target-node-labels"
85-
SvcLBSuffixLoadBalancerAttributes = "aws-load-balancer-attributes"
86-
SvcLBSuffixLoadBalancerSecurityGroups = "aws-load-balancer-security-groups"
87-
SvcLBSuffixManageSGRules = "aws-load-balancer-manage-backend-security-group-rules"
53+
SvcLBSuffixSourceRanges = "load-balancer-source-ranges"
54+
SvcLBSuffixLoadBalancerType = "aws-load-balancer-type"
55+
SvcLBSuffixTargetType = "aws-load-balancer-nlb-target-type"
56+
SvcLBSuffixLoadBalancerName = "aws-load-balancer-name"
57+
SvcLBSuffixScheme = "aws-load-balancer-scheme"
58+
SvcLBSuffixInternal = "aws-load-balancer-internal"
59+
SvcLBSuffixProxyProtocol = "aws-load-balancer-proxy-protocol"
60+
SvcLBSuffixIPAddressType = "aws-load-balancer-ip-address-type"
61+
SvcLBSuffixAccessLogEnabled = "aws-load-balancer-access-log-enabled"
62+
SvcLBSuffixAccessLogS3BucketName = "aws-load-balancer-access-log-s3-bucket-name"
63+
SvcLBSuffixAccessLogS3BucketPrefix = "aws-load-balancer-access-log-s3-bucket-prefix"
64+
SvcLBSuffixCrossZoneLoadBalancingEnabled = "aws-load-balancer-cross-zone-load-balancing-enabled"
65+
SvcLBSuffixSSLCertificate = "aws-load-balancer-ssl-cert"
66+
SvcLBSuffixSSLPorts = "aws-load-balancer-ssl-ports"
67+
SvcLBSuffixSSLNegotiationPolicy = "aws-load-balancer-ssl-negotiation-policy"
68+
SvcLBSuffixBEProtocol = "aws-load-balancer-backend-protocol"
69+
SvcLBSuffixAdditionalTags = "aws-load-balancer-additional-resource-tags"
70+
SvcLBSuffixHCHealthyThreshold = "aws-load-balancer-healthcheck-healthy-threshold"
71+
SvcLBSuffixHCUnhealthyThreshold = "aws-load-balancer-healthcheck-unhealthy-threshold"
72+
SvcLBSuffixHCTimeout = "aws-load-balancer-healthcheck-timeout"
73+
SvcLBSuffixHCInterval = "aws-load-balancer-healthcheck-interval"
74+
SvcLBSuffixHCProtocol = "aws-load-balancer-healthcheck-protocol"
75+
SvcLBSuffixHCPort = "aws-load-balancer-healthcheck-port"
76+
SvcLBSuffixHCPath = "aws-load-balancer-healthcheck-path"
77+
SvcLBSuffixHCSuccessCodes = "aws-load-balancer-healthcheck-success-codes"
78+
SvcLBSuffixTargetGroupAttributes = "aws-load-balancer-target-group-attributes"
79+
SvcLBSuffixSubnets = "aws-load-balancer-subnets"
80+
SvcLBSuffixEIPAllocations = "aws-load-balancer-eip-allocations"
81+
SvcLBSuffixPrivateIpv4Addresses = "aws-load-balancer-private-ipv4-addresses"
82+
SvcLBSuffixIpv6Addresses = "aws-load-balancer-ipv6-addresses"
83+
SvcLBSuffixALPNPolicy = "aws-load-balancer-alpn-policy"
84+
SvcLBSuffixTargetNodeLabels = "aws-load-balancer-target-node-labels"
85+
SvcLBSuffixLoadBalancerAttributes = "aws-load-balancer-attributes"
86+
SvcLBSuffixLoadBalancerSecurityGroups = "aws-load-balancer-security-groups"
87+
SvcLBSuffixManageSGRules = "aws-load-balancer-manage-backend-security-group-rules"
88+
SvcLBSuffixEnforceSGInboundRulesOnPrivateLinkTraffic = "aws-load-balancer-enforce-sg-inbound-private-link-traffic"
8889
)

‎pkg/deploy/elbv2/load_balancer_manager.go

Lines changed: 46 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package elbv2
33
import (
44
"context"
55
"fmt"
6+
67
awssdk "github.com/aws/aws-sdk-go/aws"
78
elbv2sdk "github.com/aws/aws-sdk-go/service/elbv2"
89
"github.com/go-logr/logr"
@@ -75,6 +76,12 @@ func (m *defaultLoadBalancerManager) Create(ctx context.Context, resLB *elbv2mod
7576
return elbv2model.LoadBalancerStatus{}, err
7677
}
7778

79+
if resLB.Spec.Type == elbv2model.LoadBalancerTypeNetwork && string(*resLB.Spec.SecurityGroupsInboundRulesOnPrivateLink) != "" {
80+
if err := m.updateSDKLoadBalancerWithSecurityGroups(ctx, resLB, sdkLB); err != nil {
81+
return elbv2model.LoadBalancerStatus{}, err
82+
}
83+
}
84+
7885
return buildResLoadBalancerStatus(sdkLB), nil
7986
}
8087

@@ -186,20 +193,52 @@ func (m *defaultLoadBalancerManager) updateSDKLoadBalancerWithSecurityGroups(ctx
186193
}
187194
desiredSecurityGroups := sets.NewString(awssdk.StringValueSlice(securityGroups)...)
188195
currentSecurityGroups := sets.NewString(awssdk.StringValueSlice(sdkLB.LoadBalancer.SecurityGroups)...)
189-
if desiredSecurityGroups.Equal(currentSecurityGroups) {
196+
197+
if desiredSecurityGroups.Equal(currentSecurityGroups) && (resLB.Spec.Type != elbv2model.LoadBalancerTypeNetwork || string(*resLB.Spec.SecurityGroupsInboundRulesOnPrivateLink) == "") {
190198
return nil
191199
}
192200

201+
if !desiredSecurityGroups.Equal(currentSecurityGroups) {
202+
changeSecurityGroupsDesc := fmt.Sprintf("%v => %v", currentSecurityGroups.List(), desiredSecurityGroups.List())
203+
m.logger.Info("modifying loadBalancer security groups",
204+
"stackID", resLB.Stack().StackID(),
205+
"resourceID", resLB.ID(),
206+
"arn", awssdk.StringValue(sdkLB.LoadBalancer.LoadBalancerArn),
207+
"changeSecurityGroups", changeSecurityGroupsDesc)
208+
}
209+
193210
req := &elbv2sdk.SetSecurityGroupsInput{
194211
LoadBalancerArn: sdkLB.LoadBalancer.LoadBalancerArn,
195212
SecurityGroups: securityGroups,
196213
}
197-
changeDesc := fmt.Sprintf("%v => %v", currentSecurityGroups.List(), desiredSecurityGroups.List())
198-
m.logger.Info("modifying loadBalancer securityGroups",
199-
"stackID", resLB.Stack().StackID(),
200-
"resourceID", resLB.ID(),
201-
"arn", awssdk.StringValue(sdkLB.LoadBalancer.LoadBalancerArn),
202-
"change", changeDesc)
214+
215+
if resLB.Spec.Type == elbv2model.LoadBalancerTypeNetwork {
216+
desiredEnforceSecurityGroupInboundRulesOnPrivateLinkTraffic := string(*resLB.Spec.SecurityGroupsInboundRulesOnPrivateLink)
217+
currentEnforceSecurityGroupInboundRulesOnPrivateLinkTraffic := "on"
218+
if sdkLB.LoadBalancer.EnforceSecurityGroupInboundRulesOnPrivateLinkTraffic != nil {
219+
currentEnforceSecurityGroupInboundRulesOnPrivateLinkTraffic = awssdk.StringValue(sdkLB.LoadBalancer.EnforceSecurityGroupInboundRulesOnPrivateLinkTraffic)
220+
}
221+
if desiredSecurityGroups.Equal(currentSecurityGroups) && desiredEnforceSecurityGroupInboundRulesOnPrivateLinkTraffic == currentEnforceSecurityGroupInboundRulesOnPrivateLinkTraffic {
222+
return nil
223+
}
224+
225+
if string(*resLB.Spec.SecurityGroupsInboundRulesOnPrivateLink) != "" && desiredEnforceSecurityGroupInboundRulesOnPrivateLinkTraffic != currentEnforceSecurityGroupInboundRulesOnPrivateLinkTraffic {
226+
changeEnforceSecurityGroupInboundRulesOnPrivateLinkTrafficDesc := fmt.Sprintf("%v => %v", currentEnforceSecurityGroupInboundRulesOnPrivateLinkTraffic, desiredEnforceSecurityGroupInboundRulesOnPrivateLinkTraffic)
227+
m.logger.Info("modifying loadBalancer enforce security group inbound rules on privateLink traffic",
228+
"stackID", resLB.Stack().StackID(),
229+
"resourceID", resLB.ID(),
230+
"arn", awssdk.StringValue(sdkLB.LoadBalancer.LoadBalancerArn),
231+
"changeEnforceSecurityGroupInboundRulesOnPrivateLinkTraffic", changeEnforceSecurityGroupInboundRulesOnPrivateLinkTrafficDesc)
232+
233+
req = &elbv2sdk.SetSecurityGroupsInput{
234+
LoadBalancerArn: sdkLB.LoadBalancer.LoadBalancerArn,
235+
SecurityGroups: securityGroups,
236+
EnforceSecurityGroupInboundRulesOnPrivateLinkTraffic: awssdk.String(desiredEnforceSecurityGroupInboundRulesOnPrivateLinkTraffic),
237+
}
238+
}
239+
240+
}
241+
203242
if _, err := m.elbv2Client.SetSecurityGroupsWithContext(ctx, req); err != nil {
204243
return err
205244
}

‎pkg/model/elbv2/load_balancer.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package elbv2
22

33
import (
44
"context"
5+
56
"github.com/pkg/errors"
67
"sigs.k8s.io/aws-load-balancer-controller/pkg/model/core"
78
)
@@ -86,6 +87,13 @@ const (
8687
IPAddressTypeDualStack IPAddressType = "dualstack"
8788
)
8889

90+
type SecurityGroupsInboundRulesOnPrivateLinkStatus string
91+
92+
const (
93+
SecurityGroupsInboundRulesOnPrivateLinkOn SecurityGroupsInboundRulesOnPrivateLinkStatus = "on"
94+
SecurityGroupsInboundRulesOnPrivateLinkOff SecurityGroupsInboundRulesOnPrivateLinkStatus = "off"
95+
)
96+
8997
type LoadBalancerScheme string
9098

9199
const (
@@ -143,6 +151,10 @@ type LoadBalancerSpec struct {
143151
// +optional
144152
SecurityGroups []core.StringToken `json:"securityGroups,omitempty"`
145153

154+
// [Network Load Balancers] The status of the security groups inbound rules on private link.
155+
// +optional
156+
SecurityGroupsInboundRulesOnPrivateLink *SecurityGroupsInboundRulesOnPrivateLinkStatus `json:"securityGroupsInboundRulesOnPrivateLink,omitempty"`
157+
146158
// [Application Load Balancers on Outposts] The ID of the customer-owned address pool (CoIP pool).
147159
// +optional
148160
CustomerOwnedIPv4Pool *string `json:"customerOwnedIPv4Pool,omitempty"`

‎pkg/service/model_build_load_balancer.go

Lines changed: 30 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -73,15 +73,21 @@ func (t *defaultModelBuildTask) buildLoadBalancerSpec(ctx context.Context, schem
7373
if err != nil {
7474
return elbv2model.LoadBalancerSpec{}, err
7575
}
76+
securityGroupsInboundRulesOnPrivateLink, err := t.buildSecurityGroupsInboundRulesOnPrivateLink(ctx)
77+
if err != nil {
78+
return elbv2model.LoadBalancerSpec{}, err
79+
}
80+
7681
spec := elbv2model.LoadBalancerSpec{
77-
Name: name,
78-
Type: elbv2model.LoadBalancerTypeNetwork,
79-
Scheme: &scheme,
80-
IPAddressType: &ipAddressType,
81-
SecurityGroups: securityGroups,
82-
SubnetMappings: subnetMappings,
83-
LoadBalancerAttributes: lbAttributes,
84-
Tags: tags,
82+
Name: name,
83+
Type: elbv2model.LoadBalancerTypeNetwork,
84+
Scheme: &scheme,
85+
IPAddressType: &ipAddressType,
86+
SecurityGroups: securityGroups,
87+
SubnetMappings: subnetMappings,
88+
LoadBalancerAttributes: lbAttributes,
89+
Tags: tags,
90+
SecurityGroupsInboundRulesOnPrivateLink: &securityGroupsInboundRulesOnPrivateLink,
8591
}
8692
return spec, nil
8793
}
@@ -173,6 +179,22 @@ func (t *defaultModelBuildTask) buildLoadBalancerIPAddressType(_ context.Context
173179
}
174180
}
175181

182+
func (t *defaultModelBuildTask) buildSecurityGroupsInboundRulesOnPrivateLink(_ context.Context) (elbv2model.SecurityGroupsInboundRulesOnPrivateLinkStatus, error) {
183+
securityGroupsInboundRulesOnPrivateLink := ""
184+
if exists := t.annotationParser.ParseStringAnnotation(annotations.SvcLBSuffixEnforceSGInboundRulesOnPrivateLinkTraffic, &securityGroupsInboundRulesOnPrivateLink, t.service.Annotations); !exists {
185+
return "", nil
186+
}
187+
188+
switch securityGroupsInboundRulesOnPrivateLink {
189+
case string(elbv2model.SecurityGroupsInboundRulesOnPrivateLinkOn):
190+
return elbv2model.SecurityGroupsInboundRulesOnPrivateLinkOn, nil
191+
case string(elbv2model.SecurityGroupsInboundRulesOnPrivateLinkOff):
192+
return elbv2model.SecurityGroupsInboundRulesOnPrivateLinkOff, nil
193+
default:
194+
return "", errors.Errorf("Invalid value for securityGroupsInboundRulesOnPrivateLink status: %v, value must be one of [%v, %v]", securityGroupsInboundRulesOnPrivateLink, string(elbv2model.SecurityGroupsInboundRulesOnPrivateLinkOn), string(elbv2model.SecurityGroupsInboundRulesOnPrivateLinkOff))
195+
}
196+
}
197+
176198
func (t *defaultModelBuildTask) buildLoadBalancerScheme(ctx context.Context) (elbv2model.LoadBalancerScheme, error) {
177199
scheme, explicitSchemeSpecified, err := t.buildLoadBalancerSchemeViaAnnotation(ctx)
178200
if err != nil {

‎pkg/service/model_builder.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"sync"
77

88
"github.com/aws/aws-sdk-go/service/ec2"
9+
"github.com/go-logr/logr"
910
"github.com/pkg/errors"
1011
corev1 "k8s.io/api/core/v1"
1112
"k8s.io/apimachinery/pkg/util/sets"
@@ -40,7 +41,7 @@ func NewDefaultModelBuilder(annotationParser annotations.Parser, subnetsResolver
4041
elbv2TaggingManager elbv2deploy.TaggingManager, ec2Client services.EC2, featureGates config.FeatureGates, clusterName string, defaultTags map[string]string,
4142
externalManagedTags []string, defaultSSLPolicy string, defaultTargetType string, enableIPTargetType bool, serviceUtils ServiceUtils,
4243
backendSGProvider networking.BackendSGProvider, sgResolver networking.SecurityGroupResolver, enableBackendSG bool,
43-
disableRestrictedSGRules bool) *defaultModelBuilder {
44+
disableRestrictedSGRules bool, logger logr.Logger) *defaultModelBuilder {
4445
return &defaultModelBuilder{
4546
annotationParser: annotationParser,
4647
subnetsResolver: subnetsResolver,
@@ -61,6 +62,7 @@ func NewDefaultModelBuilder(annotationParser annotations.Parser, subnetsResolver
6162
ec2Client: ec2Client,
6263
enableBackendSG: enableBackendSG,
6364
disableRestrictedSGRules: disableRestrictedSGRules,
65+
logger: logger,
6466
}
6567
}
6668

@@ -87,6 +89,7 @@ type defaultModelBuilder struct {
8789
defaultSSLPolicy string
8890
defaultTargetType elbv2model.TargetType
8991
enableIPTargetType bool
92+
logger logr.Logger
9093
}
9194

9295
func (b *defaultModelBuilder) Build(ctx context.Context, service *corev1.Service) (core.Stack, *elbv2model.LoadBalancer, bool, error) {
@@ -107,6 +110,7 @@ func (b *defaultModelBuilder) Build(ctx context.Context, service *corev1.Service
107110
ec2Client: b.ec2Client,
108111
enableBackendSG: b.enableBackendSG,
109112
disableRestrictedSGRules: b.disableRestrictedSGRules,
113+
logger: b.logger,
110114

111115
service: service,
112116
stack: stack,
@@ -162,6 +166,7 @@ type defaultModelBuildTask struct {
162166
serviceUtils ServiceUtils
163167
enableIPTargetType bool
164168
ec2Client services.EC2
169+
logger logr.Logger
165170

166171
service *corev1.Service
167172

0 commit comments

Comments
 (0)