Skip to content

Commit 48d7533

Browse files
authored
add reconciler for securityGroup rules (#1421)
* add reconciler for securityGroup rules * add networking manager for targetGroupBindings * add sg deployer * fix crd types * add tags handling for securityGroup
1 parent 0288b8e commit 48d7533

19 files changed

+1388
-151
lines changed

apis/elbv2/v1alpha1/targetgroupbinding_types.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ type NetworkingPort struct {
7979
// The port which traffic must match.
8080
// If unspecified, defaults to all port.
8181
// +optional
82-
Port *intstr.IntOrString `json:"port,omitempty"`
82+
Port *int64 `json:"port,omitempty"`
8383

8484
// The protocol which traffic must match.
8585
// If unspecified, defaults to all protocol.

apis/elbv2/v1alpha1/zz_generated.deepcopy.go

Lines changed: 1 addition & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

config/crd/bases/elbv2.k8s.aws_targetgroupbindings.yaml

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -81,12 +81,10 @@ spec:
8181
items:
8282
properties:
8383
port:
84-
anyOf:
85-
- type: integer
86-
- type: string
8784
description: The port which traffic must match. If unspecified,
8885
defaults to all port.
89-
x-kubernetes-int-or-string: true
86+
format: int64
87+
type: integer
9088
protocol:
9189
description: The protocol which traffic must match.
9290
If unspecified, defaults to all protocol.

controllers/ingress/group_controller.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,15 +20,16 @@ import (
2020
const controllerName = "ingress"
2121

2222
// NewGroupReconciler constructs new GroupReconciler
23-
func NewGroupReconciler(k8sClient client.Client, eventRecorder record.EventRecorder, ec2Client services.EC2, elbv2Client services.ELBV2, vpcID string, clusterName string,
23+
func NewGroupReconciler(k8sClient client.Client, eventRecorder record.EventRecorder, ec2Client services.EC2, elbv2Client services.ELBV2,
24+
networkingSGManager networkingpkg.SecurityGroupManager, networkingSGReconciler networkingpkg.SecurityGroupReconciler, vpcID string, clusterName string,
2425
subnetsResolver networkingpkg.SubnetsResolver, logger logr.Logger) *GroupReconciler {
2526
annotationParser := annotations.NewSuffixAnnotationParser("alb.ingress.kubernetes.io")
2627
authConfigBuilder := ingress.NewDefaultAuthConfigBuilder(annotationParser)
2728
enhancedBackendBuilder := ingress.NewDefaultEnhancedBackendBuilder(annotationParser)
2829
modelBuilder := ingress.NewDefaultModelBuilder(k8sClient, eventRecorder, ec2Client, vpcID, clusterName, annotationParser, subnetsResolver,
2930
authConfigBuilder, enhancedBackendBuilder)
3031
stackMarshaller := deploy.NewDefaultStackMarshaller()
31-
stackDeployer := deploy.NewDefaultStackDeployer(k8sClient, elbv2Client, vpcID, clusterName, "ingress.k8s.aws", logger)
32+
stackDeployer := deploy.NewDefaultStackDeployer(k8sClient, ec2Client, elbv2Client, networkingSGManager, networkingSGReconciler, vpcID, clusterName, "ingress.k8s.aws", logger)
3233
groupLoader := ingress.NewDefaultGroupLoader(k8sClient, annotationParser, "alb")
3334
finalizerManager := ingress.NewDefaultFinalizerManager(k8sClient)
3435

controllers/service/service_controller.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,15 +28,17 @@ const (
2828
controllerName = "service"
2929
)
3030

31-
func NewServiceReconciler(k8sClient client.Client, elbv2Client services.ELBV2, vpcID string, clusterName string, resolver networking.SubnetsResolver, logger logr.Logger) *ServiceReconciler {
31+
func NewServiceReconciler(k8sClient client.Client, ec2Client services.EC2, elbv2Client services.ELBV2,
32+
sgManager networking.SecurityGroupManager, sgReconciler networking.SecurityGroupReconciler,
33+
vpcID string, clusterName string, resolver networking.SubnetsResolver, logger logr.Logger) *ServiceReconciler {
3234
return &ServiceReconciler{
3335
k8sClient: k8sClient,
3436
logger: logger,
3537
annotationParser: annotations.NewSuffixAnnotationParser(ServiceAnnotationPrefix),
3638
finalizerManager: k8s.NewDefaultFinalizerManager(k8sClient, logger),
3739
subnetsResolver: resolver,
3840
stackMarshaller: deploy.NewDefaultStackMarshaller(),
39-
stackDeployer: deploy.NewDefaultStackDeployer(k8sClient, elbv2Client, vpcID, clusterName, DefaultTagPrefix, logger),
41+
stackDeployer: deploy.NewDefaultStackDeployer(k8sClient, ec2Client, elbv2Client, sgManager, sgReconciler, vpcID, clusterName, DefaultTagPrefix, logger),
4042
}
4143
}
4244

main.go

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -101,24 +101,33 @@ func main() {
101101
os.Exit(1)
102102
}
103103

104-
subnetResolver := networking.NewSubnetsResolver(cloud.EC2(), cloud.VpcID(), k8sClusterName, ctrl.Log.WithName("subnets-resolver"))
105-
ingGroupReconciler := ingress.NewGroupReconciler(mgr.GetClient(), mgr.GetEventRecorderFor("ingress"), cloud.EC2(), cloud.ELBV2(), cloud.VpcID(), k8sClusterName, subnetResolver, ctrl.Log)
106-
if err = ingGroupReconciler.SetupWithManager(mgr); err != nil {
107-
setupLog.Error(err, "unable to create controller", "controller", "Ingress")
108-
os.Exit(1)
109-
}
110-
104+
podENIResolver := networking.NewDefaultPodENIInfoResolver(cloud.EC2(), cloud.VpcID(), ctrl.Log)
105+
nodeENIResolver := networking.NewDefaultNodeENIInfoResolver(cloud.EC2(), ctrl.Log)
106+
sgManager := networking.NewDefaultSecurityGroupManager(cloud.EC2(), ctrl.Log)
107+
sgReconciler := networking.NewDefaultSecurityGroupReconciler(sgManager, ctrl.Log)
111108
finalizerManager := k8s.NewDefaultFinalizerManager(mgr.GetClient(), ctrl.Log)
112-
tgbResManager := targetgroupbinding.NewDefaultResourceManager(mgr.GetClient(), cloud.ELBV2(), ctrl.Log)
109+
tgbResManager := targetgroupbinding.NewDefaultResourceManager(mgr.GetClient(), cloud.ELBV2(),
110+
podENIResolver, nodeENIResolver, sgManager, sgReconciler, cloud.VpcID(), k8sClusterName, ctrl.Log)
111+
112+
subnetResolver := networking.NewSubnetsResolver(cloud.EC2(), cloud.VpcID(), k8sClusterName, ctrl.Log.WithName("subnets-resolver"))
113+
ingGroupReconciler := ingress.NewGroupReconciler(mgr.GetClient(), mgr.GetEventRecorderFor("ingress"), cloud.EC2(), cloud.ELBV2(),
114+
sgManager, sgReconciler, cloud.VpcID(), k8sClusterName, subnetResolver, ctrl.Log)
113115
tgbReconciler := elbv2controller.NewTargetGroupBindingReconciler(mgr.GetClient(), mgr.GetFieldIndexer(), finalizerManager, tgbResManager,
114116
ctrl.Log.WithName("controllers").WithName("TargetGroupBinding"))
115117
svcReconciler := service.NewServiceReconciler(
116118
mgr.GetClient(),
119+
cloud.EC2(),
117120
cloud.ELBV2(),
121+
sgManager,
122+
sgReconciler,
118123
cloud.VpcID(),
119124
k8sClusterName,
120125
subnetResolver,
121126
ctrl.Log.WithName("controllers").WithName("Service"))
127+
if err = ingGroupReconciler.SetupWithManager(mgr); err != nil {
128+
setupLog.Error(err, "unable to create controller", "controller", "Ingress")
129+
os.Exit(1)
130+
}
122131
if err := tgbReconciler.SetupWithManager(context.Background(), mgr); err != nil {
123132
setupLog.Error(err, "unable to create controller", "controller", "TargetGroupBinding")
124133
os.Exit(1)
Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
package ec2
2+
3+
import (
4+
"context"
5+
"errors"
6+
awssdk "github.com/aws/aws-sdk-go/aws"
7+
ec2sdk "github.com/aws/aws-sdk-go/service/ec2"
8+
"github.com/go-logr/logr"
9+
"k8s.io/apimachinery/pkg/util/sets"
10+
"sigs.k8s.io/aws-alb-ingress-controller/pkg/algorithm"
11+
"sigs.k8s.io/aws-alb-ingress-controller/pkg/aws/services"
12+
"sigs.k8s.io/aws-alb-ingress-controller/pkg/deploy/tagging"
13+
ec2model "sigs.k8s.io/aws-alb-ingress-controller/pkg/model/ec2"
14+
"sigs.k8s.io/aws-alb-ingress-controller/pkg/networking"
15+
)
16+
17+
// SecurityGroupManager is responsible for create/update/delete SecurityGroup resources.
18+
type SecurityGroupManager interface {
19+
Create(ctx context.Context, resSG *ec2model.SecurityGroup) (ec2model.SecurityGroupStatus, error)
20+
21+
Update(ctx context.Context, resSG *ec2model.SecurityGroup, sdkSG networking.SecurityGroupInfo) (ec2model.SecurityGroupStatus, error)
22+
23+
Delete(ctx context.Context, sdkSG networking.SecurityGroupInfo) error
24+
}
25+
26+
// NewDefaultSecurityGroupManager constructs new defaultSecurityGroupManager.
27+
func NewDefaultSecurityGroupManager(ec2Client services.EC2, taggingProvider tagging.Provider, networkingSGReconciler networking.SecurityGroupReconciler, vpcID string, logger logr.Logger) *defaultSecurityGroupManager {
28+
return &defaultSecurityGroupManager{
29+
ec2Client: ec2Client,
30+
taggingProvider: taggingProvider,
31+
networkingSGReconciler: networkingSGReconciler,
32+
vpcID: vpcID,
33+
logger: logger,
34+
}
35+
}
36+
37+
// default implementation for SecurityGroupManager.
38+
type defaultSecurityGroupManager struct {
39+
ec2Client services.EC2
40+
taggingProvider tagging.Provider
41+
networkingSGReconciler networking.SecurityGroupReconciler
42+
vpcID string
43+
logger logr.Logger
44+
}
45+
46+
func (m *defaultSecurityGroupManager) Create(ctx context.Context, resSG *ec2model.SecurityGroup) (ec2model.SecurityGroupStatus, error) {
47+
sgTags := m.taggingProvider.ResourceTags(resSG.Stack(), resSG, resSG.Spec.Tags)
48+
sdkTags := convertTagsToSDKTags(sgTags)
49+
permissionInfos, err := buildIPPermissionInfos(resSG.Spec.Ingress)
50+
if err != nil {
51+
return ec2model.SecurityGroupStatus{}, err
52+
}
53+
54+
req := &ec2sdk.CreateSecurityGroupInput{
55+
VpcId: awssdk.String(m.vpcID),
56+
GroupName: awssdk.String(resSG.Spec.GroupName),
57+
Description: awssdk.String(resSG.Spec.Description),
58+
TagSpecifications: []*ec2sdk.TagSpecification{
59+
{
60+
ResourceType: awssdk.String("security-group"),
61+
Tags: sdkTags,
62+
},
63+
},
64+
}
65+
m.logger.Info("creating securityGroup",
66+
"resourceID", resSG.ID())
67+
resp, err := m.ec2Client.CreateSecurityGroupWithContext(ctx, req)
68+
if err != nil {
69+
return ec2model.SecurityGroupStatus{}, err
70+
}
71+
sgID := awssdk.StringValue(resp.GroupId)
72+
m.logger.Info("created securityGroup",
73+
"resourceID", resSG.ID(),
74+
"securityGroupID", sgID)
75+
76+
if err := m.networkingSGReconciler.ReconcileIngress(ctx, sgID, permissionInfos); err != nil {
77+
return ec2model.SecurityGroupStatus{}, err
78+
}
79+
80+
return ec2model.SecurityGroupStatus{
81+
GroupID: sgID,
82+
}, nil
83+
}
84+
85+
func (m *defaultSecurityGroupManager) Update(ctx context.Context, resSG *ec2model.SecurityGroup, sdkSG networking.SecurityGroupInfo) (ec2model.SecurityGroupStatus, error) {
86+
permissionInfos, err := buildIPPermissionInfos(resSG.Spec.Ingress)
87+
if err != nil {
88+
return ec2model.SecurityGroupStatus{}, err
89+
}
90+
if err := m.updateSDKSecurityGroupGroupWithTags(ctx, resSG, sdkSG); err != nil {
91+
return ec2model.SecurityGroupStatus{}, err
92+
}
93+
if err := m.networkingSGReconciler.ReconcileIngress(ctx, sdkSG.SecurityGroupID, permissionInfos); err != nil {
94+
return ec2model.SecurityGroupStatus{}, err
95+
}
96+
return ec2model.SecurityGroupStatus{
97+
GroupID: sdkSG.SecurityGroupID,
98+
}, nil
99+
}
100+
101+
func (m *defaultSecurityGroupManager) Delete(ctx context.Context, sdkSG networking.SecurityGroupInfo) error {
102+
req := &ec2sdk.DeleteSecurityGroupInput{
103+
GroupId: awssdk.String(sdkSG.SecurityGroupID),
104+
}
105+
m.logger.Info("deleting securityGroup",
106+
"securityGroupID", sdkSG.SecurityGroupID)
107+
if _, err := m.ec2Client.DeleteSecurityGroupWithContext(ctx, req); err != nil {
108+
return err
109+
}
110+
m.logger.Info("deleted securityGroup",
111+
"securityGroupID", sdkSG.SecurityGroupID)
112+
return nil
113+
}
114+
115+
func (m *defaultSecurityGroupManager) updateSDKSecurityGroupGroupWithTags(ctx context.Context, resSG *ec2model.SecurityGroup, sdkSG networking.SecurityGroupInfo) error {
116+
desiredTags := m.taggingProvider.ResourceTags(resSG.Stack(), resSG, resSG.Spec.Tags)
117+
tagsToUpdate, tagsToRemove := algorithm.DiffStringMap(desiredTags, sdkSG.Tags)
118+
if len(tagsToUpdate) > 0 {
119+
req := &ec2sdk.CreateTagsInput{
120+
Resources: []*string{awssdk.String(sdkSG.SecurityGroupID)},
121+
Tags: convertTagsToSDKTags(tagsToUpdate),
122+
}
123+
124+
m.logger.Info("adding securityGroup tags",
125+
"securityGroupID", sdkSG.SecurityGroupID,
126+
"change", tagsToUpdate)
127+
if _, err := m.ec2Client.CreateTagsWithContext(ctx, req); err != nil {
128+
return err
129+
}
130+
m.logger.Info("added securityGroup tags",
131+
"securityGroupID", sdkSG.SecurityGroupID)
132+
}
133+
134+
if len(tagsToRemove) > 0 {
135+
req := &ec2sdk.DeleteTagsInput{
136+
Resources: []*string{awssdk.String(sdkSG.SecurityGroupID)},
137+
Tags: convertTagsToSDKTags(tagsToRemove),
138+
}
139+
140+
m.logger.Info("removing securityGroup tags",
141+
"securityGroupID", sdkSG.SecurityGroupID,
142+
"change", tagsToRemove)
143+
if _, err := m.ec2Client.DeleteTagsWithContext(ctx, req); err != nil {
144+
return err
145+
}
146+
m.logger.Info("removed securityGroup tags",
147+
"securityGroupID", sdkSG.SecurityGroupID)
148+
}
149+
return nil
150+
}
151+
152+
func buildIPPermissionInfos(permissions []ec2model.IPPermission) ([]networking.IPPermissionInfo, error) {
153+
permissionInfos := make([]networking.IPPermissionInfo, 0, len(permissions))
154+
for _, permission := range permissions {
155+
permissionInfo, err := buildIPPermissionInfo(permission)
156+
if err != nil {
157+
return nil, err
158+
}
159+
permissionInfos = append(permissionInfos, permissionInfo)
160+
}
161+
return permissionInfos, nil
162+
}
163+
164+
func buildIPPermissionInfo(permission ec2model.IPPermission) (networking.IPPermissionInfo, error) {
165+
protocol := permission.IPProtocol
166+
if len(permission.IPRanges) == 1 {
167+
labels := networking.NewIPPermissionLabelsForRawDescription(permission.IPRanges[0].Description)
168+
return networking.NewCIDRIPPermission(protocol, permission.FromPort, permission.ToPort, permission.IPRanges[0].CIDRIP, labels), nil
169+
}
170+
if len(permission.IPV6Range) == 1 {
171+
labels := networking.NewIPPermissionLabelsForRawDescription(permission.IPV6Range[0].Description)
172+
return networking.NewCIDRIPPermission(protocol, permission.FromPort, permission.ToPort, permission.IPV6Range[0].CIDRIPv6, labels), nil
173+
}
174+
if len(permission.UserIDGroupPairs) == 1 {
175+
labels := networking.NewIPPermissionLabelsForRawDescription(permission.UserIDGroupPairs[0].Description)
176+
return networking.NewGroupIDIPPermission(protocol, permission.FromPort, permission.ToPort, permission.UserIDGroupPairs[0].GroupID, labels), nil
177+
}
178+
return networking.IPPermissionInfo{}, errors.New("invalid ipPermission")
179+
}
180+
181+
// convert tags into AWS SDK tag presentation.
182+
func convertTagsToSDKTags(tags map[string]string) []*ec2sdk.Tag {
183+
if len(tags) == 0 {
184+
return nil
185+
}
186+
sdkTags := make([]*ec2sdk.Tag, 0, len(tags))
187+
188+
for _, key := range sets.StringKeySet(tags).List() {
189+
sdkTags = append(sdkTags, &ec2sdk.Tag{
190+
Key: awssdk.String(key),
191+
Value: awssdk.String(tags[key]),
192+
})
193+
}
194+
return sdkTags
195+
}

0 commit comments

Comments
 (0)