Skip to content

Commit a08ca0d

Browse files
M00nF1shM00nF1sh
authored
add COIP support for ALB on outpost (#1685)
* add COIP support for ALB on outpost * check for drifted coIPv4Pool settings
1 parent 6b3bfb5 commit a08ca0d

File tree

7 files changed

+408
-0
lines changed

7 files changed

+408
-0
lines changed

‎docs/guide/ingress/annotations.md

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ You can add annotations to kubernetes Ingress and Service objects to customize t
2323
|[alb.ingress.kubernetes.io/scheme](#scheme)|internal \| internet-facing|internal|Ingress|Exclusive|
2424
|[alb.ingress.kubernetes.io/subnets](#subnets)|stringList|N/A|Ingress|Exclusive|
2525
|[alb.ingress.kubernetes.io/security-groups](#security-groups)|stringList|N/A|Ingress|Exclusive|
26+
|[alb.ingress.kubernetes.io/customer-owned-ipv4-pool](#customer-owned-ipv4-pool)|string|N/A|Ingress|Exclusive|
2627
|[alb.ingress.kubernetes.io/load-balancer-attributes](#load-balancer-attributes)|stringMap|N/A|Ingress|Merge|
2728
|[alb.ingress.kubernetes.io/wafv2-acl-arn](#wafv2-acl-arn)|string|N/A|Ingress|Exclusive|
2829
|[alb.ingress.kubernetes.io/waf-acl-id](#waf-acl-id)|string|N/A|Ingress|Exclusive|
@@ -124,6 +125,16 @@ Traffic Listening can be controlled with following annotations:
124125
alb.ingress.kubernetes.io/ip-address-type: ipv4
125126
```
126127

128+
- <a name="customer-owned-ipv4-pool">`alb.ingress.kubernetes.io/customer-owned-ipv4-pool`</a> specifies the customer-owned IPv4 address pool for ALB on Outpost.
129+
130+
!!!warning ""
131+
This annotation should be treated as immutable. To remove or change coIPv4Pool, you need to recreate Ingress.
132+
133+
!!!example
134+
```
135+
alb.ingress.kubernetes.io/customer-owned-ipv4-pool: ipv4pool-coip-xxxxxxxx
136+
```
137+
127138
## Traffic Routing
128139
Traffic Routing can be controlled with following annotations:
129140

‎pkg/annotations/constants.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ const (
1111
IngressSuffixIPAddressType = "ip-address-type"
1212
IngressSuffixScheme = "scheme"
1313
IngressSuffixSubnets = "subnets"
14+
IngressSuffixCustomerOwnedIPv4Pool = "customer-owned-ipv4-pool"
1415
IngressSuffixLoadBalancerAttributes = "load-balancer-attributes"
1516
IngressSuffixWAFv2ACLARN = "wafv2-acl-arn"
1617
IngressSuffixWAFACLID = "waf-acl-id"

‎pkg/deploy/elbv2/load_balancer_manager.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
awssdk "github.com/aws/aws-sdk-go/aws"
77
elbv2sdk "github.com/aws/aws-sdk-go/service/elbv2"
88
"github.com/go-logr/logr"
9+
"github.com/pkg/errors"
910
"k8s.io/apimachinery/pkg/util/sets"
1011
"sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services"
1112
"sigs.k8s.io/aws-load-balancer-controller/pkg/deploy/tracking"
@@ -92,6 +93,9 @@ func (m *defaultLoadBalancerManager) Update(ctx context.Context, resLB *elbv2mod
9293
if err := m.attributesReconciler.Reconcile(ctx, resLB, sdkLB); err != nil {
9394
return elbv2model.LoadBalancerStatus{}, err
9495
}
96+
if err := m.checkSDKLoadBalancerWithCOIPv4Pool(ctx, resLB, sdkLB); err != nil {
97+
return elbv2model.LoadBalancerStatus{}, err
98+
}
9599
return buildResLoadBalancerStatus(sdkLB), nil
96100
}
97101

@@ -206,6 +210,13 @@ func (m *defaultLoadBalancerManager) updateSDKLoadBalancerWithSecurityGroups(ctx
206210
return nil
207211
}
208212

213+
func (m *defaultLoadBalancerManager) checkSDKLoadBalancerWithCOIPv4Pool(_ context.Context, resLB *elbv2model.LoadBalancer, sdkLB LoadBalancerWithTags) error {
214+
if awssdk.StringValue(resLB.Spec.CustomerOwnedIPv4Pool) != awssdk.StringValue(sdkLB.LoadBalancer.CustomerOwnedIpv4Pool) {
215+
return errors.New("loadBalancer has drifted CustomerOwnedIPv4Pool setting")
216+
}
217+
return nil
218+
}
219+
209220
func (m *defaultLoadBalancerManager) updateSDKLoadBalancerWithTags(ctx context.Context, resLB *elbv2model.LoadBalancer, sdkLB LoadBalancerWithTags) error {
210221
desiredLBTags := m.trackingProvider.ResourceTags(resLB.Stack(), resLB, resLB.Spec.Tags)
211222
return m.taggingManager.ReconcileTags(ctx, awssdk.StringValue(sdkLB.LoadBalancer.LoadBalancerArn), desiredLBTags,
@@ -236,6 +247,8 @@ func buildSDKCreateLoadBalancerInput(lbSpec elbv2model.LoadBalancerSpec) (*elbv2
236247
} else {
237248
sdkObj.SecurityGroups = sdkSecurityGroups
238249
}
250+
251+
sdkObj.CustomerOwnedIpv4Pool = lbSpec.CustomerOwnedIPv4Pool
239252
return sdkObj, nil
240253
}
241254

‎pkg/deploy/elbv2/load_balancer_manager_test.go

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package elbv2
22

33
import (
4+
"context"
5+
"errors"
46
awssdk "github.com/aws/aws-sdk-go/aws"
57
elbv2sdk "github.com/aws/aws-sdk-go/service/elbv2"
68
"github.com/stretchr/testify/assert"
@@ -92,6 +94,46 @@ func Test_buildSDKCreateLoadBalancerInput(t *testing.T) {
9294
},
9395
},
9496
},
97+
{
98+
name: "application loadBalancer - with CoIP pool",
99+
args: args{
100+
lbSpec: elbv2model.LoadBalancerSpec{
101+
Name: "my-alb",
102+
Type: elbv2model.LoadBalancerTypeApplication,
103+
Scheme: &schemeInternetFacing,
104+
IPAddressType: &addressTypeDualStack,
105+
SubnetMappings: []elbv2model.SubnetMapping{
106+
{
107+
SubnetID: "subnet-A",
108+
},
109+
{
110+
SubnetID: "subnet-B",
111+
},
112+
},
113+
SecurityGroups: []coremodel.StringToken{
114+
coremodel.LiteralStringToken("sg-A"),
115+
coremodel.LiteralStringToken("sg-B"),
116+
},
117+
CustomerOwnedIPv4Pool: awssdk.String("coIP-pool-x"),
118+
},
119+
},
120+
want: &elbv2sdk.CreateLoadBalancerInput{
121+
Name: awssdk.String("my-alb"),
122+
Type: awssdk.String("application"),
123+
IpAddressType: awssdk.String("dualstack"),
124+
Scheme: awssdk.String("internet-facing"),
125+
SubnetMappings: []*elbv2sdk.SubnetMapping{
126+
{
127+
SubnetId: awssdk.String("subnet-A"),
128+
},
129+
{
130+
SubnetId: awssdk.String("subnet-B"),
131+
},
132+
},
133+
SecurityGroups: awssdk.StringSlice([]string{"sg-A", "sg-B"}),
134+
CustomerOwnedIpv4Pool: awssdk.String("coIP-pool-x"),
135+
},
136+
},
95137
}
96138
for _, tt := range tests {
97139
t.Run(tt.name, func(t *testing.T) {
@@ -264,3 +306,107 @@ func Test_buildResLoadBalancerStatus(t *testing.T) {
264306
})
265307
}
266308
}
309+
310+
func Test_defaultLoadBalancerManager_checkSDKLoadBalancerWithCOIPv4Pool(t *testing.T) {
311+
type args struct {
312+
resLB *elbv2model.LoadBalancer
313+
sdkLB LoadBalancerWithTags
314+
}
315+
tests := []struct {
316+
name string
317+
args args
318+
wantErr error
319+
}{
320+
{
321+
name: "both resLB and sdkLB don't have CustomerOwnedIPv4Pool setting",
322+
args: args{
323+
resLB: &elbv2model.LoadBalancer{
324+
Spec: elbv2model.LoadBalancerSpec{
325+
CustomerOwnedIPv4Pool: nil,
326+
},
327+
},
328+
sdkLB: LoadBalancerWithTags{
329+
LoadBalancer: &elbv2sdk.LoadBalancer{
330+
CustomerOwnedIpv4Pool: nil,
331+
},
332+
},
333+
},
334+
wantErr: nil,
335+
},
336+
{
337+
name: "both resLB and sdkLB have same CustomerOwnedIPv4Pool setting",
338+
args: args{
339+
resLB: &elbv2model.LoadBalancer{
340+
Spec: elbv2model.LoadBalancerSpec{
341+
CustomerOwnedIPv4Pool: awssdk.String("ipv4pool-coip-abc"),
342+
},
343+
},
344+
sdkLB: LoadBalancerWithTags{
345+
LoadBalancer: &elbv2sdk.LoadBalancer{
346+
CustomerOwnedIpv4Pool: awssdk.String("ipv4pool-coip-abc"),
347+
},
348+
},
349+
},
350+
wantErr: nil,
351+
},
352+
{
353+
name: "both resLB and sdkLB have different CustomerOwnedIPv4Pool setting",
354+
args: args{
355+
resLB: &elbv2model.LoadBalancer{
356+
Spec: elbv2model.LoadBalancerSpec{
357+
CustomerOwnedIPv4Pool: awssdk.String("ipv4pool-coip-abc"),
358+
},
359+
},
360+
sdkLB: LoadBalancerWithTags{
361+
LoadBalancer: &elbv2sdk.LoadBalancer{
362+
CustomerOwnedIpv4Pool: awssdk.String("ipv4pool-coip-def"),
363+
},
364+
},
365+
},
366+
wantErr: errors.New("loadBalancer has drifted CustomerOwnedIPv4Pool setting"),
367+
},
368+
{
369+
name: "only resLB have CustomerOwnedIPv4Pool setting",
370+
args: args{
371+
resLB: &elbv2model.LoadBalancer{
372+
Spec: elbv2model.LoadBalancerSpec{
373+
CustomerOwnedIPv4Pool: awssdk.String("ipv4pool-coip-abc"),
374+
},
375+
},
376+
sdkLB: LoadBalancerWithTags{
377+
LoadBalancer: &elbv2sdk.LoadBalancer{
378+
CustomerOwnedIpv4Pool: nil,
379+
},
380+
},
381+
},
382+
wantErr: errors.New("loadBalancer has drifted CustomerOwnedIPv4Pool setting"),
383+
},
384+
{
385+
name: "only sdkLB have CustomerOwnedIPv4Pool setting",
386+
args: args{
387+
resLB: &elbv2model.LoadBalancer{
388+
Spec: elbv2model.LoadBalancerSpec{
389+
CustomerOwnedIPv4Pool: nil,
390+
},
391+
},
392+
sdkLB: LoadBalancerWithTags{
393+
LoadBalancer: &elbv2sdk.LoadBalancer{
394+
CustomerOwnedIpv4Pool: awssdk.String("ipv4pool-coip-abc"),
395+
},
396+
},
397+
},
398+
wantErr: errors.New("loadBalancer has drifted CustomerOwnedIPv4Pool setting"),
399+
},
400+
}
401+
for _, tt := range tests {
402+
t.Run(tt.name, func(t *testing.T) {
403+
m := &defaultLoadBalancerManager{}
404+
err := m.checkSDKLoadBalancerWithCOIPv4Pool(context.Background(), tt.args.resLB, tt.args.sdkLB)
405+
if tt.wantErr != nil {
406+
assert.EqualError(t, err, tt.wantErr.Error())
407+
} else {
408+
assert.NoError(t, err)
409+
}
410+
})
411+
}
412+
}

‎pkg/ingress/model_build_load_balancer.go

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"regexp"
1414
"sigs.k8s.io/aws-load-balancer-controller/pkg/annotations"
1515
"sigs.k8s.io/aws-load-balancer-controller/pkg/equality"
16+
"sigs.k8s.io/aws-load-balancer-controller/pkg/k8s"
1617
"sigs.k8s.io/aws-load-balancer-controller/pkg/model/core"
1718
elbv2model "sigs.k8s.io/aws-load-balancer-controller/pkg/model/elbv2"
1819
"sigs.k8s.io/aws-load-balancer-controller/pkg/networking"
@@ -50,6 +51,10 @@ func (t *defaultModelBuildTask) buildLoadBalancerSpec(ctx context.Context, liste
5051
if err != nil {
5152
return elbv2model.LoadBalancerSpec{}, err
5253
}
54+
coIPv4Pool, err := t.buildLoadBalancerCOIPv4Pool(ctx)
55+
if err != nil {
56+
return elbv2model.LoadBalancerSpec{}, err
57+
}
5358
loadBalancerAttributes, err := t.buildLoadBalancerAttributes(ctx)
5459
if err != nil {
5560
return elbv2model.LoadBalancerSpec{}, err
@@ -66,6 +71,7 @@ func (t *defaultModelBuildTask) buildLoadBalancerSpec(ctx context.Context, liste
6671
IPAddressType: &ipAddressType,
6772
SubnetMappings: subnetMappings,
6873
SecurityGroups: securityGroups,
74+
CustomerOwnedIPv4Pool: coIPv4Pool,
6975
LoadBalancerAttributes: loadBalancerAttributes,
7076
Tags: tags,
7177
}, nil
@@ -216,6 +222,31 @@ func (t *defaultModelBuildTask) buildLoadBalancerSecurityGroups(ctx context.Cont
216222
return sgIDTokens, nil
217223
}
218224

225+
func (t *defaultModelBuildTask) buildLoadBalancerCOIPv4Pool(_ context.Context) (*string, error) {
226+
explicitCOIPv4Pools := sets.NewString()
227+
for _, ing := range t.ingGroup.Members {
228+
rawCOIPv4Pool := ""
229+
if exists := t.annotationParser.ParseStringAnnotation(annotations.IngressSuffixCustomerOwnedIPv4Pool, &rawCOIPv4Pool, ing.Annotations); !exists {
230+
continue
231+
}
232+
if len(rawCOIPv4Pool) == 0 {
233+
return nil, errors.Errorf("cannot use empty value for %s annotation, ingress: %v",
234+
annotations.IngressSuffixCustomerOwnedIPv4Pool, k8s.NamespacedName(ing))
235+
}
236+
explicitCOIPv4Pools.Insert(rawCOIPv4Pool)
237+
}
238+
239+
if len(explicitCOIPv4Pools) == 0 {
240+
return nil, nil
241+
}
242+
if len(explicitCOIPv4Pools) > 1 {
243+
return nil, errors.Errorf("conflicting CustomerOwnedIPv4Pool: %v", explicitCOIPv4Pools.List())
244+
}
245+
246+
rawCOIPv4Pool, _ := explicitCOIPv4Pools.PopAny()
247+
return &rawCOIPv4Pool, nil
248+
}
249+
219250
func (t *defaultModelBuildTask) buildLoadBalancerAttributes(_ context.Context) ([]elbv2model.LoadBalancerAttribute, error) {
220251
mergedAttributes := make(map[string]string)
221252
for _, ing := range t.ingGroup.Members {

0 commit comments

Comments
 (0)