Skip to content

Commit 83b9132

Browse files
shoekstraM00nF1sh
authored and
Timothy-Dougherty
committed
Allow TargetGroup endpoints outside the ELB VPC (kubernetes-sigs#1862)
* Allow TargetGroup endpoints outside the ELB VPC To support the scenario where the ELB is deployed in a separate VPC to the EKS cluster, we need to conditionally set the `AvailabilityZone` field to `all` when the target is outside of the ELB VPC. This commit addresses this by checking if the pod IP is found in the ELB VPC's CIDR or a secondary CIDR; to improve performance and reduce calls to the AWS API, VPC info is cached for a configurable amount of time, defaulting to 5 minutes. Signed-off-by: Stephen Hoekstra <[email protected]> * Make isELBV2TargetInELBVPC more efficient Don't need to check vpc.CidrBlock as it is part of vpc.CidrBlockAssociationSet. isIPinCIDR is now also a bit more robust in checking if an IP is part of a CIDR. Signed-off-by: Stephen Hoekstra <[email protected]> * Add vpcID to defaultResourceManager We're already passing this into NewDefaultResourceManager so no need to also add it to NetworkingManager. Signed-off-by: Stephen Hoekstra <[email protected]> * Change VPC cache duration to 10m Also switched cache value to be time.Duration from int. Signed-off-by: Stephen Hoekstra <[email protected]> * Whoops, we should return an error if found... * Use apimachinery/pkg/util/cache for caching Signed-off-by: Stephen Hoekstra <[email protected]> * remove the aws-vpc-cache-ttl flag fix unit tests Co-authored-by: M00nF1sh <[email protected]>
1 parent f3d5e6d commit 83b9132

File tree

11 files changed

+390
-14
lines changed

11 files changed

+390
-14
lines changed

‎main.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,10 +103,11 @@ func main() {
103103
sgManager := networking.NewDefaultSecurityGroupManager(cloud.EC2(), ctrl.Log)
104104
sgReconciler := networking.NewDefaultSecurityGroupReconciler(sgManager, ctrl.Log)
105105
azInfoProvider := networking.NewDefaultAZInfoProvider(cloud.EC2(), ctrl.Log.WithName("az-info-provider"))
106+
vpcInfoProvider := networking.NewDefaultVPCInfoProvider(cloud.EC2(), ctrl.Log.WithName("vpc-info-provider"))
106107
subnetResolver := networking.NewDefaultSubnetsResolver(azInfoProvider, cloud.EC2(), cloud.VpcID(), controllerCFG.ClusterName, ctrl.Log.WithName("subnets-resolver"))
107108
vpcResolver := networking.NewDefaultVPCResolver(cloud.EC2(), cloud.VpcID(), ctrl.Log.WithName("vpc-resolver"))
108109
tgbResManager := targetgroupbinding.NewDefaultResourceManager(mgr.GetClient(), cloud.ELBV2(), cloud.EC2(),
109-
podInfoRepo, sgManager, sgReconciler, cloud.VpcID(), controllerCFG.ClusterName, mgr.GetEventRecorderFor("targetGroupBinding"), ctrl.Log, controllerCFG.EnableEndpointSlices, controllerCFG.DisableRestrictedSGRules)
110+
podInfoRepo, sgManager, sgReconciler, cloud.VpcID(), controllerCFG.ClusterName, mgr.GetEventRecorderFor("targetGroupBinding"), ctrl.Log, controllerCFG.EnableEndpointSlices, controllerCFG.DisableRestrictedSGRules, vpcInfoProvider)
110111
backendSGProvider := networking.NewBackendSGProvider(controllerCFG.ClusterName, controllerCFG.BackendSecurityGroup,
111112
cloud.VpcID(), cloud.EC2(), mgr.GetClient(), ctrl.Log.WithName("backend-sg-provider"))
112113
ingGroupReconciler := ingress.NewGroupReconciler(cloud, mgr.GetClient(), mgr.GetEventRecorderFor("ingress"),

‎pkg/aws/cloud.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ type Cloud interface {
3838
// Region for the kubernetes cluster
3939
Region() string
4040

41-
// VPC ID for the the kubernetes cluster
41+
// VpcID for the LoadBalancer resources.
4242
VpcID() string
4343
}
4444

‎pkg/aws/cloud_config.go

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,19 @@
11
package aws
22

33
import (
4+
"time"
5+
46
"github.com/spf13/pflag"
57
"sigs.k8s.io/aws-load-balancer-controller/pkg/aws/throttle"
68
)
79

810
const (
911
flagAWSRegion = "aws-region"
12+
flagAWSAPIEndpoints = "aws-api-endpoints"
1013
flagAWSAPIThrottle = "aws-api-throttle"
1114
flagAWSVpcID = "aws-vpc-id"
15+
flagAWSVpcCacheTTL = "aws-vpc-cache-ttl"
1216
flagAWSMaxRetries = "aws-max-retries"
13-
flagAWSAPIEndpoints = "aws-api-endpoints"
1417
defaultVpcID = ""
1518
defaultRegion = ""
1619
defaultAPIMaxRetries = 10
@@ -23,9 +26,12 @@ type CloudConfig struct {
2326
// Throttle settings for AWS APIs
2427
ThrottleConfig *throttle.ServiceOperationsThrottleConfig
2528

26-
// VPC ID of the Kubernetes cluster
29+
// VpcID for the LoadBalancer resources.
2730
VpcID string
2831

32+
// VPC cache TTL in minutes
33+
VpcCacheTTL time.Duration
34+
2935
// Max retries configuration for AWS APIs
3036
MaxRetries int
3137

@@ -36,7 +42,7 @@ type CloudConfig struct {
3642
func (cfg *CloudConfig) BindFlags(fs *pflag.FlagSet) {
3743
fs.StringVar(&cfg.Region, flagAWSRegion, defaultRegion, "AWS Region for the kubernetes cluster")
3844
fs.Var(cfg.ThrottleConfig, flagAWSAPIThrottle, "throttle settings for AWS APIs, format: serviceID1:operationRegex1=rate:burst,serviceID2:operationRegex2=rate:burst")
39-
fs.StringVar(&cfg.VpcID, flagAWSVpcID, defaultVpcID, "AWS VPC ID for the Kubernetes cluster")
45+
fs.StringVar(&cfg.VpcID, flagAWSVpcID, defaultVpcID, "AWS VpcID for the LoadBalancer resources")
4046
fs.IntVar(&cfg.MaxRetries, flagAWSMaxRetries, defaultAPIMaxRetries, "Maximum retries for AWS APIs")
4147
fs.StringToStringVar(&cfg.AWSEndpoints, flagAWSAPIEndpoints, nil, "Custom AWS endpoint configuration, format: serviceID1=URL1,serviceID2=URL2")
4248
}

‎pkg/networking/vpc_info_provider.go

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
package networking
2+
3+
import (
4+
"context"
5+
"sync"
6+
"time"
7+
8+
awssdk "github.com/aws/aws-sdk-go/aws"
9+
ec2sdk "github.com/aws/aws-sdk-go/service/ec2"
10+
"github.com/go-logr/logr"
11+
"k8s.io/apimachinery/pkg/util/cache"
12+
"sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services"
13+
)
14+
15+
const defaultVPCInfoCacheTTL = 10 * time.Minute
16+
17+
// VPCInfoProvider is responsible for providing VPC info.
18+
type VPCInfoProvider interface {
19+
FetchVPCInfo(ctx context.Context, vpcID string) (*ec2sdk.Vpc, error)
20+
}
21+
22+
// NewDefaultVPCInfoProvider constructs new defaultVPCInfoProvider.
23+
func NewDefaultVPCInfoProvider(ec2Client services.EC2, logger logr.Logger) *defaultVPCInfoProvider {
24+
return &defaultVPCInfoProvider{
25+
ec2Client: ec2Client,
26+
vpcInfoCache: cache.NewExpiring(),
27+
vpcInfoCacheMutex: sync.RWMutex{},
28+
vpcInfoCacheTTL: defaultVPCInfoCacheTTL,
29+
logger: logger,
30+
}
31+
}
32+
33+
var _ VPCInfoProvider = &defaultVPCInfoProvider{}
34+
35+
// default implementation for VPCInfoProvider.
36+
type defaultVPCInfoProvider struct {
37+
ec2Client services.EC2
38+
vpcInfoCache *cache.Expiring
39+
vpcInfoCacheMutex sync.RWMutex
40+
vpcInfoCacheTTL time.Duration
41+
42+
logger logr.Logger
43+
}
44+
45+
func (p *defaultVPCInfoProvider) FetchVPCInfo(ctx context.Context, vpcID string) (*ec2sdk.Vpc, error) {
46+
if vpcInfo := p.fetchVPCInfoFromCache(); vpcInfo != nil {
47+
return vpcInfo, nil
48+
}
49+
50+
// Fetch VPC info from the AWS API and cache response before returning.
51+
vpcInfo, err := p.fetchVPCInfoFromAWS(ctx, vpcID)
52+
if err != nil {
53+
return nil, err
54+
}
55+
p.saveVPCInfoToCache(vpcInfo)
56+
57+
return vpcInfo, nil
58+
}
59+
60+
func (p *defaultVPCInfoProvider) fetchVPCInfoFromCache() *ec2sdk.Vpc {
61+
p.vpcInfoCacheMutex.RLock()
62+
defer p.vpcInfoCacheMutex.RUnlock()
63+
64+
if rawCacheItem, exists := p.vpcInfoCache.Get("vpcInfo"); exists {
65+
return rawCacheItem.(*ec2sdk.Vpc)
66+
}
67+
68+
return nil
69+
}
70+
71+
func (p *defaultVPCInfoProvider) saveVPCInfoToCache(vpcInfo *ec2sdk.Vpc) {
72+
p.vpcInfoCacheMutex.Lock()
73+
defer p.vpcInfoCacheMutex.Unlock()
74+
75+
p.vpcInfoCache.Set("vpcInfo", vpcInfo, p.vpcInfoCacheTTL)
76+
}
77+
78+
// fetchVPCInfoFromAWS will fetch VPC info from the AWS API.
79+
func (p *defaultVPCInfoProvider) fetchVPCInfoFromAWS(ctx context.Context, vpcID string) (*ec2sdk.Vpc, error) {
80+
req := &ec2sdk.DescribeVpcsInput{
81+
VpcIds: []*string{awssdk.String(vpcID)},
82+
}
83+
resp, err := p.ec2Client.DescribeVpcsWithContext(ctx, req)
84+
if err != nil {
85+
return nil, err
86+
}
87+
88+
return resp.Vpcs[0], nil
89+
}

‎pkg/networking/vpc_info_provider_mocks.go

Lines changed: 50 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/networking/vpc_info_provider_test.go

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
package networking
2+
3+
import (
4+
"context"
5+
awssdk "github.com/aws/aws-sdk-go/aws"
6+
ec2sdk "github.com/aws/aws-sdk-go/service/ec2"
7+
gomock "github.com/golang/mock/gomock"
8+
"reflect"
9+
"sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services"
10+
"sigs.k8s.io/controller-runtime/pkg/log"
11+
"testing"
12+
)
13+
14+
func Test_defaultVPCInfoProvider_FetchVPCInfo(t *testing.T) {
15+
type describeVpcCall struct {
16+
input *ec2sdk.DescribeVpcsInput
17+
output *ec2sdk.DescribeVpcsOutput
18+
err error
19+
}
20+
21+
type fields struct {
22+
describeVpcCalls []describeVpcCall
23+
}
24+
tests := []struct {
25+
name string
26+
fields fields
27+
want *ec2sdk.Vpc
28+
wantErr bool
29+
}{
30+
{
31+
name: "from AWS",
32+
fields: fields{
33+
describeVpcCalls: []describeVpcCall{
34+
{
35+
input: &ec2sdk.DescribeVpcsInput{
36+
VpcIds: []*string{awssdk.String("vpc-2f09a348")},
37+
},
38+
output: &ec2sdk.DescribeVpcsOutput{
39+
Vpcs: []*ec2sdk.Vpc{{VpcId: awssdk.String("vpc-2f09a348")}},
40+
},
41+
err: nil,
42+
},
43+
},
44+
},
45+
want: &ec2sdk.Vpc{
46+
VpcId: awssdk.String("vpc-2f09a348"),
47+
},
48+
wantErr: false,
49+
},
50+
{
51+
name: "from cache",
52+
fields: fields{},
53+
want: &ec2sdk.Vpc{
54+
VpcId: awssdk.String("vpc-2f09a348"),
55+
},
56+
wantErr: false,
57+
},
58+
}
59+
60+
ctrl := gomock.NewController(t)
61+
defer ctrl.Finish()
62+
63+
ec2Client := services.NewMockEC2(ctrl)
64+
p := NewDefaultVPCInfoProvider(ec2Client, &log.NullLogger{})
65+
66+
for _, tt := range tests {
67+
t.Run(tt.name, func(t *testing.T) {
68+
for _, call := range tt.fields.describeVpcCalls {
69+
ec2Client.EXPECT().DescribeVpcsWithContext(gomock.Any(), call.input).Return(call.output, call.err)
70+
}
71+
72+
got, err := p.FetchVPCInfo(context.Background(), "vpc-2f09a348")
73+
if (err != nil) != tt.wantErr {
74+
t.Errorf("defaultVPCInfoProvider.FetchVPCInfo() error = %v, wantErr %v", err, tt.wantErr)
75+
return
76+
}
77+
if !reflect.DeepEqual(got, tt.want) {
78+
t.Errorf("defaultVPCInfoProvider.FetchVPCInfo() = %v, want %v", got, tt.want)
79+
}
80+
})
81+
}
82+
}

‎pkg/targetgroupbinding/networking_manager.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,10 @@ package targetgroupbinding
33
import (
44
"context"
55
"fmt"
6+
"net"
7+
"strings"
8+
"sync"
9+
610
awssdk "github.com/aws/aws-sdk-go/aws"
711
"github.com/aws/aws-sdk-go/aws/awserr"
812
ec2sdk "github.com/aws/aws-sdk-go/service/ec2"
@@ -13,14 +17,11 @@ import (
1317
"k8s.io/apimachinery/pkg/types"
1418
"k8s.io/apimachinery/pkg/util/intstr"
1519
"k8s.io/apimachinery/pkg/util/sets"
16-
"net"
1720
elbv2api "sigs.k8s.io/aws-load-balancer-controller/apis/elbv2/v1beta1"
1821
"sigs.k8s.io/aws-load-balancer-controller/pkg/backend"
1922
"sigs.k8s.io/aws-load-balancer-controller/pkg/k8s"
2023
"sigs.k8s.io/aws-load-balancer-controller/pkg/networking"
2124
"sigs.k8s.io/controller-runtime/pkg/client"
22-
"strings"
23-
"sync"
2425
)
2526

2627
const (

0 commit comments

Comments
 (0)