Skip to content

Commit 0521a6b

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #2160 from M00nF1sh/release-2.2.3
cherry-pick into release-2.2: fix the regression of IP mode support for fargate pods (#2158)
2 parents 84a207f + e5e1839 commit 0521a6b

File tree

5 files changed

+722
-27
lines changed

5 files changed

+722
-27
lines changed

‎pkg/k8s/node_utils.go

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,16 @@ package k8s
33
import (
44
"github.com/pkg/errors"
55
corev1 "k8s.io/api/core/v1"
6+
"regexp"
67
"strings"
78
)
89

910
const (
1011
toBeDeletedByCATaint = "ToBeDeletedByClusterAutoscaler"
1112
)
1213

14+
var awsInstanceIDRegex = regexp.MustCompile("^i-[^/]*$")
15+
1316
// IsNodeReady returns whether node is ready.
1417
func IsNodeReady(node *corev1.Node) bool {
1518
nodeReadyCond := GetNodeCondition(node, corev1.NodeReady)
@@ -47,6 +50,10 @@ func ExtractNodeInstanceID(node *corev1.Node) (string, error) {
4750
return "", errors.Errorf("providerID is not specified for node: %s", node.Name)
4851
}
4952

50-
p := strings.Split(providerID, "/")
51-
return p[len(p)-1], nil
53+
providerIDParts := strings.Split(providerID, "/")
54+
instanceID := providerIDParts[len(providerIDParts)-1]
55+
if !awsInstanceIDRegex.MatchString(instanceID) {
56+
return "", errors.Errorf("providerID %s is invalid for EC2 instances, node: %s", providerID, node.Name)
57+
}
58+
return instanceID, nil
5259
}

‎pkg/k8s/node_utils_test.go

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -212,7 +212,7 @@ func TestExtractNodeInstanceID(t *testing.T) {
212212
wantErr: errors.New("providerID is not specified for node: my-node-name"),
213213
},
214214
{
215-
name: "node with providerID",
215+
name: "node by EC2 instance",
216216
args: args{
217217
node: &corev1.Node{
218218
ObjectMeta: metav1.ObjectMeta{
@@ -225,6 +225,20 @@ func TestExtractNodeInstanceID(t *testing.T) {
225225
},
226226
want: "i-abcdefg0",
227227
},
228+
{
229+
name: "node by EKS Fargate",
230+
args: args{
231+
node: &corev1.Node{
232+
ObjectMeta: metav1.ObjectMeta{
233+
Name: "fargate-ip-192-168-138-30.us-west-2.compute.internal",
234+
},
235+
Spec: corev1.NodeSpec{
236+
ProviderID: "aws:///us-west-2b/368270442a-793d42d32c704bb793ca88a6a14ddd6e/fargate-ip-192-168-138-30.us-west-2.compute.internal",
237+
},
238+
},
239+
},
240+
wantErr: errors.New("providerID aws:///us-west-2b/368270442a-793d42d32c704bb793ca88a6a14ddd6e/fargate-ip-192-168-138-30.us-west-2.compute.internal is invalid for EC2 instances, node: fargate-ip-192-168-138-30.us-west-2.compute.internal"),
241+
},
228242
}
229243
for _, tt := range tests {
230244
t.Run(tt.name, func(t *testing.T) {

‎pkg/networking/pod_eni_info_resolver.go

Lines changed: 85 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"k8s.io/apimachinery/pkg/util/cache"
1212
"k8s.io/apimachinery/pkg/util/sets"
1313
"net"
14+
"sigs.k8s.io/aws-load-balancer-controller/pkg/algorithm"
1415
"sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services"
1516
"sigs.k8s.io/aws-load-balancer-controller/pkg/k8s"
1617
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -20,6 +21,10 @@ import (
2021

2122
const (
2223
defaultPodENIInfoCacheTTL = 10 * time.Minute
24+
// EC2:DescribeNetworkInterface supports up to 200 filters per call.
25+
describeNetworkInterfacesFiltersLimit = 200
26+
27+
labelEKSComputeType = "eks.amazonaws.com/compute-type"
2328
)
2429

2530
// PodENIInfoResolver is responsible for resolve the AWS VPC ENI that supports pod network.
@@ -29,15 +34,17 @@ type PodENIInfoResolver interface {
2934
}
3035

3136
// NewDefaultPodENIInfoResolver constructs new defaultPodENIInfoResolver.
32-
func NewDefaultPodENIInfoResolver(k8sClient client.Client, ec2Client services.EC2, nodeInfoProvider NodeInfoProvider, logger logr.Logger) *defaultPodENIInfoResolver {
37+
func NewDefaultPodENIInfoResolver(k8sClient client.Client, ec2Client services.EC2, nodeInfoProvider NodeInfoProvider, vpcID string, logger logr.Logger) *defaultPodENIInfoResolver {
3338
return &defaultPodENIInfoResolver{
34-
k8sClient: k8sClient,
35-
ec2Client: ec2Client,
36-
nodeInfoProvider: nodeInfoProvider,
37-
logger: logger,
38-
podENIInfoCache: cache.NewExpiring(),
39-
podENIInfoCacheMutex: sync.RWMutex{},
40-
podENIInfoCacheTTL: defaultPodENIInfoCacheTTL,
39+
k8sClient: k8sClient,
40+
ec2Client: ec2Client,
41+
nodeInfoProvider: nodeInfoProvider,
42+
vpcID: vpcID,
43+
logger: logger,
44+
podENIInfoCache: cache.NewExpiring(),
45+
podENIInfoCacheMutex: sync.RWMutex{},
46+
podENIInfoCacheTTL: defaultPodENIInfoCacheTTL,
47+
describeNetworkInterfacesIPChunkSize: describeNetworkInterfacesFiltersLimit - 1, // we used 1 filter for VPC.
4148
}
4249
}
4350

@@ -51,6 +58,8 @@ type defaultPodENIInfoResolver struct {
5158
ec2Client services.EC2
5259
// nodeInfoProvider
5360
nodeInfoProvider NodeInfoProvider
61+
// vpcID
62+
vpcID string
5463
// logger
5564
logger logr.Logger
5665

@@ -62,6 +71,9 @@ type defaultPodENIInfoResolver struct {
6271
// TTL for each cache entries.
6372
// Note: we assume pod's ENI information(e.g. securityGroups) haven't changed per podENICacheTTL.
6473
podENIInfoCacheTTL time.Duration
74+
75+
// chunkSize when describe network interface with IPAddress filter.
76+
describeNetworkInterfacesIPChunkSize int
6577
}
6678

6779
func (r *defaultPodENIInfoResolver) Resolve(ctx context.Context, pods []k8s.PodInfo) (map[types.NamespacedName]ENIInfo, error) {
@@ -129,7 +141,8 @@ func (r *defaultPodENIInfoResolver) saveENIInfosToCache(pods []k8s.PodInfo, eniI
129141
func (r *defaultPodENIInfoResolver) resolveViaCascadedLookup(ctx context.Context, pods []k8s.PodInfo) (map[types.NamespacedName]ENIInfo, error) {
130142
resolveFuncs := []func(ctx context.Context, pods []k8s.PodInfo) (map[types.NamespacedName]ENIInfo, error){
131143
r.resolveViaPodENIAnnotation,
132-
r.resolveViaVPCIPAddress,
144+
r.resolveViaNodeENIs,
145+
r.resolveViaVPCENIs,
133146
// TODO, add support for kubenet CNI plugin(kops) by resolve via routeTable.
134147
}
135148

@@ -151,7 +164,8 @@ func (r *defaultPodENIInfoResolver) resolveViaCascadedLookup(ctx context.Context
151164
return eniInfoByPodKey, nil
152165
}
153166

154-
// resolveViaPodENIAnnotation tries to resolve a pod ENI via the branch ENI annotation.
167+
// resolveViaPodENIAnnotation tries to resolve pod ENI by lookup pod's ENIInfo annotation.
168+
// with aws-vpc-cni CNI plugin's SecurityGroups for pods feature, podIP is supported by branchENI, whose information is exposed as pod annotation.
155169
func (r *defaultPodENIInfoResolver) resolveViaPodENIAnnotation(ctx context.Context, pods []k8s.PodInfo) (map[types.NamespacedName]ENIInfo, error) {
156170
podKeysByENIID := make(map[string][]types.NamespacedName)
157171
for _, pod := range pods {
@@ -191,8 +205,9 @@ func (r *defaultPodENIInfoResolver) resolveViaPodENIAnnotation(ctx context.Conte
191205
return eniInfoByPodKey, nil
192206
}
193207

194-
// resolveViaVPCIPAddress tries to resolve Pod ENI through the Pod IPAddress within VPC.
195-
func (r *defaultPodENIInfoResolver) resolveViaVPCIPAddress(ctx context.Context, pods []k8s.PodInfo) (map[types.NamespacedName]ENIInfo, error) {
208+
// resolveViaNodeENIs tries to resolve Pod ENI by matching podIP against ENIs on EC2 node's ENIs.
209+
// with aws-vpc-cni CNI plugin, podIP can be supported by either IPv4Addresses or IPv4Prefixes on ENI.
210+
func (r *defaultPodENIInfoResolver) resolveViaNodeENIs(ctx context.Context, pods []k8s.PodInfo) (map[types.NamespacedName]ENIInfo, error) {
196211
nodeKeysSet := make(map[types.NamespacedName]sets.Empty)
197212
for _, pod := range pods {
198213
nodeKey := types.NamespacedName{Name: pod.NodeName}
@@ -204,13 +219,20 @@ func (r *defaultPodENIInfoResolver) resolveViaVPCIPAddress(ctx context.Context,
204219
if err := r.k8sClient.Get(ctx, nodeKey, node); err != nil {
205220
return nil, err
206221
}
222+
// Fargate based nodes are not EC2 instances
223+
if node.Labels[labelEKSComputeType] == "fargate" {
224+
continue
225+
}
207226
nodes = append(nodes, node)
208227
}
228+
if len(nodes) == 0 {
229+
return nil, nil
230+
}
231+
209232
nodeInstanceByNodeKey, err := r.nodeInfoProvider.FetchNodeInstances(ctx, nodes)
210233
if err != nil {
211234
return nil, err
212235
}
213-
214236
eniInfoByPodKey := make(map[types.NamespacedName]ENIInfo, len(pods))
215237
for _, pod := range pods {
216238
nodeKey := types.NamespacedName{Name: pod.NodeName}
@@ -226,6 +248,56 @@ func (r *defaultPodENIInfoResolver) resolveViaVPCIPAddress(ctx context.Context,
226248
return eniInfoByPodKey, nil
227249
}
228250

251+
// resolveViaVPCENIs tries to resolve pod ENI by matching podIP against ENIs in vpc.
252+
// with EKS fargate pods, podIP is supported by an ENI in vpc.
253+
func (r *defaultPodENIInfoResolver) resolveViaVPCENIs(ctx context.Context, pods []k8s.PodInfo) (map[types.NamespacedName]ENIInfo, error) {
254+
podKeysByIP := make(map[string][]types.NamespacedName, len(pods))
255+
for _, pod := range pods {
256+
podKeysByIP[pod.PodIP] = append(podKeysByIP[pod.PodIP], pod.Key)
257+
}
258+
if len(podKeysByIP) == 0 {
259+
return nil, nil
260+
}
261+
262+
podIPs := sets.StringKeySet(podKeysByIP).List()
263+
podIPChunks := algorithm.ChunkStrings(podIPs, r.describeNetworkInterfacesIPChunkSize)
264+
eniByID := make(map[string]*ec2sdk.NetworkInterface)
265+
for _, podIPChunk := range podIPChunks {
266+
req := &ec2sdk.DescribeNetworkInterfacesInput{
267+
Filters: []*ec2sdk.Filter{
268+
{
269+
Name: awssdk.String("vpc-id"),
270+
Values: awssdk.StringSlice([]string{r.vpcID}),
271+
},
272+
{
273+
Name: awssdk.String("addresses.private-ip-address"),
274+
Values: awssdk.StringSlice(podIPChunk),
275+
},
276+
},
277+
}
278+
enis, err := r.ec2Client.DescribeNetworkInterfacesAsList(ctx, req)
279+
if err != nil {
280+
return nil, err
281+
}
282+
for _, eni := range enis {
283+
eniID := awssdk.StringValue(eni.NetworkInterfaceId)
284+
eniByID[eniID] = eni
285+
}
286+
}
287+
288+
eniInfoByPodKey := make(map[types.NamespacedName]ENIInfo)
289+
for _, eni := range eniByID {
290+
eniInfo := buildENIInfoViaENI(eni)
291+
for _, addr := range eni.PrivateIpAddresses {
292+
eniIP := awssdk.StringValue(addr.PrivateIpAddress)
293+
for _, podKey := range podKeysByIP[eniIP] {
294+
eniInfoByPodKey[podKey] = eniInfo
295+
}
296+
}
297+
}
298+
return eniInfoByPodKey, nil
299+
}
300+
229301
// isPodSupportedByNodeENI checks whether pod is supported by specific nodeENI.
230302
func (r *defaultPodENIInfoResolver) isPodSupportedByNodeENI(pod k8s.PodInfo, nodeENI *ec2sdk.InstanceNetworkInterface) bool {
231303
for _, ipv4Address := range nodeENI.PrivateIpAddresses {

0 commit comments

Comments
 (0)