Skip to content

Commit 1f406d7

Browse files
kishorjTimothy-Dougherty
kishorj
authored and
Timothy-Dougherty
committed
additional e2e tests for service (kubernetes-sigs#1898)
1 parent 3dedc09 commit 1f406d7

File tree

5 files changed

+228
-4
lines changed

5 files changed

+228
-4
lines changed

‎test/e2e/service/aws_resource_verifier.go

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"github.com/pkg/errors"
99
"sigs.k8s.io/aws-load-balancer-controller/test/framework"
1010
"sigs.k8s.io/aws-load-balancer-controller/test/framework/utils"
11+
"sort"
1112
"strconv"
1213
)
1314

@@ -107,6 +108,34 @@ func verifyLoadBalancerListeners(ctx context.Context, f *framework.Framework, lb
107108
return nil
108109
}
109110

111+
func verifyLoadBalancerListenerCertificates(ctx context.Context, f *framework.Framework, lbARN string, expectedCertARNS []string) error {
112+
listeners, err := f.LBManager.GetLoadBalancerListeners(ctx, lbARN)
113+
Expect(err).ToNot(HaveOccurred())
114+
Expect(len(listeners)).Should(BeNumerically(">", 0))
115+
listenerCerts, err := f.LBManager.GetLoadBalancerListenerCertificates(ctx, awssdk.StringValue(listeners[0].ListenerArn))
116+
Expect(err).ToNot(HaveOccurred())
117+
118+
var observedCertArns []string
119+
var defaultCert string
120+
for _, cert := range listenerCerts {
121+
if awssdk.BoolValue(cert.IsDefault) {
122+
defaultCert = awssdk.StringValue(cert.CertificateArn)
123+
}
124+
observedCertArns = append(observedCertArns, awssdk.StringValue(cert.CertificateArn))
125+
}
126+
if defaultCert != expectedCertARNS[0] {
127+
return errors.New("default cert does not match")
128+
}
129+
//Expect(defaultCert).To(Equal(expectedCertARNS[0]))
130+
if len(expectedCertARNS) != len(observedCertArns) {
131+
return errors.New("cert len mismatch")
132+
}
133+
sort.Strings(observedCertArns)
134+
sort.Strings(expectedCertARNS)
135+
Expect(expectedCertARNS).To(Equal(observedCertArns))
136+
return nil
137+
}
138+
110139
func verifyLoadBalancerTargetGroups(ctx context.Context, f *framework.Framework, lbARN string, expected LoadBalancerExpectation) error {
111140
targetGroups, err := f.TGManager.GetTargetGroupsForLoadBalancer(ctx, lbARN)
112141
Expect(err).ToNot(HaveOccurred())

‎test/e2e/service/nlb_instance_target.go

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,9 @@ import (
66
corev1 "k8s.io/api/core/v1"
77
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
88
"k8s.io/apimachinery/pkg/util/intstr"
9+
"sigs.k8s.io/aws-load-balancer-controller/pkg/k8s"
910
"sigs.k8s.io/aws-load-balancer-controller/test/framework"
11+
"sigs.k8s.io/controller-runtime/pkg/client"
1012
)
1113

1214
const (
@@ -28,10 +30,14 @@ func (s *NLBInstanceTestStack) Deploy(ctx context.Context, f *framework.Framewor
2830
return s.resourceStack.Deploy(ctx, f)
2931
}
3032

31-
func (s *NLBInstanceTestStack) UpdateServiceAnnotation(ctx context.Context, f *framework.Framework, svcAnnotations map[string]string) error {
33+
func (s *NLBInstanceTestStack) UpdateServiceAnnotations(ctx context.Context, f *framework.Framework, svcAnnotations map[string]string) error {
3234
return s.resourceStack.UpdateServiceAnnotations(ctx, f, svcAnnotations)
3335
}
3436

37+
func (s *NLBInstanceTestStack) DeleteServiceAnnotations(ctx context.Context, f *framework.Framework, annotationKeys []string) error {
38+
return s.resourceStack.DeleteServiceAnnotations(ctx, f, annotationKeys)
39+
}
40+
3541
func (s *NLBInstanceTestStack) UpdateServiceTrafficPolicy(ctx context.Context, f *framework.Framework, trafficPolicy corev1.ServiceExternalTrafficPolicyType) error {
3642
return s.resourceStack.UpdateServiceTrafficPolicy(ctx, f, trafficPolicy)
3743
}
@@ -48,6 +54,28 @@ func (s *NLBInstanceTestStack) GetLoadBalancerIngressHostName() string {
4854
return s.resourceStack.GetLoadBalancerIngressHostname()
4955
}
5056

57+
func (s *NLBInstanceTestStack) GetWorkerNodes(ctx context.Context, f *framework.Framework) ([]corev1.Node, error) {
58+
nodeList := &corev1.NodeList{}
59+
err := f.K8sClient.List(ctx, nodeList)
60+
if err != nil {
61+
return nil, err
62+
}
63+
return nodeList.Items, nil
64+
}
65+
66+
func (s *NLBInstanceTestStack) ApplyNodeLabels(ctx context.Context, f *framework.Framework, node *corev1.Node, labels map[string]string) error {
67+
f.Logger.Info("applying node labels", "node", k8s.NamespacedName(node))
68+
oldNode := node.DeepCopy()
69+
for key, value := range labels {
70+
node.Labels[key] = value
71+
}
72+
if err := f.K8sClient.Patch(ctx, node, client.MergeFrom(oldNode)); err != nil {
73+
f.Logger.Info("failed to update node", "node", k8s.NamespacedName(node))
74+
return err
75+
}
76+
return nil
77+
}
78+
5179
func (s *NLBInstanceTestStack) buildDeploymentSpec() *appsv1.Deployment {
5280
numReplicas := int32(defaultNumReplicas)
5381
labels := map[string]string{

‎test/e2e/service/nlb_instance_target_test.go

Lines changed: 144 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,13 @@ package service
33
import (
44
"context"
55
"fmt"
6+
awssdk "github.com/aws/aws-sdk-go/aws"
67
. "github.com/onsi/ginkgo"
78
. "github.com/onsi/gomega"
89
corev1 "k8s.io/api/core/v1"
910
"sigs.k8s.io/aws-load-balancer-controller/test/framework/http"
1011
"sigs.k8s.io/aws-load-balancer-controller/test/framework/utils"
12+
"strings"
1113
)
1214

1315
var _ = Describe("test k8s service reconciled by the aws load balancer controller", func() {
@@ -84,7 +86,7 @@ var _ = Describe("test k8s service reconciled by the aws load balancer controlle
8486
})
8587

8688
By("enabling cross zone load balancing", func() {
87-
err := stack.UpdateServiceAnnotation(ctx, tf, map[string]string{
89+
err := stack.UpdateServiceAnnotations(ctx, tf, map[string]string{
8890
"service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled": "true",
8991
})
9092
Expect(err).NotTo(HaveOccurred())
@@ -97,7 +99,7 @@ var _ = Describe("test k8s service reconciled by the aws load balancer controlle
9799
})
98100

99101
By("specifying load balancer tags", func() {
100-
err := stack.UpdateServiceAnnotation(ctx, tf, map[string]string{
102+
err := stack.UpdateServiceAnnotations(ctx, tf, map[string]string{
101103
"service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags": "instance-mode=true, key1=value1",
102104
})
103105
Expect(err).NotTo(HaveOccurred())
@@ -174,7 +176,7 @@ var _ = Describe("test k8s service reconciled by the aws load balancer controlle
174176
Expect(err).NotTo(HaveOccurred())
175177
})
176178
By("specifying target group attributes annotation", func() {
177-
err := stack.UpdateServiceAnnotation(ctx, tf, map[string]string{
179+
err := stack.UpdateServiceAnnotations(ctx, tf, map[string]string{
178180
"service.beta.kubernetes.io/aws-load-balancer-target-group-attributes": "preserve_client_ip.enabled=false, proxy_protocol_v2.enabled=true, deregistration_delay.timeout_seconds=120",
179181
})
180182
Expect(err).NotTo(HaveOccurred())
@@ -188,5 +190,144 @@ var _ = Describe("test k8s service reconciled by the aws load balancer controlle
188190
}, utils.PollTimeoutShort, utils.PollIntervalMedium).Should(BeTrue())
189191
})
190192
})
193+
It("should create TLS listeners", func() {
194+
if len(tf.Options.CertificateARNs) == 0 {
195+
Skip("Skipping tests, certificates not specified")
196+
}
197+
By("deploying stack", func() {
198+
err := stack.Deploy(ctx, tf, map[string]string{
199+
"service.beta.kubernetes.io/aws-load-balancer-ssl-cert": tf.Options.CertificateARNs,
200+
})
201+
Expect(err).NotTo(HaveOccurred())
202+
})
203+
By("checking service status for lb dns name", func() {
204+
dnsName = stack.GetLoadBalancerIngressHostName()
205+
Expect(dnsName).ToNot(BeEmpty())
206+
})
207+
208+
By("querying AWS loadbalancer from the dns name", func() {
209+
var err error
210+
lbARN, err = tf.LBManager.FindLoadBalancerByDNSName(ctx, dnsName)
211+
Expect(err).NotTo(HaveOccurred())
212+
Expect(lbARN).ToNot(BeEmpty())
213+
})
214+
By("verifying AWS loadbalancer resources", func() {
215+
err := verifyAWSLoadBalancerResources(ctx, tf, lbARN, LoadBalancerExpectation{
216+
Type: "network",
217+
Scheme: "internet-facing",
218+
TargetType: "instance",
219+
Listeners: map[string]string{
220+
"80": "TLS",
221+
},
222+
TargetGroups: stack.resourceStack.getTargetGroupNodePortMap(),
223+
NumTargets: 0,
224+
TargetGroupHC: &TargetGroupHC{
225+
Protocol: "TCP",
226+
Port: "traffic-port",
227+
Interval: 10,
228+
Timeout: 10,
229+
HealthyThreshold: 3,
230+
UnhealthyThreshold: 3,
231+
},
232+
})
233+
Expect(err).NotTo(HaveOccurred())
234+
})
235+
By("verifying listener certificates", func() {
236+
expectedARNs := strings.Split(tf.Options.CertificateARNs, ",")
237+
Eventually(func() bool {
238+
return verifyLoadBalancerListenerCertificates(ctx, tf, lbARN, expectedARNs) == nil
239+
}, utils.PollTimeoutShort, utils.PollIntervalMedium).Should(BeTrue())
240+
})
241+
By("removing first certificate from annotation and updating the service", func() {
242+
certs := strings.Split(tf.Options.CertificateARNs, ",")[1:]
243+
if len(certs) == 0 {
244+
return
245+
}
246+
err := stack.UpdateServiceAnnotations(ctx, tf, map[string]string{
247+
"service.beta.kubernetes.io/aws-load-balancer-ssl-cert": strings.Join(certs, ","),
248+
})
249+
Expect(err).NotTo(HaveOccurred())
250+
Eventually(func() bool {
251+
return verifyLoadBalancerListenerCertificates(ctx, tf, lbARN, certs) == nil
252+
}, utils.PollTimeoutShort, utils.PollIntervalMedium).Should(BeTrue())
253+
})
254+
})
255+
It("should enable proxy protocol v2", func() {
256+
By("deploying stack", func() {
257+
err := stack.Deploy(ctx, tf, map[string]string{
258+
"service.beta.kubernetes.io/aws-load-balancer-proxy-protocol": "*",
259+
})
260+
Expect(err).ToNot(HaveOccurred())
261+
dnsName = stack.GetLoadBalancerIngressHostName()
262+
Expect(dnsName).ToNot(BeEmpty())
263+
lbARN, err = tf.LBManager.FindLoadBalancerByDNSName(ctx, dnsName)
264+
Expect(err).NotTo(HaveOccurred())
265+
Expect(lbARN).ToNot(BeEmpty())
266+
})
267+
By("verifying target group attributes", func() {
268+
verified := verifyTargetGroupAttributes(ctx, tf, lbARN, map[string]string{
269+
"proxy_protocol_v2.enabled": "true",
270+
})
271+
Expect(verified).To(BeTrue())
272+
})
273+
By("verifying precedence with target group attributes configuration", func() {
274+
err := stack.UpdateServiceAnnotations(ctx, tf, map[string]string{
275+
"service.beta.kubernetes.io/aws-load-balancer-target-group-attributes": "proxy_protocol_v2.enabled=false, deregistration_delay.timeout_seconds=120",
276+
})
277+
Expect(err).NotTo(HaveOccurred())
278+
Eventually(func() bool {
279+
return verifyTargetGroupAttributes(ctx, tf, lbARN, map[string]string{
280+
"proxy_protocol_v2.enabled": "true",
281+
"deregistration_delay.timeout_seconds": "120",
282+
})
283+
}, utils.PollTimeoutShort, utils.PollIntervalMedium).Should(BeTrue())
284+
})
285+
})
286+
})
287+
288+
Context("with NLB instance target configuration with target node labels", func() {
289+
It("should add only the labelled nodes to the target group", func() {
290+
By("deploying stack", func() {
291+
err := stack.Deploy(ctx, tf, map[string]string{
292+
"service.beta.kubernetes.io/aws-load-balancer-target-node-labels": "service.node.label/key1=value1",
293+
})
294+
Expect(err).ToNot(HaveOccurred())
295+
dnsName = stack.GetLoadBalancerIngressHostName()
296+
Expect(dnsName).ToNot(BeEmpty())
297+
lbARN, err = tf.LBManager.FindLoadBalancerByDNSName(ctx, dnsName)
298+
Expect(err).NotTo(HaveOccurred())
299+
Expect(lbARN).ToNot(BeEmpty())
300+
})
301+
By("applying label to 1 worker node", func() {
302+
nodes, err := stack.GetWorkerNodes(ctx, tf)
303+
Expect(err).ToNot(HaveOccurred())
304+
Expect(len(nodes)).To(BeNumerically(">", 0))
305+
err = stack.ApplyNodeLabels(ctx, tf, &nodes[0], map[string]string{"service.node.label/key1": "value1"})
306+
Expect(err).ToNot(HaveOccurred())
307+
308+
targetGroups, err := tf.TGManager.GetTargetGroupsForLoadBalancer(ctx, lbARN)
309+
Expect(err).ToNot(HaveOccurred())
310+
Expect(len(targetGroups)).To(Equal(1))
311+
tgARN := awssdk.StringValue(targetGroups[0].TargetGroupArn)
312+
313+
err = verifyTargetGroupNumRegistered(ctx, tf, tgARN, 1)
314+
Expect(err).ToNot(HaveOccurred())
315+
})
316+
By("removing target-node-labels annotation from the service", func() {
317+
err := stack.DeleteServiceAnnotations(ctx, tf, []string{"service.beta.kubernetes.io/aws-load-balancer-target-node-labels"})
318+
Expect(err).ToNot(HaveOccurred())
319+
320+
targetGroups, err := tf.TGManager.GetTargetGroupsForLoadBalancer(ctx, lbARN)
321+
Expect(err).ToNot(HaveOccurred())
322+
Expect(len(targetGroups)).To(Equal(1))
323+
tgARN := awssdk.StringValue(targetGroups[0].TargetGroupArn)
324+
325+
nodes, err := stack.GetWorkerNodes(ctx, tf)
326+
Expect(err).ToNot(HaveOccurred())
327+
328+
err = verifyTargetGroupNumRegistered(ctx, tf, tgARN, len(nodes))
329+
Expect(err).ToNot(HaveOccurred())
330+
})
331+
})
191332
})
192333
})

‎test/e2e/service/resource_stack.go

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,16 @@ func (s *resourceStack) UpdateServiceAnnotations(ctx context.Context, f *framewo
6565
return nil
6666
}
6767

68+
func (s *resourceStack) DeleteServiceAnnotations(ctx context.Context, f *framework.Framework, annotationKeys []string) error {
69+
if err := s.removeServiceAnnotations(ctx, f, annotationKeys); err != nil {
70+
return err
71+
}
72+
if err := s.waitUntilServiceReady(ctx, f); err != nil {
73+
return err
74+
}
75+
return nil
76+
}
77+
6878
func (s *resourceStack) UpdateServiceTrafficPolicy(ctx context.Context, f *framework.Framework, trafficPolicy corev1.ServiceExternalTrafficPolicyType) error {
6979
if err := s.updateServiceTrafficPolicy(ctx, f, trafficPolicy); err != nil {
7080
return err
@@ -142,6 +152,15 @@ func (s *resourceStack) updateServiceAnnotations(ctx context.Context, f *framewo
142152
return s.updateService(ctx, f, oldSvc)
143153
}
144154

155+
func (s *resourceStack) removeServiceAnnotations(ctx context.Context, f *framework.Framework, annotationKeys []string) error {
156+
f.Logger.Info("removing service annotations", "svc", k8s.NamespacedName(s.svc))
157+
oldSvc := s.svc.DeepCopy()
158+
for _, key := range annotationKeys {
159+
delete(s.svc.Annotations, key)
160+
}
161+
return s.updateService(ctx, f, oldSvc)
162+
}
163+
145164
func (s *resourceStack) updateService(ctx context.Context, f *framework.Framework, oldSvc *corev1.Service) error {
146165
f.Logger.Info("updating service", "svc", k8s.NamespacedName(s.svc))
147166
if err := f.K8sClient.Patch(ctx, s.svc, client.MergeFrom(oldSvc)); err != nil {

‎test/framework/resources/aws/load_balancer.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ type LoadBalancerManager interface {
1515
WaitUntilLoadBalancerAvailable(ctx context.Context, lbARN string) error
1616
GetLoadBalancerFromARN(ctx context.Context, lbARN string) (*elbv2sdk.LoadBalancer, error)
1717
GetLoadBalancerListeners(ctx context.Context, lbARN string) ([]*elbv2sdk.Listener, error)
18+
GetLoadBalancerListenerCertificates(ctx context.Context, listnerARN string) ([]*elbv2sdk.Certificate, error)
1819
GetLoadBalancerAttributes(ctx context.Context, lbARN string) ([]*elbv2sdk.LoadBalancerAttribute, error)
1920
GetLoadBalancerTags(ctx context.Context, lbARN string) ([]*elbv2sdk.Tag, error)
2021
}
@@ -80,6 +81,12 @@ func (m *defaultLoadBalancerManager) GetLoadBalancerListeners(ctx context.Contex
8081
return listeners.Listeners, nil
8182
}
8283

84+
func (m *defaultLoadBalancerManager) GetLoadBalancerListenerCertificates(ctx context.Context, listnerARN string) ([]*elbv2sdk.Certificate, error) {
85+
return m.elbv2Client.DescribeListenerCertificatesAsList(ctx, &elbv2sdk.DescribeListenerCertificatesInput{
86+
ListenerArn: awssdk.String(listnerARN),
87+
})
88+
}
89+
8390
func (m *defaultLoadBalancerManager) GetLoadBalancerAttributes(ctx context.Context, lbARN string) ([]*elbv2sdk.LoadBalancerAttribute, error) {
8491
resp, err := m.elbv2Client.DescribeLoadBalancerAttributesWithContext(ctx, &elbv2sdk.DescribeLoadBalancerAttributesInput{
8592
LoadBalancerArn: awssdk.String(lbARN),

0 commit comments

Comments
 (0)