Skip to content

Add default tags to backend SG #2425

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jan 7, 2022
Merged

Add default tags to backend SG #2425

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2d33d3f
Add default tags to backend SG
kishorj Dec 22, 2021
a93737b
update controller configurations doc
kishorj Dec 22, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions docs/deploy/configurations.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,23 +66,28 @@ Currently, you can set only 1 namespace to watch in this flag. See [this Kuberne

|Flag | Type | Default | Description |
|---------------------------------------|---------------------------------|-----------------|-------------|
|aws-api-endpoints | AWS API Endpoints Config | | AWS API endpoints mapping, format: serviceID1=URL1,serviceID2=URL2 |
|aws-api-throttle | AWS Throttle Config | [default value](#default-throttle-config ) | throttle settings for AWS APIs, format: serviceID1:operationRegex1=rate:burst,serviceID2:operationRegex2=rate:burst |
|aws-max-retries | int | 10 | Maximum retries for AWS APIs |
|aws-region | string | [instance metadata](#instance-metadata) | AWS Region for the kubernetes cluster |
|aws-vpc-id | string | [instance metadata](#instance-metadata) | AWS VPC ID for the Kubernetes cluster |
|aws-api-endpoints | AWS API Endpoints Config | | AWS API endpoints mapping, format: serviceID1=URL1,serviceID2=URL2 |
|backend-security-group | string | | Backend security group id to use for the ingress rules on the worker node SG|
|cluster-name | string | | Kubernetes cluster name|
|default-tags | stringMap | | AWS Tags that will be applied to all AWS resources managed by this controller. Specified Tags takes highest priority |
|default-ssl-policy | string | ELBSecurityPolicy-2016-08 | Default SSL Policy that will be applied to all Ingresses or Services that do not have the SSL Policy annotation |
|default-tags | stringMap | | AWS Tags that will be applied to all AWS resources managed by this controller. Specified Tags takes highest priority |
|[disable-ingress-class-annotation](#disable-ingress-class-annotation) | boolean | false | Disable new usage of the `kubernetes.io/ingress.class` annotation |
|[disable-ingress-group-name-annotation](#disable-ingress-group-name-annotation) | boolean | false | Disallow new use of the `alb.ingress.kubernetes.io/group.name` annotation |
|disable-restricted-sg-rules | boolean | false | Disable the usage of restricted security group rules |
|enable-backend-security-group | boolean | true | Enable sharing of security groups for backend traffic |
|enable-endpoint-slices | boolean | false | Use EndpointSlices instead of Endpoints for pod endpoint and TargetGroupBinding resolution for load balancers with IP targets. |
|enable-leader-election | boolean | true | Enable leader election for the load balancer controller manager. Enabling this will ensure there is only one active controller manager |
|enable-pod-readiness-gate-inject | boolean | true | If enabled, targetHealth readiness gate will get injected to the pod spec for the matching endpoint pods |
|enable-shield | boolean | true | Enable Shield addon for ALB |
|enable-waf | boolean | true | Enable WAF addon for ALB |
|enable-wafv2 | boolean | true | Enable WAF V2 addon for ALB |
|external-managed-tags | stringList | | AWS Tag keys that will be managed externally. Specified Tags are ignored during reconciliation |
|[feature-gates](#feature-gates) | stringMap | | A set of key=value pairs to enable or disable features |
|health-probe-bind-addr | string | :61779 | The address the health probes binds to |
|ingress-class | string | alb | Name of the ingress class this controller satisfies |
|ingress-max-concurrent-reconciles | int | 3 | Maximum number of concurrently running reconcile loops for ingress |
|kubeconfig | string | in-cluster config | Path to the kubeconfig file containing authorization and API server information |
Expand All @@ -94,7 +99,6 @@ Currently, you can set only 1 namespace to watch in this flag. See [this Kuberne
|sync-period | duration | 1h0m0s | Period at which the controller forces the repopulation of its local object stores|
|targetgroupbinding-max-concurrent-reconciles | int | 3 | Maximum number of concurrently running reconcile loops for targetGroupBinding |
|targetgroupbinding-max-exponential-backoff-delay | duration | 16m40s | Maximum duration of exponential backoff for targetGroupBinding reconcile failures |
|enable-endpoint-slices | boolean | false | Use EndpointSlices instead of Endpoints for pod endpoint and TargetGroupBinding resolution for load balancers with IP targets. |
|watch-namespace | string | | Namespace the controller watches for updates to Kubernetes objects, If empty, all namespaces are watched. |
|webhook-bind-port | int | 9443 | The TCP port the Webhook server binds to |
|webhook-cert-dir | string | /tmp/k8s-webhook-server/serving-certs | The directory that contains the server key and certificate |
Expand Down
2 changes: 1 addition & 1 deletion main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ func main() {
tgbResManager := targetgroupbinding.NewDefaultResourceManager(mgr.GetClient(), cloud.ELBV2(), cloud.EC2(),
podInfoRepo, sgManager, sgReconciler, cloud.VpcID(), controllerCFG.ClusterName, mgr.GetEventRecorderFor("targetGroupBinding"), ctrl.Log, controllerCFG.EnableEndpointSlices, controllerCFG.DisableRestrictedSGRules, vpcInfoProvider)
backendSGProvider := networking.NewBackendSGProvider(controllerCFG.ClusterName, controllerCFG.BackendSecurityGroup,
cloud.VpcID(), cloud.EC2(), mgr.GetClient(), ctrl.Log.WithName("backend-sg-provider"))
cloud.VpcID(), cloud.EC2(), mgr.GetClient(), controllerCFG.DefaultTags, ctrl.Log.WithName("backend-sg-provider"))
ingGroupReconciler := ingress.NewGroupReconciler(cloud, mgr.GetClient(), mgr.GetEventRecorderFor("ingress"),
finalizerManager, sgManager, sgReconciler, subnetResolver,
controllerCFG, backendSGProvider, ctrl.Log.WithName("controllers").WithName("ingress"))
Expand Down
1 change: 1 addition & 0 deletions pkg/config/controller_config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ const (
var (
trackingTagKeys = sets.NewString(
"elbv2.k8s.aws/cluster",
"elbv2.k8s.aws/resource",
"ingress.k8s.aws/stack",
"ingress.k8s.aws/resource",
"service.k8s.aws/stack",
Expand Down
28 changes: 21 additions & 7 deletions pkg/networking/backend_sg_provider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,21 @@ import (
"crypto/sha256"
"encoding/hex"
"fmt"
"regexp"
"sort"
"strings"
"sync"
"time"

awssdk "github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
ec2sdk "github.com/aws/aws-sdk-go/service/ec2"
"github.com/go-logr/logr"
"github.com/pkg/errors"
networking "k8s.io/api/networking/v1"
"regexp"
"sigs.k8s.io/aws-load-balancer-controller/pkg/aws/services"
"sigs.k8s.io/aws-load-balancer-controller/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"strings"
"sync"
"time"
)

const (
Expand Down Expand Up @@ -45,11 +47,12 @@ type BackendSGProvider interface {

// NewBackendSGProvider constructs a new defaultBackendSGProvider
func NewBackendSGProvider(clusterName string, backendSG string, vpcID string,
ec2Client services.EC2, k8sClient client.Client, logger logr.Logger) *defaultBackendSGProvider {
ec2Client services.EC2, k8sClient client.Client, defaultTags map[string]string, logger logr.Logger) *defaultBackendSGProvider {
return &defaultBackendSGProvider{
vpcID: vpcID,
clusterName: clusterName,
backendSG: backendSG,
defaultTags: defaultTags,
ec2Client: ec2Client,
k8sClient: k8sClient,
logger: logger,
Expand All @@ -69,6 +72,7 @@ type defaultBackendSGProvider struct {

backendSG string
autoGeneratedSG string
defaultTags map[string]string
ec2Client services.EC2
k8sClient client.Client
logger logr.Logger
Expand Down Expand Up @@ -135,10 +139,20 @@ func (p *defaultBackendSGProvider) allocateBackendSG(ctx context.Context) error
}

func (p *defaultBackendSGProvider) buildBackendSGTags(_ context.Context) []*ec2sdk.TagSpecification {
var defaultTags []*ec2sdk.Tag
for key, val := range p.defaultTags {
defaultTags = append(defaultTags, &ec2sdk.Tag{
Key: awssdk.String(key),
Value: awssdk.String(val),
})
}
sort.Slice(defaultTags, func(i, j int) bool {
return awssdk.StringValue(defaultTags[i].Key) < awssdk.StringValue(defaultTags[j].Key)
})
return []*ec2sdk.TagSpecification{
{
ResourceType: awssdk.String(resourceTypeSecurityGroup),
Tags: []*ec2sdk.Tag{
Tags: append(defaultTags, []*ec2sdk.Tag{
{
Key: awssdk.String(tagKeyK8sCluster),
Value: awssdk.String(p.clusterName),
Expand All @@ -147,7 +161,7 @@ func (p *defaultBackendSGProvider) buildBackendSGTags(_ context.Context) []*ec2s
Key: awssdk.String(tagKeyResource),
Value: awssdk.String(tagValueBackend),
},
},
}...),
},
}
}
Expand Down
64 changes: 62 additions & 2 deletions pkg/networking/backend_sg_provider_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ func Test_defaultBackendSGProvider_Get(t *testing.T) {
}
type fields struct {
backendSG string
defaultTags map[string]string
describeSGCalls []describeSecurityGroupsAsListCall
createSGCalls []createSecurityGroupWithContexCall
}
Expand Down Expand Up @@ -125,6 +126,64 @@ func Test_defaultBackendSGProvider_Get(t *testing.T) {
},
want: "sg-newauto",
},
{
name: "backend sg enabled, auto-gen new SG with additional defaultTags",
fields: fields{
describeSGCalls: []describeSecurityGroupsAsListCall{
{
req: &ec2sdk.DescribeSecurityGroupsInput{
Filters: defaultEC2Filters,
},
err: awserr.New("InvalidGroup.NotFound", "", nil),
},
},
createSGCalls: []createSecurityGroupWithContexCall{
{
req: &ec2sdk.CreateSecurityGroupInput{
Description: awssdk.String(sgDescription),
GroupName: awssdk.String("k8s-traffic-testCluster-411a1bcdb1"),
TagSpecifications: []*ec2sdk.TagSpecification{
{
ResourceType: awssdk.String("security-group"),
Tags: []*ec2sdk.Tag{
{
Key: awssdk.String("KubernetesCluster"),
Value: awssdk.String(defaultClusterName),
},
{
Key: awssdk.String("defaultTag"),
Value: awssdk.String("specified"),
},
{
Key: awssdk.String("zzzKey"),
Value: awssdk.String("value"),
},
{
Key: awssdk.String("elbv2.k8s.aws/cluster"),
Value: awssdk.String(defaultClusterName),
},
{
Key: awssdk.String("elbv2.k8s.aws/resource"),
Value: awssdk.String("backend-sg"),
},
},
},
},
VpcId: awssdk.String(defaultVPCID),
},
resp: &ec2sdk.CreateSecurityGroupOutput{
GroupId: awssdk.String("sg-newauto"),
},
},
},
defaultTags: map[string]string{
"zzzKey": "value",
"KubernetesCluster": defaultClusterName,
"defaultTag": "specified",
},
},
want: "sg-newauto",
},
{
name: "describe SG call returns error",
fields: fields{
Expand Down Expand Up @@ -193,7 +252,7 @@ func Test_defaultBackendSGProvider_Get(t *testing.T) {
}
k8sClient := mock_client.NewMockClient(ctrl)
sgProvider := NewBackendSGProvider(defaultClusterName, tt.fields.backendSG,
defaultVPCID, ec2Client, k8sClient, &log.NullLogger{})
defaultVPCID, ec2Client, k8sClient, tt.fields.defaultTags, &log.NullLogger{})

got, err := sgProvider.Get(context.Background())
if tt.wantErr != nil {
Expand Down Expand Up @@ -222,6 +281,7 @@ func Test_defaultBackendSGProvider_Release(t *testing.T) {
type fields struct {
autogenSG string
backendSG string
defaultTags map[string]string
listIngressCalls []listIngressCall
deleteSGCalls []deleteSecurityGroupWithContextCall
}
Expand Down Expand Up @@ -365,7 +425,7 @@ func Test_defaultBackendSGProvider_Release(t *testing.T) {
ec2Client := services.NewMockEC2(ctrl)
k8sClient := mock_client.NewMockClient(ctrl)
sgProvider := NewBackendSGProvider(defaultClusterName, tt.fields.backendSG,
defaultVPCID, ec2Client, k8sClient, &log.NullLogger{})
defaultVPCID, ec2Client, k8sClient, tt.fields.defaultTags, &log.NullLogger{})
if len(tt.fields.autogenSG) > 0 {
sgProvider.backendSG = ""
sgProvider.autoGeneratedSG = tt.fields.autogenSG
Expand Down