Skip to content

Fix the default healthCheck setting for GRPC #1698

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Nov 30, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 18 additions & 8 deletions docs/guide/ingress/annotations.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,12 @@ You can add annotations to kubernetes Ingress and Service objects to customize t
|[alb.ingress.kubernetes.io/target-group-attributes](#target-group-attributes)|stringMap|N/A|Ingress,Service|N/A|
|[alb.ingress.kubernetes.io/healthcheck-port](#healthcheck-port)|integer \| traffic-port|traffic-port|Ingress,Service|N/A|
|[alb.ingress.kubernetes.io/healthcheck-protocol](#healthcheck-protocol)|HTTP \| HTTPS|HTTP|Ingress,Service|N/A|
|[alb.ingress.kubernetes.io/healthcheck-path](#healthcheck-path)|string|/|Ingress,Service|N/A|
|[alb.ingress.kubernetes.io/healthcheck-path](#healthcheck-path)|string|/ \| /AWS.ALB/healthcheck |Ingress,Service|N/A|
|[alb.ingress.kubernetes.io/healthcheck-interval-seconds](#healthcheck-interval-seconds)|integer|'15'|Ingress,Service|N/A|
|[alb.ingress.kubernetes.io/healthcheck-timeout-seconds](#healthcheck-timeout-seconds)|integer|'5'|Ingress,Service|N/A|
|[alb.ingress.kubernetes.io/healthy-threshold-count](#healthy-threshold-count)|integer|'2'|Ingress,Service|N/A|
|[alb.ingress.kubernetes.io/unhealthy-threshold-count](#unhealthy-threshold-count)|integer|'2'|Ingress,Service|N/A|
|[alb.ingress.kubernetes.io/success-codes](#success-codes)|string|'200'|Ingress,Service|N/A|
|[alb.ingress.kubernetes.io/success-codes](#success-codes)|string|'200' \| '12' |Ingress,Service|N/A|
|[alb.ingress.kubernetes.io/auth-type](#auth-type)|none\|oidc\|cognito|none|Ingress,Service|N/A|
|[alb.ingress.kubernetes.io/auth-idp-cognito](#auth-idp-cognito)|json|N/A|Ingress,Service|N/A|
|[alb.ingress.kubernetes.io/auth-idp-oidc](#auth-idp-oidc)|json|N/A|Ingress,Service|N/A|
Expand Down Expand Up @@ -170,9 +170,14 @@ Traffic Routing can be controlled with following annotations:
- <a name="backend-protocol-version">`alb.ingress.kubernetes.io/backend-protocol-version`</a> specifies the application protocol used to route traffic to pods. Only valid when HTTP or HTTPS is used as the backend protocol.

!!!example
```
alb.ingress.kubernetes.io/backend-protocol-version: HTTP2
```
- HTTP2
```
alb.ingress.kubernetes.io/backend-protocol-version: HTTP2
```
- GRPC
```
alb.ingress.kubernetes.io/backend-protocol-version: GRPC
```

- <a name="subnets">`alb.ingress.kubernetes.io/subnets`</a> specifies the [Availability Zone](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html) that ALB will route traffic to. See [Load Balancer subnets](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-subnets.html) for more details.

Expand Down Expand Up @@ -521,9 +526,14 @@ Health check on target groups can be controlled with following annotations:
- <a name="healthcheck-path">`alb.ingress.kubernetes.io/healthcheck-path`</a> specifies the HTTP path when performing health check on targets.

!!!example
```
alb.ingress.kubernetes.io/healthcheck-path: /ping
```
- HTTP
```
alb.ingress.kubernetes.io/healthcheck-path: /ping
```
- GRPC
```
alb.ingress.kubernetes.io/healthcheck-path: /package.service/method
```

- <a name="healthcheck-interval-seconds">`alb.ingress.kubernetes.io/healthcheck-interval-seconds`</a> specifies the interval(in seconds) between health check of an individual target.

Expand Down
21 changes: 17 additions & 4 deletions pkg/ingress/model_build_target_group.go
Original file line number Diff line number Diff line change
Expand Up @@ -230,7 +230,7 @@ func (t *defaultModelBuildTask) buildTargetGroupHealthCheckConfig(ctx context.Co
if err != nil {
return elbv2model.TargetGroupHealthCheckConfig{}, err
}
healthCheckPath := t.buildTargetGroupHealthCheckPath(ctx, svcAndIngAnnotations)
healthCheckPath := t.buildTargetGroupHealthCheckPath(ctx, svcAndIngAnnotations, tgProtocolVersion)
healthCheckMatcher := t.buildTargetGroupHealthCheckMatcher(ctx, svcAndIngAnnotations, tgProtocolVersion)
healthCheckIntervalSeconds, err := t.buildTargetGroupHealthCheckIntervalSeconds(ctx, svcAndIngAnnotations)
if err != nil {
Expand Down Expand Up @@ -299,14 +299,27 @@ func (t *defaultModelBuildTask) buildTargetGroupHealthCheckProtocol(_ context.Co
}
}

func (t *defaultModelBuildTask) buildTargetGroupHealthCheckPath(_ context.Context, svcAndIngAnnotations map[string]string) string {
rawHealthCheckPath := t.defaultHealthCheckPath
func (t *defaultModelBuildTask) buildTargetGroupHealthCheckPath(_ context.Context, svcAndIngAnnotations map[string]string, tgProtocolVersion elbv2model.ProtocolVersion) string {
var rawHealthCheckPath string
switch tgProtocolVersion {
case elbv2model.ProtocolVersionHTTP1, elbv2model.ProtocolVersionHTTP2:
rawHealthCheckPath = t.defaultHealthCheckPathHTTP
case elbv2model.ProtocolVersionGRPC:
rawHealthCheckPath = t.defaultHealthCheckPathGRPC
}
_ = t.annotationParser.ParseStringAnnotation(annotations.IngressSuffixHealthCheckPath, &rawHealthCheckPath, svcAndIngAnnotations)
return rawHealthCheckPath
}

func (t *defaultModelBuildTask) buildTargetGroupHealthCheckMatcher(_ context.Context, svcAndIngAnnotations map[string]string, tgProtocolVersion elbv2model.ProtocolVersion) elbv2model.HealthCheckMatcher {
rawHealthCheckMatcherHTTPCode := t.defaultHealthCheckMatcherHTTPCode
var rawHealthCheckMatcherHTTPCode string
switch tgProtocolVersion {
case elbv2model.ProtocolVersionHTTP1, elbv2model.ProtocolVersionHTTP2:
rawHealthCheckMatcherHTTPCode = t.defaultHealthCheckMatcherHTTPCode
case elbv2model.ProtocolVersionGRPC:
rawHealthCheckMatcherHTTPCode = t.defaultHealthCheckMatcherGRPCCode
}

_ = t.annotationParser.ParseStringAnnotation(annotations.IngressSuffixSuccessCodes, &rawHealthCheckMatcherHTTPCode, svcAndIngAnnotations)
if tgProtocolVersion == elbv2model.ProtocolVersionGRPC {
return elbv2model.HealthCheckMatcher{
Expand Down
227 changes: 227 additions & 0 deletions pkg/ingress/model_build_target_group_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package ingress

import (
"context"
awssdk "github.com/aws/aws-sdk-go/aws"
"github.com/stretchr/testify/assert"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down Expand Up @@ -283,3 +284,229 @@ func Test_defaultModelBuildTask_buildTargetGroupTags(t *testing.T) {
})
}
}

func Test_defaultModelBuildTask_buildTargetGroupHealthCheckPath(t *testing.T) {
type fields struct {
defaultHealthCheckPathHTTP string
defaultHealthCheckPathGRPC string
}
type args struct {
svcAndIngAnnotations map[string]string
tgProtocolVersion elbv2model.ProtocolVersion
}
tests := []struct {
name string
fields fields
args args
want string
}{
{
name: "HTTP1, without annotation configured",
fields: fields{
defaultHealthCheckPathHTTP: "/",
defaultHealthCheckPathGRPC: "/AWS.ALB/healthcheck",
},
args: args{
svcAndIngAnnotations: nil,
tgProtocolVersion: elbv2model.ProtocolVersionHTTP1,
},
want: "/",
},
{
name: "HTTP2, without annotation configured",
fields: fields{
defaultHealthCheckPathHTTP: "/",
defaultHealthCheckPathGRPC: "/AWS.ALB/healthcheck",
},
args: args{
svcAndIngAnnotations: nil,
tgProtocolVersion: elbv2model.ProtocolVersionHTTP2,
},
want: "/",
},
{
name: "GRPC, without annotation configured",
fields: fields{
defaultHealthCheckPathHTTP: "/",
defaultHealthCheckPathGRPC: "/AWS.ALB/healthcheck",
},
args: args{
svcAndIngAnnotations: nil,
tgProtocolVersion: elbv2model.ProtocolVersionGRPC,
},
want: "/AWS.ALB/healthcheck",
},
{
name: "HTTP1, with annotation configured",
fields: fields{
defaultHealthCheckPathHTTP: "/",
defaultHealthCheckPathGRPC: "/AWS.ALB/healthcheck",
},
args: args{
svcAndIngAnnotations: map[string]string{
"alb.ingress.kubernetes.io/healthcheck-path": "/ping",
},
tgProtocolVersion: elbv2model.ProtocolVersionHTTP1,
},
want: "/ping",
},
{
name: "HTTP2, with annotation configured",
fields: fields{
defaultHealthCheckPathHTTP: "/",
defaultHealthCheckPathGRPC: "/AWS.ALB/healthcheck",
},
args: args{
svcAndIngAnnotations: map[string]string{
"alb.ingress.kubernetes.io/healthcheck-path": "/ping",
},
tgProtocolVersion: elbv2model.ProtocolVersionHTTP2,
},
want: "/ping",
},
{
name: "GRPC, with annotation configured",
fields: fields{
defaultHealthCheckPathHTTP: "/",
defaultHealthCheckPathGRPC: "/AWS.ALB/healthcheck",
},
args: args{
svcAndIngAnnotations: map[string]string{
"alb.ingress.kubernetes.io/healthcheck-path": "/package.service/method",
},
tgProtocolVersion: elbv2model.ProtocolVersionGRPC,
},
want: "/package.service/method",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
task := &defaultModelBuildTask{
annotationParser: annotations.NewSuffixAnnotationParser("alb.ingress.kubernetes.io"),
defaultHealthCheckPathHTTP: tt.fields.defaultHealthCheckPathHTTP,
defaultHealthCheckPathGRPC: tt.fields.defaultHealthCheckPathGRPC,
}
got := task.buildTargetGroupHealthCheckPath(context.Background(), tt.args.svcAndIngAnnotations, tt.args.tgProtocolVersion)
assert.Equal(t, tt.want, got)
})
}
}

func Test_defaultModelBuildTask_buildTargetGroupHealthCheckMatcher(t *testing.T) {
type fields struct {
defaultHealthCheckMatcherHTTPCode string
defaultHealthCheckMatcherGRPCCode string
}
type args struct {
svcAndIngAnnotations map[string]string
tgProtocolVersion elbv2model.ProtocolVersion
}
tests := []struct {
name string
fields fields
args args
want elbv2model.HealthCheckMatcher
}{
{
name: "HTTP1, without annotation configured",
fields: fields{
defaultHealthCheckMatcherHTTPCode: "200",
defaultHealthCheckMatcherGRPCCode: "12",
},
args: args{
svcAndIngAnnotations: nil,
tgProtocolVersion: elbv2model.ProtocolVersionHTTP1,
},
want: elbv2model.HealthCheckMatcher{
HTTPCode: awssdk.String("200"),
},
},
{
name: "HTTP2, without annotation configured",
fields: fields{
defaultHealthCheckMatcherHTTPCode: "200",
defaultHealthCheckMatcherGRPCCode: "12",
},
args: args{
svcAndIngAnnotations: nil,
tgProtocolVersion: elbv2model.ProtocolVersionHTTP2,
},
want: elbv2model.HealthCheckMatcher{
HTTPCode: awssdk.String("200"),
},
},
{
name: "GRPC, without annotation configured",
fields: fields{
defaultHealthCheckMatcherHTTPCode: "200",
defaultHealthCheckMatcherGRPCCode: "12",
},
args: args{
svcAndIngAnnotations: nil,
tgProtocolVersion: elbv2model.ProtocolVersionGRPC,
},
want: elbv2model.HealthCheckMatcher{
GRPCCode: awssdk.String("12"),
},
},
{
name: "HTTP1, with annotation configured",
fields: fields{
defaultHealthCheckMatcherHTTPCode: "200",
defaultHealthCheckMatcherGRPCCode: "12",
},
args: args{
svcAndIngAnnotations: map[string]string{
"alb.ingress.kubernetes.io/success-codes": "200-300",
},
tgProtocolVersion: elbv2model.ProtocolVersionHTTP1,
},
want: elbv2model.HealthCheckMatcher{
HTTPCode: awssdk.String("200-300"),
},
},
{
name: "HTTP2, with annotation configured",
fields: fields{
defaultHealthCheckMatcherHTTPCode: "200",
defaultHealthCheckMatcherGRPCCode: "12",
},
args: args{
svcAndIngAnnotations: map[string]string{
"alb.ingress.kubernetes.io/success-codes": "200-300",
},
tgProtocolVersion: elbv2model.ProtocolVersionHTTP2,
},
want: elbv2model.HealthCheckMatcher{
HTTPCode: awssdk.String("200-300"),
},
},
{
name: "GRPC, with annotation configured",
fields: fields{
defaultHealthCheckMatcherHTTPCode: "200",
defaultHealthCheckMatcherGRPCCode: "12",
},
args: args{
svcAndIngAnnotations: map[string]string{
"alb.ingress.kubernetes.io/success-codes": "0",
},
tgProtocolVersion: elbv2model.ProtocolVersionGRPC,
},
want: elbv2model.HealthCheckMatcher{
GRPCCode: awssdk.String("0"),
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
task := &defaultModelBuildTask{
annotationParser: annotations.NewSuffixAnnotationParser("alb.ingress.kubernetes.io"),
defaultHealthCheckMatcherHTTPCode: tt.fields.defaultHealthCheckMatcherHTTPCode,
defaultHealthCheckMatcherGRPCCode: tt.fields.defaultHealthCheckMatcherGRPCCode,
}
got := task.buildTargetGroupHealthCheckMatcher(context.Background(), tt.args.svcAndIngAnnotations, tt.args.tgProtocolVersion)
assert.Equal(t, tt.want, got)
})
}
}
8 changes: 6 additions & 2 deletions pkg/ingress/model_builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,12 +103,14 @@ func (b *defaultModelBuilder) Build(ctx context.Context, ingGroup Group) (core.S
defaultTargetType: elbv2model.TargetTypeInstance,
defaultBackendProtocol: elbv2model.ProtocolHTTP,
defaultBackendProtocolVersion: elbv2model.ProtocolVersionHTTP1,
defaultHealthCheckPath: "/",
defaultHealthCheckPathHTTP: "/",
defaultHealthCheckPathGRPC: "/AWS.ALB/healthcheck",
defaultHealthCheckIntervalSeconds: 15,
defaultHealthCheckTimeoutSeconds: 5,
defaultHealthCheckHealthyThresholdCount: 2,
defaultHealthCheckUnhealthyThresholdCount: 2,
defaultHealthCheckMatcherHTTPCode: "200",
defaultHealthCheckMatcherGRPCCode: "12",

loadBalancer: nil,
tgByResID: make(map[string]*elbv2model.TargetGroup),
Expand Down Expand Up @@ -144,12 +146,14 @@ type defaultModelBuildTask struct {
defaultTargetType elbv2model.TargetType
defaultBackendProtocol elbv2model.Protocol
defaultBackendProtocolVersion elbv2model.ProtocolVersion
defaultHealthCheckPath string
defaultHealthCheckPathHTTP string
defaultHealthCheckPathGRPC string
defaultHealthCheckTimeoutSeconds int64
defaultHealthCheckIntervalSeconds int64
defaultHealthCheckHealthyThresholdCount int64
defaultHealthCheckUnhealthyThresholdCount int64
defaultHealthCheckMatcherHTTPCode string
defaultHealthCheckMatcherGRPCCode string

loadBalancer *elbv2model.LoadBalancer
managedSG *ec2model.SecurityGroup
Expand Down