Skip to content

Commit 8c4701d

Browse files
KiranmayiBKiranmayiB
committed
feat: GFE latency and GFE Header missing count
1 parent 29209f8 commit 8c4701d

File tree

5 files changed

+610
-2
lines changed

5 files changed

+610
-2
lines changed

‎google-cloud-spanner/pom.xml

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@
110110
<groupId>org.apache.maven.plugins</groupId>
111111
<artifactId>maven-dependency-plugin</artifactId>
112112
<configuration>
113-
<ignoredDependencies>io.grpc:grpc-protobuf-lite,org.hamcrest:hamcrest,org.hamcrest:hamcrest-core,com.google.errorprone:error_prone_annotations,org.openjdk.jmh:jmh-generator-annprocess,com.google.api.grpc:grpc-google-cloud-spanner-v1,com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1,com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1,javax.annotation:javax.annotation-api</ignoredDependencies>
113+
<ignoredDependencies>io.grpc:grpc-protobuf-lite,org.hamcrest:hamcrest,org.hamcrest:hamcrest-core,com.google.errorprone:error_prone_annotations,org.openjdk.jmh:jmh-generator-annprocess,com.google.api.grpc:grpc-google-cloud-spanner-v1,com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1,com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1,javax.annotation:javax.annotation-api,io.opencensus:opencensus-impl</ignoredDependencies>
114114
</configuration>
115115
</plugin>
116116
</plugins>
@@ -191,6 +191,11 @@
191191
<groupId>io.opencensus</groupId>
192192
<artifactId>opencensus-contrib-grpc-util</artifactId>
193193
</dependency>
194+
<dependency>
195+
<groupId>io.opencensus</groupId>
196+
<artifactId>opencensus-impl</artifactId>
197+
<scope>test</scope>
198+
</dependency>
194199
<dependency>
195200
<groupId>com.google.auth</groupId>
196201
<artifactId>google-auth-library-oauth2-http</artifactId>

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
/*
2+
* Copyright 2021 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.cloud.spanner.spi.v1;
17+
18+
import static com.google.cloud.spanner.spi.v1.SpannerRpcViews.*;
19+
20+
import io.grpc.*;
21+
import io.grpc.ForwardingClientCall.SimpleForwardingClientCall;
22+
import io.grpc.ForwardingClientCallListener.SimpleForwardingClientCallListener;
23+
import io.opencensus.stats.*;
24+
import io.opencensus.tags.*;
25+
import java.util.logging.Level;
26+
import java.util.logging.Logger;
27+
import java.util.regex.Matcher;
28+
import java.util.regex.Pattern;
29+
30+
/**
31+
* Intercepts all gRPC calls to extract server-timing header. Captures GFELatency and GFE Header
32+
* Missing count metrics
33+
*/
34+
class HeaderInterceptor implements ClientInterceptor {
35+
36+
private static final Metadata.Key<String> SERVER_TIMING_HEADER_KEY =
37+
Metadata.Key.of("server-timing", Metadata.ASCII_STRING_MARSHALLER);
38+
private static final Pattern SERVER_TIMING_HEADER_PATTERN = Pattern.compile(".*dur=(?<dur>\\d+)");
39+
private static final Metadata.Key<String> GOOGLE_CLOUD_RESOURCE_PREFIX_KEY =
40+
Metadata.Key.of("google-cloud-resource-prefix", Metadata.ASCII_STRING_MARSHALLER);
41+
private static final Pattern GOOGLE_CLOUD_RESOURCE_PREFIX_PATTERN =
42+
Pattern.compile(
43+
".*projects/(?<project>\\w\\p{ASCII}+)/instances/(?<instance>\\w\\p{ASCII}+)/databases/(?<database>\\w\\p{ASCII}+)");
44+
45+
// Get the global singleton Tagger object.
46+
private static final Tagger tagger = Tags.getTagger();
47+
48+
private static final Logger logger = Logger.getLogger(HeaderInterceptor.class.getName());
49+
private static final Level level = Level.INFO;
50+
51+
HeaderInterceptor() {}
52+
53+
@Override
54+
public <ReqT, RespT> ClientCall<ReqT, RespT> interceptCall(
55+
MethodDescriptor<ReqT, RespT> method, CallOptions callOptions, Channel next) {
56+
return new SimpleForwardingClientCall<ReqT, RespT>(next.newCall(method, callOptions)) {
57+
@Override
58+
public void start(Listener<RespT> responseListener, Metadata headers) {
59+
TagContext tagContext = getTagContext(headers, method.getFullMethodName());
60+
super.start(
61+
new SimpleForwardingClientCallListener<RespT>(responseListener) {
62+
@Override
63+
public void onHeaders(Metadata metadata) {
64+
processHeader(metadata, tagContext);
65+
super.onHeaders(metadata);
66+
}
67+
},
68+
headers);
69+
}
70+
};
71+
}
72+
73+
private void processHeader(Metadata metadata, TagContext tagContext) {
74+
MeasureMap measureMap = STATS_RECORDER.newMeasureMap();
75+
if (metadata.get(SERVER_TIMING_HEADER_KEY) != null) {
76+
String serverTiming = metadata.get(SERVER_TIMING_HEADER_KEY);
77+
Matcher matcher = SERVER_TIMING_HEADER_PATTERN.matcher(serverTiming);
78+
if (matcher.find()) {
79+
try {
80+
long latency = Long.parseLong(matcher.group("dur"));
81+
measureMap.put(SPANNER_GFE_LATENCY, latency).record(tagContext);
82+
measureMap.put(SPANNER_GFE_HEADER_MISSING_COUNT, 0L).record(tagContext);
83+
} catch (NumberFormatException e) {
84+
logger.log(level, "invalid server-timing object in header");
85+
}
86+
}
87+
} else {
88+
measureMap.put(SPANNER_GFE_HEADER_MISSING_COUNT, 1L).record(tagContext);
89+
}
90+
}
91+
92+
private TagContext getTagContext(
93+
String method, String projectId, String instanceId, String databaseId) {
94+
return tagger
95+
.currentBuilder()
96+
.putLocal(SpannerRpcViews.PROJECT_ID, TagValue.create(projectId))
97+
.putLocal(INSTANCE_ID, TagValue.create(instanceId))
98+
.putLocal(DATABASE_ID, TagValue.create(databaseId))
99+
.putLocal(SpannerRpcViews.METHOD, TagValue.create(method))
100+
.build();
101+
}
102+
103+
private TagContext getTagContext(String method) {
104+
return tagger
105+
.currentBuilder()
106+
.putLocal(PROJECT_ID, TagValue.create("undefined-project"))
107+
.putLocal(INSTANCE_ID, TagValue.create("undefined-instance"))
108+
.putLocal(DATABASE_ID, TagValue.create("undefined-database"))
109+
.putLocal(SpannerRpcViews.METHOD, TagValue.create(method))
110+
.build();
111+
}
112+
113+
private TagContext getTagContext(Metadata headers, String method) {
114+
if (headers.get(GOOGLE_CLOUD_RESOURCE_PREFIX_KEY) != null) {
115+
String googleResourcePrefix = headers.get(GOOGLE_CLOUD_RESOURCE_PREFIX_KEY);
116+
Matcher matcher = GOOGLE_CLOUD_RESOURCE_PREFIX_PATTERN.matcher(googleResourcePrefix);
117+
if (matcher.find()) {
118+
String projectId = matcher.group("project");
119+
String instanceId = matcher.group("instance");
120+
String databaseId = matcher.group("database");
121+
return getTagContext(method, projectId, instanceId, databaseId);
122+
}
123+
}
124+
return getTagContext(method);
125+
}
126+
}

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/SpannerInterceptorProvider.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,8 @@ public class SpannerInterceptorProvider implements GrpcInterceptorProvider {
3333
private static final List<ClientInterceptor> defaultInterceptors =
3434
ImmutableList.of(
3535
new SpannerErrorInterceptor(),
36-
new LoggingInterceptor(Logger.getLogger(GapicSpannerRpc.class.getName()), Level.FINER));
36+
new LoggingInterceptor(Logger.getLogger(GapicSpannerRpc.class.getName()), Level.FINER),
37+
new HeaderInterceptor());
3738

3839
private final List<ClientInterceptor> clientInterceptors;
3940

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/SpannerRpcViews.java

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
/*
2+
* Copyright 2021 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.cloud.spanner.spi.v1;
17+
18+
import com.google.common.annotations.VisibleForTesting;
19+
import com.google.common.collect.ImmutableList;
20+
import io.opencensus.stats.*;
21+
import io.opencensus.stats.Aggregation.Distribution;
22+
import io.opencensus.stats.Aggregation.Sum;
23+
import io.opencensus.stats.Measure.MeasureLong;
24+
import io.opencensus.tags.TagKey;
25+
import java.util.Arrays;
26+
import java.util.Collections;
27+
import java.util.List;
28+
29+
@VisibleForTesting
30+
public class SpannerRpcViews {
31+
32+
/** Unit to represent milliseconds. */
33+
private static final String MILLISECOND = "ms";
34+
/** Unit to represent counts. */
35+
private static final String COUNT = "1";
36+
37+
/** TagKeys */
38+
public static final TagKey METHOD = TagKey.create("grpc_client_method");
39+
40+
public static final TagKey PROJECT_ID = TagKey.create("spanner_project_id");
41+
public static final TagKey INSTANCE_ID = TagKey.create("spanner_instance_id");
42+
public static final TagKey DATABASE_ID = TagKey.create("spanner_database_id");
43+
44+
public static final StatsRecorder STATS_RECORDER = Stats.getStatsRecorder();
45+
/** GFE t4t7 latency extracted from server-timing header. */
46+
public static final MeasureLong SPANNER_GFE_LATENCY =
47+
MeasureLong.create(
48+
"cloud.google.com/java/spanner/gfe_latency",
49+
"Latency between Google's network receiving an RPC and reading back the first byte of the response",
50+
MILLISECOND);
51+
/** Number of responses without the server-timing header. */
52+
public static final MeasureLong SPANNER_GFE_HEADER_MISSING_COUNT =
53+
MeasureLong.create(
54+
"cloud.google.com/java/spanner/gfe_header_missing_count",
55+
"Number of RPC responses received without the server-timing header, most likely means that the RPC never reached Google's network",
56+
COUNT);
57+
58+
static final List<Double> RPC_MILLIS_BUCKET_BOUNDARIES =
59+
Collections.unmodifiableList(
60+
Arrays.asList(
61+
0.0, 0.01, 0.05, 0.1, 0.3, 0.6, 0.8, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 8.0, 10.0, 13.0,
62+
16.0, 20.0, 25.0, 30.0, 40.0, 50.0, 65.0, 80.0, 100.0, 130.0, 160.0, 200.0, 250.0,
63+
300.0, 400.0, 500.0, 650.0, 800.0, 1000.0, 2000.0, 5000.0, 10000.0, 20000.0, 50000.0,
64+
100000.0));
65+
static final Aggregation AGGREGATION_WITH_MILLIS_HISTOGRAM =
66+
Distribution.create(BucketBoundaries.create(RPC_MILLIS_BUCKET_BOUNDARIES));
67+
static final View SPANNER_GFE_LATENCY_VIEW =
68+
View.create(
69+
View.Name.create("cloud.google.com/java/spanner/gfe_latency"),
70+
"Latency between Google's network receiving an RPC and reading back the first byte of the response",
71+
SPANNER_GFE_LATENCY,
72+
AGGREGATION_WITH_MILLIS_HISTOGRAM,
73+
ImmutableList.of(METHOD, PROJECT_ID, INSTANCE_ID, DATABASE_ID));
74+
75+
private static final Aggregation SUM = Sum.create();
76+
static final View SPANNER_GFE_HEADER_MISSING_COUNT_VIEW =
77+
View.create(
78+
View.Name.create("cloud.google.com/java/spanner/gfe_header_missing_count"),
79+
"Number of RPC responses received without the server-timing header, most likely means that the RPC never reached Google's network",
80+
SPANNER_GFE_HEADER_MISSING_COUNT,
81+
SUM,
82+
ImmutableList.of(METHOD, PROJECT_ID, INSTANCE_ID, DATABASE_ID));
83+
84+
public static ViewManager viewManager = Stats.getViewManager();
85+
86+
/**
87+
* Register views for GFE metrics, including gfe_latency and gfe_header_missing_count. gfe_latency
88+
* measures the latency between Google's network receives an RPC and reads back the first byte of
89+
* the response. gfe_header_missing_count is a counter of the number of RPC responses without a
90+
* server-timing header.
91+
*/
92+
@VisibleForTesting
93+
public static void registerGfeLatencyAndHeaderMissingCountViews() {
94+
viewManager.registerView(SPANNER_GFE_LATENCY_VIEW);
95+
viewManager.registerView(SPANNER_GFE_HEADER_MISSING_COUNT_VIEW);
96+
}
97+
98+
/**
99+
* Register GFE Latency view. gfe_latency measures the latency between Google's network receives
100+
* an RPC and reads back the first byte of the response.
101+
*/
102+
@VisibleForTesting
103+
public static void registerGfeLatencyView() {
104+
viewManager.registerView(SPANNER_GFE_LATENCY_VIEW);
105+
}
106+
107+
/**
108+
* Register GFE Header Missing Count view. gfe_header_missing_count is a counter of the number of
109+
* RPC responses without a server-timing header.
110+
*/
111+
@VisibleForTesting
112+
public static void registerGfeHeaderMissingCountView() {
113+
viewManager.registerView(SPANNER_GFE_HEADER_MISSING_COUNT_VIEW);
114+
}
115+
}

0 commit comments

Comments
 (0)