Add http redirect support to the cct transport backend. (#750)

Redirects are followed up to 4 times(after initial request).

Author: vkryachko

transport/transport-backend-cct/src/main/java/com/google/android/datatransport/cct/CctTransportBackend.java

@@ -14,10 +14,13 @@
 
 package com.google.android.datatransport.cct;
 
+import static com.google.android.datatransport.runtime.retries.Retries.retry;
+
 import android.content.Context;
 import android.net.ConnectivityManager;
 import android.net.NetworkInfo;
 import android.os.Build;
+import androidx.annotation.Nullable;
 import androidx.annotation.VisibleForTesting;
 import com.google.android.datatransport.backend.cct.BuildConfig;
 import com.google.android.datatransport.cct.proto.AndroidClientInfo;
@@ -36,7 +39,6 @@
 import com.google.android.datatransport.runtime.backends.TransportBackend;
 import com.google.android.datatransport.runtime.time.Clock;
 import com.google.protobuf.ByteString;
-import com.google.protobuf.InvalidProtocolBufferException;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -212,8 +214,9 @@ private BatchedLogRequest getRequestBody(BackendRequest backendRequest) {
     return batchedRequestBuilder.build();
   }
 
-  private BackendResponse doSend(BatchedLogRequest requestBody, String apiKey) throws IOException {
-    HttpURLConnection connection = (HttpURLConnection) endPoint.openConnection();
+  private HttpResponse doSend(HttpRequest request) throws IOException {
+
+    HttpURLConnection connection = (HttpURLConnection) request.url.openConnection();
     connection.setConnectTimeout(CONNECTION_TIME_OUT);
     connection.setReadTimeout(readTimeout);
     connection.setDoOutput(true);
@@ -224,8 +227,8 @@ private BackendResponse doSend(BatchedLogRequest requestBody, String apiKey) thr
     connection.setRequestProperty(CONTENT_ENCODING_HEADER_KEY, GZIP_CONTENT_ENCODING);
     connection.setRequestProperty(CONTENT_TYPE_HEADER_KEY, PROTOBUF_CONTENT_TYPE);
 
-    if (apiKey != null) {
-      connection.setRequestProperty(API_KEY_HEADER_KEY, apiKey);
+    if (request.apiKey != null) {
+      connection.setRequestProperty(API_KEY_HEADER_KEY, request.apiKey);
     }
 
     WritableByteChannel channel = Channels.newChannel(connection.getOutputStream());
@@ -234,32 +237,29 @@ private BackendResponse doSend(BatchedLogRequest requestBody, String apiKey) thr
       GZIPOutputStream gzipOutputStream = new GZIPOutputStream(output);
 
       try {
-        requestBody.writeTo(gzipOutputStream);
+        request.requestBody.writeTo(gzipOutputStream);
       } finally {
         gzipOutputStream.close();
       }
       channel.write(ByteBuffer.wrap(output.toByteArray()));
       int responseCode = connection.getResponseCode();
       LOGGER.info("Status Code: " + responseCode);
 
-      long nextRequestMillis;
+      if (responseCode == 302 || responseCode == 301) {
+        String redirect = connection.getHeaderField("Location");
+        return new HttpResponse(responseCode, new URL(redirect), 0);
+      }
+      if (responseCode != 200) {
+        return new HttpResponse(responseCode, null, 0);
+      }
+
       InputStream inputStream = connection.getInputStream();
       try {
-        try {
-          nextRequestMillis = LogResponse.parseFrom(inputStream).getNextRequestWaitMillis();
-        } catch (InvalidProtocolBufferException e) {
-          return BackendResponse.fatalError();
-        }
+        long nextRequestMillis = LogResponse.parseFrom(inputStream).getNextRequestWaitMillis();
+        return new HttpResponse(responseCode, null, nextRequestMillis);
       } finally {
         inputStream.close();
       }
-      if (responseCode == 200) {
-        return BackendResponse.ok(nextRequestMillis);
-      } else if (responseCode >= 500 || responseCode == 404) {
-        return BackendResponse.transientError();
-      } else {
-        return BackendResponse.fatalError();
-      }
     } finally {
       channel.close();
     }
@@ -275,7 +275,27 @@ public BackendResponse send(BackendRequest request) {
         request.getExtras() == null ? null : LegacyFlgDestination.decodeExtras(request.getExtras());
 
     try {
-      return doSend(requestBody, apiKey);
+      HttpResponse response =
+          retry(
+              5,
+              new HttpRequest(endPoint, requestBody, apiKey),
+              this::doSend,
+              (req, resp) -> {
+                if (resp.redirectUrl != null) {
+                  // retry with different url
+                  return req.withUrl(resp.redirectUrl);
+                }
+                // don't retry
+                return null;
+              });
+
+      if (response.code == 200) {
+        return BackendResponse.ok(response.nextRequestMillis);
+      } else if (response.code >= 500 || response.code == 404) {
+        return BackendResponse.transientError();
+      } else {
+        return BackendResponse.fatalError();
+      }
     } catch (IOException e) {
       LOGGER.log(Level.SEVERE, "Could not make request to the backend", e);
       return BackendResponse.transientError();
@@ -288,4 +308,32 @@ static long getTzOffset() {
     TimeZone tz = TimeZone.getDefault();
     return tz.getOffset(Calendar.getInstance().getTimeInMillis()) / 1000;
   }
+
+  static final class HttpResponse {
+    final int code;
+    @Nullable final URL redirectUrl;
+    final long nextRequestMillis;
+
+    HttpResponse(int code, @Nullable URL redirectUrl, long nextRequestMillis) {
+      this.code = code;
+      this.redirectUrl = redirectUrl;
+      this.nextRequestMillis = nextRequestMillis;
+    }
+  }
+
+  static final class HttpRequest {
+    final URL url;
+    final BatchedLogRequest requestBody;
+    @Nullable final String apiKey;
+
+    HttpRequest(URL url, BatchedLogRequest requestBody, @Nullable String apiKey) {
+      this.url = url;
+      this.requestBody = requestBody;
+      this.apiKey = apiKey;
+    }
+
+    HttpRequest withUrl(URL newUrl) {
+      return new HttpRequest(newUrl, requestBody, apiKey);
+    }
+  }
 }

transport/transport-backend-cct/src/test/java/com/google/android/datatransport/cct/CctTransportBackendTest.java

@@ -237,7 +237,7 @@ public void testGarbageFromServer() {
     verify(
         postRequestedFor(urlEqualTo("/api"))
             .withHeader("Content-Type", equalTo("application/x-protobuf")));
-    assertEquals(BackendResponse.fatalError(), response);
+    assertEquals(BackendResponse.transientError(), response);
   }
 
   @Test
@@ -305,6 +305,69 @@ public void decorate_whenOffline_shouldProperlyPopulateNetworkInfo() {
             String.valueOf(NetworkConnectionInfo.MobileSubtype.UNKNOWN_MOBILE_SUBTYPE_VALUE));
   }
 
+  @Test
+  public void send_whenBackendRedirects_shouldCorrectlyFollowTheRedirectViaPost() {
+    stubFor(
+        post(urlEqualTo("/api"))
+            .willReturn(
+                aResponse().withStatus(302).withHeader("Location", TEST_ENDPOINT + "/hello")));
+    stubFor(
+        post(urlEqualTo("/api/hello"))
+            .willReturn(
+                aResponse()
+                    .withStatus(200)
+                    .withHeader("Content-Type", "application/x-protobuf;charset=UTF8;hello=world")
+                    .withBody(
+                        LogResponse.newBuilder()
+                            .setNextRequestWaitMillis(3)
+                            .build()
+                            .toByteArray())));
+    BackendRequest backendRequest = getCCTBackendRequest();
+    wallClock.tick();
+    uptimeClock.tick();
+
+    BackendResponse response = BACKEND.send(backendRequest);
+
+    verify(
+        postRequestedFor(urlEqualTo("/api"))
+            .withHeader("Content-Type", equalTo("application/x-protobuf")));
+
+    verify(
+        postRequestedFor(urlEqualTo("/api/hello"))
+            .withHeader("Content-Type", equalTo("application/x-protobuf")));
+
+    assertEquals(BackendResponse.ok(3), response);
+  }
+
+  @Test
+  public void send_whenBackendRedirectsMoreThan5Times_shouldOnlyRedirect4Times() {
+    stubFor(
+        post(urlEqualTo("/api"))
+            .willReturn(
+                aResponse().withStatus(302).withHeader("Location", TEST_ENDPOINT + "/hello")));
+    stubFor(
+        post(urlEqualTo("/api/hello"))
+            .willReturn(
+                aResponse().withStatus(302).withHeader("Location", TEST_ENDPOINT + "/hello")));
+
+    BackendRequest backendRequest = getCCTBackendRequest();
+    wallClock.tick();
+    uptimeClock.tick();
+
+    BackendResponse response = BACKEND.send(backendRequest);
+
+    verify(
+        postRequestedFor(urlEqualTo("/api"))
+            .withHeader("Content-Type", equalTo("application/x-protobuf")));
+
+    verify(
+        4,
+        postRequestedFor(urlEqualTo("/api/hello"))
+            .withHeader("Content-Type", equalTo("application/x-protobuf")));
+
+    assertEquals(BackendResponse.fatalError(), response);
+  }
+
   // When there is no active network, the ConnectivityManager returns null when
   // getActiveNetworkInfo() is called.
   @Implements(ConnectivityManager.class)

transport/transport-runtime/src/main/java/com/google/android/datatransport/runtime/retries/Function.java

@@ -0,0 +1,20 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.android.datatransport.runtime.retries;
+
+/** Generic functional interface which supports checked exceptions. */
+public interface Function<TInput, TResult, TException extends Throwable> {
+  TResult apply(TInput input) throws TException;
+}

transport/transport-runtime/src/main/java/com/google/android/datatransport/runtime/retries/Retries.java

@@ -0,0 +1,62 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.android.datatransport.runtime.retries;
+
+/** Provides utilities to retry function calls. */
+public final class Retries {
+  private Retries() {}
+
+  /**
+   * Retries given {@link Function} upto {@code maxAttempts} times.
+   *
+   * <p>It takes an {@code input} parameter that is passed to the first {@link Function} call. The
+   * rest of the retries are called with the value produced by the {@code retryStrategy}. If the
+   * {@code retryStrategy} returns null, the retries are stopped and the result of the last retry is
+   * returned.
+   *
+   * <p>Example
+   *
+   * <pre>{@code
+   * int initialParameter = 10;
+   *
+   * // finalResult is 12.
+   * int finalResult = retry(5, initialParameter, Integer::valueOf, (input, result) -> {
+   *   if ( result.equals(12)) {
+   *     return null;
+   *   }
+   *   return input + 1;
+   * });
+   * }</pre>
+   */
+  public static <TInput, TResult, TException extends Throwable> TResult retry(
+      int maxAttempts,
+      TInput input,
+      Function<TInput, TResult, TException> function,
+      RetryStrategy<TInput, TResult> retryStrategy)
+      throws TException {
+    if (maxAttempts < 1) {
+      return function.apply(input);
+    }
+
+    while (true) {
+      TResult result = function.apply(input);
+      input = retryStrategy.shouldRetry(input, result);
+
+      if (input == null || --maxAttempts < 1) {
+        return result;
+      }
+    }
+  }
+}

transport/transport-runtime/src/main/java/com/google/android/datatransport/runtime/retries/RetryStrategy.java

@@ -0,0 +1,28 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.android.datatransport.runtime.retries;
+
+import androidx.annotation.Nullable;
+
+public interface RetryStrategy<TInput, TResult> {
+  /**
+   * Whether a function call should be retried with a new input.
+   *
+   * <p>Given an input and result of a function call, return a new input to cause call to be
+   * retried, returning null will prevent further retries.
+   */
+  @Nullable
+  TInput shouldRetry(TInput input, TResult result);
+}

transport/transport-runtime/src/main/java/com/google/android/datatransport/runtime/retries/package-info.java

@@ -0,0 +1,16 @@
+// Copyright 2019 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/** @hide */
+package com.google.android.datatransport.runtime.retries;