Skip to content

fix: Enabled automatic HTTP retries for FirebaseProjectManagement #356

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Feb 10, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 15 additions & 3 deletions src/main/java/com/google/firebase/internal/ApiClientUtils.java
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
*/
public class ApiClientUtils {

private static final RetryConfig DEFAULT_RETRY_CONFIG = RetryConfig.builder()
static final RetryConfig DEFAULT_RETRY_CONFIG = RetryConfig.builder()
.setMaxRetries(4)
.setRetryStatusCodes(ImmutableList.of(500, 503))
.setMaxIntervalMillis(60 * 1000)
Expand All @@ -43,9 +43,21 @@ public class ApiClientUtils {
* @return A new {@code HttpRequestFactory} instance.
*/
public static HttpRequestFactory newAuthorizedRequestFactory(FirebaseApp app) {
return newAuthorizedRequestFactory(app, DEFAULT_RETRY_CONFIG);
}

/**
* Creates a new {@code HttpRequestFactory} which provides authorization (OAuth2), timeouts and
* automatic retries.
*
* @param app {@link FirebaseApp} from which to obtain authorization credentials.
* @param retryConfig {@link RetryConfig} instance or null to disable retries.
* @return A new {@code HttpRequestFactory} instance.
*/
public static HttpRequestFactory newAuthorizedRequestFactory(
FirebaseApp app, @Nullable RetryConfig retryConfig) {
HttpTransport transport = app.getOptions().getHttpTransport();
return transport.createRequestFactory(
new FirebaseRequestInitializer(app, DEFAULT_RETRY_CONFIG));
return transport.createRequestFactory(new FirebaseRequestInitializer(app, retryConfig));
}

public static HttpRequestFactory newUnauthorizedRequestFactory(FirebaseApp app) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;

import com.google.api.client.http.HttpRequestFactory;
import com.google.api.client.http.HttpResponseInterceptor;
import com.google.api.client.util.Base64;
import com.google.api.client.util.Key;
Expand All @@ -32,8 +33,8 @@
import com.google.common.collect.ImmutableMap;
import com.google.firebase.FirebaseApp;
import com.google.firebase.ImplFirebaseTrampolines;
import com.google.firebase.internal.ApiClientUtils;
import com.google.firebase.internal.CallableOperation;
import com.google.firebase.internal.FirebaseRequestInitializer;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
Expand All @@ -55,6 +56,7 @@ class FirebaseProjectManagementServiceImpl implements AndroidAppService, IosAppS
private final FirebaseApp app;
private final Sleeper sleeper;
private final Scheduler scheduler;
private final HttpRequestFactory requestFactory;
private final HttpHelper httpHelper;

private final CreateAndroidAppFromAppIdFunction createAndroidAppFromAppIdFunction =
Expand All @@ -63,17 +65,26 @@ class FirebaseProjectManagementServiceImpl implements AndroidAppService, IosAppS
new CreateIosAppFromAppIdFunction();

FirebaseProjectManagementServiceImpl(FirebaseApp app) {
this(app, Sleeper.DEFAULT, new FirebaseAppScheduler(app));
this(
app,
Sleeper.DEFAULT,
new FirebaseAppScheduler(app),
ApiClientUtils.newAuthorizedRequestFactory(app));
}

FirebaseProjectManagementServiceImpl(FirebaseApp app, Sleeper sleeper, Scheduler scheduler) {
@VisibleForTesting
FirebaseProjectManagementServiceImpl(
FirebaseApp app, Sleeper sleeper, Scheduler scheduler, HttpRequestFactory requestFactory) {
this.app = checkNotNull(app);
this.sleeper = checkNotNull(sleeper);
this.scheduler = checkNotNull(scheduler);
this.httpHelper = new HttpHelper(
app.getOptions().getJsonFactory(),
app.getOptions().getHttpTransport().createRequestFactory(
new FirebaseRequestInitializer(app)));
this.requestFactory = checkNotNull(requestFactory);
this.httpHelper = new HttpHelper(app.getOptions().getJsonFactory(), requestFactory);
}

@VisibleForTesting
HttpRequestFactory getRequestFactory() {
return requestFactory;
}

@VisibleForTesting
Expand Down
13 changes: 13 additions & 0 deletions src/test/java/com/google/firebase/internal/ApiClientUtilsTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,19 @@ public void testAuthorizedHttpClient() throws IOException {
assertEquals(retryConfig.getRetryStatusCodes(), ImmutableList.of(500, 503));
}

@Test
public void testAuthorizedHttpClientWithoutRetry() throws IOException {
FirebaseApp app = FirebaseApp.initializeApp(TEST_OPTIONS);

HttpRequestFactory requestFactory = ApiClientUtils.newAuthorizedRequestFactory(app, null);

assertTrue(requestFactory.getInitializer() instanceof FirebaseRequestInitializer);
HttpRequest request = requestFactory.buildGetRequest(TEST_URL);
assertEquals("Bearer test-token", request.getHeaders().getAuthorization());
HttpUnsuccessfulResponseHandler retryHandler = request.getUnsuccessfulResponseHandler();
assertFalse(retryHandler instanceof RetryHandlerDecorator);
}

@Test
public void testUnauthorizedHttpClient() throws IOException {
FirebaseApp app = FirebaseApp.initializeApp(TEST_OPTIONS);
Expand Down
95 changes: 95 additions & 0 deletions src/test/java/com/google/firebase/internal/TestApiClientUtils.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
/*
* Copyright 2020 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.google.firebase.internal;

import static com.google.firebase.internal.ApiClientUtils.DEFAULT_RETRY_CONFIG;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;

import com.google.api.client.http.GenericUrl;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestFactory;
import com.google.api.client.http.HttpUnsuccessfulResponseHandler;
import com.google.api.client.testing.util.MockSleeper;
import com.google.firebase.FirebaseApp;
import com.google.firebase.internal.RetryInitializer.RetryHandlerDecorator;
import java.io.IOException;

public class TestApiClientUtils {

private static final RetryConfig TEST_RETRY_CONFIG = RetryConfig.builder()
.setMaxRetries(DEFAULT_RETRY_CONFIG.getMaxRetries())
.setRetryStatusCodes(DEFAULT_RETRY_CONFIG.getRetryStatusCodes())
.setMaxIntervalMillis(DEFAULT_RETRY_CONFIG.getMaxIntervalMillis())
.setSleeper(new MockSleeper())
.build();

private static final GenericUrl TEST_URL = new GenericUrl("https://firebase.google.com");

/**
* Creates a new {@code HttpRequestFactory} which provides authorization (OAuth2), timeouts and
* automatic retries. Bypasses exponential backoff between consecutive retries for faster
* execution during tests.
*
* @param app {@link FirebaseApp} from which to obtain authorization credentials.
* @return A new {@code HttpRequestFactory} instance.
*/
public static HttpRequestFactory delayBypassedRequestFactory(FirebaseApp app) {
return ApiClientUtils.newAuthorizedRequestFactory(app, TEST_RETRY_CONFIG);
}

/**
* Creates a new {@code HttpRequestFactory} which provides authorization (OAuth2), timeouts but
* no retries.
*
* @param app {@link FirebaseApp} from which to obtain authorization credentials.
* @return A new {@code HttpRequestFactory} instance.
*/
public static HttpRequestFactory retryDisabledRequestFactory(FirebaseApp app) {
return ApiClientUtils.newAuthorizedRequestFactory(app, null);
}

/**
* Checks whther the given HttpRequestFactory has been configured for authorization and
* automatic retries.
*
* @param requestFactory The HttpRequestFactory to check.
*/
public static void assertAuthAndRetrySupport(HttpRequestFactory requestFactory) {
assertTrue(requestFactory.getInitializer() instanceof FirebaseRequestInitializer);
HttpRequest request;
try {
request = requestFactory.buildGetRequest(TEST_URL);
} catch (IOException e) {
throw new RuntimeException("Failed to initialize request", e);
}

// Verify authorization
assertTrue(request.getHeaders().getAuthorization().startsWith("Bearer "));

// Verify retry support
HttpUnsuccessfulResponseHandler retryHandler = request.getUnsuccessfulResponseHandler();
assertTrue(retryHandler instanceof RetryHandlerDecorator);
Copy link
Contributor

@weixifan weixifan Feb 8, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

At first I kind of want to say we should bake this assertion into the API infrastructure itself instead of directly verifying in a unit test that no one touched the code. But then the OO design I came up with to facilitate this turns out to be impossible due to various classes being final. I was hoping that we can write a subclass of HttpRequest called RetryingHttpRequest that declares at compile time that getUnsuccessfulResponseHandler will return RetryHandlerDecorator or a suitable base class, but alas, HttpRequest is final. (And so is HttpRequestFactory, which is the other class I wanted to subclass.)

Without the above OO structure, the only other way to structure the test to be more behavioural would be to treat the retrying mechanism completely like a black box, and empirically measure retry intervals and retry counts after forcing an unsuccessful response. This is going to be painfully flaky, so I think what you have is probably for the best.

RetryConfig retryConfig = ((RetryHandlerDecorator) retryHandler).getRetryHandler()
.getRetryConfig();
assertEquals(DEFAULT_RETRY_CONFIG.getMaxRetries(), retryConfig.getMaxRetries());
assertEquals(DEFAULT_RETRY_CONFIG.getMaxIntervalMillis(), retryConfig.getMaxIntervalMillis());
assertFalse(retryConfig.isRetryOnIOExceptions());
assertEquals(DEFAULT_RETRY_CONFIG.getRetryStatusCodes(), retryConfig.getRetryStatusCodes());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@

import com.google.api.client.googleapis.util.Utils;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestFactory;
import com.google.api.client.http.HttpResponse;
import com.google.api.client.http.HttpResponseInterceptor;
import com.google.api.client.json.JsonParser;
Expand All @@ -43,6 +44,7 @@
import com.google.firebase.FirebaseOptions;
import com.google.firebase.TestOnlyImplFirebaseTrampolines;
import com.google.firebase.auth.MockGoogleCredentials;
import com.google.firebase.internal.TestApiClientUtils;
import com.google.firebase.testing.MultiRequestMockHttpTransport;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
Expand Down Expand Up @@ -370,7 +372,7 @@ public void listIosAppsAsyncMultiplePages() throws Exception {
MockLowLevelHttpResponse secondRpcResponse = new MockLowLevelHttpResponse();
secondRpcResponse.setContent(LIST_IOS_APPS_PAGE_2_RESPONSE);
serviceImpl = initServiceImpl(
ImmutableList.<MockLowLevelHttpResponse>of(firstRpcResponse, secondRpcResponse),
ImmutableList.of(firstRpcResponse, secondRpcResponse),
interceptor);

List<IosApp> iosAppList = serviceImpl.listIosAppsAsync(PROJECT_ID).get();
Expand Down Expand Up @@ -400,7 +402,7 @@ public void createIosApp() throws Exception {
MockLowLevelHttpResponse thirdRpcResponse = new MockLowLevelHttpResponse();
thirdRpcResponse.setContent(CREATE_IOS_GET_OPERATION_ATTEMPT_2_RESPONSE);
serviceImpl = initServiceImpl(
ImmutableList.<MockLowLevelHttpResponse>of(
ImmutableList.of(
firstRpcResponse, secondRpcResponse, thirdRpcResponse),
interceptor);

Expand Down Expand Up @@ -624,7 +626,7 @@ public void listAndroidAppsMultiplePages() throws Exception {
MockLowLevelHttpResponse secondRpcResponse = new MockLowLevelHttpResponse();
secondRpcResponse.setContent(LIST_ANDROID_APPS_PAGE_2_RESPONSE);
serviceImpl = initServiceImpl(
ImmutableList.<MockLowLevelHttpResponse>of(firstRpcResponse, secondRpcResponse),
ImmutableList.of(firstRpcResponse, secondRpcResponse),
interceptor);

List<AndroidApp> androidAppList = serviceImpl.listAndroidApps(PROJECT_ID);
Expand Down Expand Up @@ -652,7 +654,7 @@ public void listAndroidAppsAsyncMultiplePages() throws Exception {
MockLowLevelHttpResponse secondRpcResponse = new MockLowLevelHttpResponse();
secondRpcResponse.setContent(LIST_ANDROID_APPS_PAGE_2_RESPONSE);
serviceImpl = initServiceImpl(
ImmutableList.<MockLowLevelHttpResponse>of(firstRpcResponse, secondRpcResponse),
ImmutableList.of(firstRpcResponse, secondRpcResponse),
interceptor);

List<AndroidApp> androidAppList = serviceImpl.listAndroidAppsAsync(PROJECT_ID).get();
Expand Down Expand Up @@ -682,7 +684,7 @@ public void createAndroidApp() throws Exception {
MockLowLevelHttpResponse thirdRpcResponse = new MockLowLevelHttpResponse();
thirdRpcResponse.setContent(CREATE_ANDROID_GET_OPERATION_ATTEMPT_2_RESPONSE);
serviceImpl = initServiceImpl(
ImmutableList.<MockLowLevelHttpResponse>of(
ImmutableList.of(
firstRpcResponse, secondRpcResponse, thirdRpcResponse),
interceptor);

Expand Down Expand Up @@ -714,7 +716,7 @@ public void createAndroidAppAsync() throws Exception {
MockLowLevelHttpResponse thirdRpcResponse = new MockLowLevelHttpResponse();
thirdRpcResponse.setContent(CREATE_ANDROID_GET_OPERATION_ATTEMPT_2_RESPONSE);
serviceImpl = initServiceImpl(
ImmutableList.<MockLowLevelHttpResponse>of(
ImmutableList.of(
firstRpcResponse, secondRpcResponse, thirdRpcResponse),
interceptor);

Expand Down Expand Up @@ -915,10 +917,48 @@ public void deleteShaCertificateAsync() throws Exception {
checkRequestHeader(expectedUrl, HttpMethod.DELETE);
}

@Test
public void testAuthAndRetriesSupport() {
FirebaseOptions options = new FirebaseOptions.Builder()
.setCredentials(new MockGoogleCredentials("test-token"))
.setProjectId(PROJECT_ID)
.build();
FirebaseApp app = FirebaseApp.initializeApp(options);

FirebaseProjectManagementServiceImpl serviceImpl =
new FirebaseProjectManagementServiceImpl(app);

TestApiClientUtils.assertAuthAndRetrySupport(serviceImpl.getRequestFactory());
}

@Test
public void testHttpRetries() throws Exception {
List<MockLowLevelHttpResponse> mockResponses = ImmutableList.of(
firstRpcResponse.setStatusCode(503).setContent("{}"),
new MockLowLevelHttpResponse().setContent("{}"));
MockHttpTransport transport = new MultiRequestMockHttpTransport(mockResponses);
FirebaseOptions options = new FirebaseOptions.Builder()
.setCredentials(new MockGoogleCredentials("test-token"))
.setProjectId(PROJECT_ID)
.setHttpTransport(transport)
.build();
FirebaseApp app = FirebaseApp.initializeApp(options);
HttpRequestFactory requestFactory = TestApiClientUtils.delayBypassedRequestFactory(app);
FirebaseProjectManagementServiceImpl serviceImpl = new FirebaseProjectManagementServiceImpl(
app, new MockSleeper(), new MockScheduler(), requestFactory);
serviceImpl.setInterceptor(interceptor);

serviceImpl.deleteShaCertificate(SHA1_RESOURCE_NAME);

String expectedUrl = String.format(
"%s/v1beta1/%s", FIREBASE_PROJECT_MANAGEMENT_URL, SHA1_RESOURCE_NAME);
checkRequestHeader(expectedUrl, HttpMethod.DELETE);
}

private static FirebaseProjectManagementServiceImpl initServiceImpl(
MockLowLevelHttpResponse mockResponse,
MultiRequestTestResponseInterceptor interceptor) {
return initServiceImpl(ImmutableList.<MockLowLevelHttpResponse>of(mockResponse), interceptor);
return initServiceImpl(ImmutableList.of(mockResponse), interceptor);
}

private static FirebaseProjectManagementServiceImpl initServiceImpl(
Expand All @@ -931,8 +971,9 @@ private static FirebaseProjectManagementServiceImpl initServiceImpl(
.setHttpTransport(transport)
.build();
FirebaseApp app = FirebaseApp.initializeApp(options);
FirebaseProjectManagementServiceImpl serviceImpl =
new FirebaseProjectManagementServiceImpl(app, new MockSleeper(), new MockScheduler());
HttpRequestFactory requestFactory = TestApiClientUtils.retryDisabledRequestFactory(app);
FirebaseProjectManagementServiceImpl serviceImpl = new FirebaseProjectManagementServiceImpl(
app, new MockSleeper(), new MockScheduler(), requestFactory);
serviceImpl.setInterceptor(interceptor);
return serviceImpl;
}
Expand Down