Skip to content

Commit 07346d8

Browse files
authored
fix: wrap BatchRequest InputStream with BufferedInputStream (#1749)
If reading byte-by-byte from a GZipInputStream is slow, then wrapping in a BufferedInputStream should minimize this effect by reading in larger, buffered chunks. Added a test to ensure that parsing a large BatchResponse does not take an exorbitant amount of time. Fixes #1573 ☕️
1 parent 602cc24 commit 07346d8

File tree

2 files changed

+123
-1
lines changed

2 files changed

+123
-1
lines changed

google-api-client/src/main/java/com/google/api/client/googleapis/batch/BatchRequest.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import com.google.api.client.http.MultipartContent;
2828
import com.google.api.client.util.Preconditions;
2929
import com.google.api.client.util.Sleeper;
30+
import java.io.BufferedInputStream;
3031
import java.io.IOException;
3132
import java.io.InputStream;
3233
import java.util.ArrayList;
@@ -259,7 +260,7 @@ public void execute() throws IOException {
259260
String boundary = "--" + response.getMediaType().getParameter("boundary");
260261

261262
// Parse the content stream.
262-
InputStream contentStream = response.getContent();
263+
InputStream contentStream = new BufferedInputStream(response.getContent());
263264
batchResponse =
264265
new BatchUnparsedResponse(contentStream, boundary, requestInfos, retryAllowed);
265266

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
/*
2+
* Copyright 2021 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
5+
* in compliance with the License. You may obtain a copy of the License at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software distributed under the License
10+
* is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
11+
* or implied. See the License for the specific language governing permissions and limitations under
12+
* the License.
13+
*/
14+
15+
package com.google.api.client.googleapis.batch;
16+
17+
import static org.junit.Assert.assertEquals;
18+
19+
import com.google.api.client.http.ByteArrayContent;
20+
import com.google.api.client.http.GenericUrl;
21+
import com.google.api.client.http.HttpHeaders;
22+
import com.google.api.client.http.HttpMediaType;
23+
import com.google.api.client.http.HttpRequest;
24+
import com.google.api.client.http.HttpTransport;
25+
import com.google.api.client.http.LowLevelHttpRequest;
26+
import com.google.api.client.http.LowLevelHttpResponse;
27+
import com.google.api.client.testing.http.MockHttpTransport;
28+
import com.google.api.client.testing.http.MockLowLevelHttpRequest;
29+
import com.google.api.client.testing.http.MockLowLevelHttpResponse;
30+
import com.google.api.client.util.Charsets;
31+
import java.io.IOException;
32+
import java.io.PipedInputStream;
33+
import java.io.PipedOutputStream;
34+
import java.util.Arrays;
35+
import java.util.concurrent.atomic.AtomicInteger;
36+
import java.util.zip.GZIPOutputStream;
37+
import org.junit.BeforeClass;
38+
import org.junit.Test;
39+
40+
/** Tests {@link BatchRequest}. */
41+
public class BatchRequestStressTest {
42+
43+
private static final int BATCH_SIZE = 100;
44+
public static BatchRequest batchRequest;
45+
private static AtomicInteger parseCount = new AtomicInteger(0);
46+
private static AtomicInteger errorCount = new AtomicInteger(0);
47+
48+
@BeforeClass
49+
public static void setup() throws IOException {
50+
HttpTransport transport =
51+
new MockHttpTransport() {
52+
@Override
53+
public LowLevelHttpRequest buildRequest(String method, String url) throws IOException {
54+
return new MockLowLevelHttpRequest() {
55+
@Override
56+
public LowLevelHttpResponse execute() throws IOException {
57+
MockLowLevelHttpResponse response = new MockLowLevelHttpResponse();
58+
response.setStatusCode(200);
59+
response.addHeader("Content-Type", "multipart/mixed; boundary=__END_OF_PART__");
60+
response.setContentEncoding("gzip");
61+
StringBuilder responseContent = new StringBuilder();
62+
for (int i = 0; i < BATCH_SIZE; i++) {
63+
responseContent
64+
.append("--" + "__END_OF_PART__" + "\n")
65+
.append("Content-Type: application/http\n")
66+
.append("Content-ID: response-" + i + "\n\n")
67+
.append("HTTP/1.1 200 OK\n")
68+
.append("Content-Type: application/json\n\n")
69+
.append("{}\n\n");
70+
}
71+
responseContent.append("--" + "__END_OF_PART__--" + "\n\n");
72+
73+
// gzip this content
74+
PipedInputStream is = new PipedInputStream();
75+
PipedOutputStream os = new PipedOutputStream(is);
76+
GZIPOutputStream gzip = new GZIPOutputStream(os);
77+
gzip.write(responseContent.toString().getBytes("UTF-8"));
78+
gzip.close();
79+
response.setContent(is);
80+
return response;
81+
}
82+
};
83+
}
84+
};
85+
86+
BatchCallback<Void, Void> callback =
87+
new BatchCallback<Void, Void>() {
88+
@Override
89+
public void onSuccess(Void t, HttpHeaders responseHeaders) {
90+
parseCount.incrementAndGet();
91+
}
92+
93+
@Override
94+
public void onFailure(Void e, HttpHeaders responseHeaders) {
95+
errorCount.incrementAndGet();
96+
}
97+
};
98+
batchRequest = new BatchRequest(transport, null);
99+
byte[] content = new byte[300];
100+
Arrays.fill(content, (byte) ' ');
101+
HttpRequest request1 =
102+
transport
103+
.createRequestFactory()
104+
.buildRequest(
105+
"POST",
106+
new GenericUrl("http://www.google.com/"),
107+
new ByteArrayContent(
108+
new HttpMediaType("text/plain").setCharsetParameter(Charsets.UTF_8).build(),
109+
content));
110+
for (int i = 0; i < BATCH_SIZE; i++) {
111+
batchRequest.queue(request1, Void.class, Void.class, callback);
112+
}
113+
}
114+
115+
@Test(timeout = 4000)
116+
public void testResponse() throws IOException {
117+
batchRequest.execute();
118+
assertEquals(BATCH_SIZE, parseCount.get());
119+
assertEquals(0, errorCount.get());
120+
}
121+
}

0 commit comments

Comments
 (0)