Skip to content

Commit bc7fda3

Browse files
dgray16dgray16
authored
[bq] describe how to run batch query
1 parent 56f3fc6 commit bc7fda3

File tree

8 files changed

+239
-32
lines changed

8 files changed

+239
-32
lines changed

‎spring-batch-bigquery/src/main/java/org/springframework/batch/extensions/bigquery/reader/BigQueryInteractiveQueryItemReader.java renamed to ‎spring-batch-bigquery/src/main/java/org/springframework/batch/extensions/bigquery/reader/BigQueryQueryItemReader.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,10 @@
4343
* @author Volodymyr Perebykivskyi
4444
* @since 0.2.0
4545
* @see <a href="https://cloud.google.com/bigquery/docs/running-queries#queries">Interactive queries</a>
46+
* @see <a href="https://cloud.google.com/bigquery/docs/running-queries#batch">Batch queries</a>
4647
* @see <a href="https://cloud.google.com/bigquery/quotas#concurrent_rate_interactive_queries">Concurrency limits</a>
4748
*/
48-
public class BigQueryInteractiveQueryItemReader<T> implements ItemReader<T>, InitializingBean {
49+
public class BigQueryQueryItemReader<T> implements ItemReader<T>, InitializingBean {
4950

5051
private final Log logger = LogFactory.getLog(getClass());
5152

‎spring-batch-bigquery/src/main/java/org/springframework/batch/extensions/bigquery/reader/builder/BigQueryInteractiveQueryItemReaderBuilder.java renamed to ‎spring-batch-bigquery/src/main/java/org/springframework/batch/extensions/bigquery/reader/builder/BigQueryQueryItemReaderBuilder.java

Lines changed: 20 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -20,21 +20,22 @@
2020
import com.google.cloud.bigquery.FieldValueList;
2121
import com.google.cloud.bigquery.QueryJobConfiguration;
2222
import org.apache.commons.lang3.StringUtils;
23-
import org.springframework.batch.extensions.bigquery.reader.BigQueryInteractiveQueryItemReader;
23+
import org.springframework.batch.extensions.bigquery.reader.BigQueryQueryItemReader;
2424
import org.springframework.core.convert.converter.Converter;
2525
import org.springframework.util.Assert;
2626

2727
import java.util.Objects;
2828

2929
/**
30-
* A builder for {@link BigQueryInteractiveQueryItemReader}.
30+
* A builder for {@link BigQueryQueryItemReader}.
3131
*
3232
* @param <T> your DTO type
3333
* @author Volodymyr Perebykivskyi
3434
* @since 0.2.0
3535
* @see <a href="https://github.com/spring-projects/spring-batch-extensions/tree/main/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/reader/builder/BigQueryInteractiveQueryItemReaderBuilderTests.java">Examples</a>
36+
* @see <a href="https://github.com/spring-projects/spring-batch-extensions/tree/main/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/reader/builder/BigQueryBatchQueryItemReaderBuilderTests.java">Examples</a>
3637
*/
37-
public class BigQueryInteractiveQueryItemReaderBuilder<T> {
38+
public class BigQueryQueryItemReaderBuilder<T> {
3839

3940
private BigQuery bigQuery;
4041
private String query;
@@ -45,10 +46,10 @@ public class BigQueryInteractiveQueryItemReaderBuilder<T> {
4546
* BigQuery service, responsible for API calls.
4647
*
4748
* @param bigQuery BigQuery service
48-
* @return {@link BigQueryInteractiveQueryItemReaderBuilder}
49-
* @see BigQueryInteractiveQueryItemReader#setBigQuery(BigQuery)
49+
* @return {@link BigQueryQueryItemReaderBuilder}
50+
* @see BigQueryQueryItemReader#setBigQuery(BigQuery)
5051
*/
51-
public BigQueryInteractiveQueryItemReaderBuilder<T> bigQuery(BigQuery bigQuery) {
52+
public BigQueryQueryItemReaderBuilder<T> bigQuery(BigQuery bigQuery) {
5253
this.bigQuery = bigQuery;
5354
return this;
5455
}
@@ -60,10 +61,10 @@ public BigQueryInteractiveQueryItemReaderBuilder<T> bigQuery(BigQuery bigQuery)
6061
* because BigQuery charges you for the amount of data that is being processed.
6162
*
6263
* @param query your query to run
63-
* @return {@link BigQueryInteractiveQueryItemReaderBuilder}
64-
* @see BigQueryInteractiveQueryItemReader#setJobConfiguration(QueryJobConfiguration)
64+
* @return {@link BigQueryQueryItemReaderBuilder}
65+
* @see BigQueryQueryItemReader#setJobConfiguration(QueryJobConfiguration)
6566
*/
66-
public BigQueryInteractiveQueryItemReaderBuilder<T> query(String query) {
67+
public BigQueryQueryItemReaderBuilder<T> query(String query) {
6768
this.query = query;
6869
return this;
6970
}
@@ -72,10 +73,10 @@ public BigQueryInteractiveQueryItemReaderBuilder<T> query(String query) {
7273
* Row mapper which transforms single BigQuery row into desired type.
7374
*
7475
* @param rowMapper your row mapper
75-
* @return {@link BigQueryInteractiveQueryItemReaderBuilder}
76-
* @see BigQueryInteractiveQueryItemReader#setRowMapper(Converter)
76+
* @return {@link BigQueryQueryItemReaderBuilder}
77+
* @see BigQueryQueryItemReader#setRowMapper(Converter)
7778
*/
78-
public BigQueryInteractiveQueryItemReaderBuilder<T> rowMapper(Converter<FieldValueList, T> rowMapper) {
79+
public BigQueryQueryItemReaderBuilder<T> rowMapper(Converter<FieldValueList, T> rowMapper) {
7980
this.rowMapper = rowMapper;
8081
return this;
8182
}
@@ -84,21 +85,21 @@ public BigQueryInteractiveQueryItemReaderBuilder<T> rowMapper(Converter<FieldVal
8485
* Specifies query to run, destination table, etc.
8586
*
8687
* @param jobConfiguration BigQuery job configuration
87-
* @return {@link BigQueryInteractiveQueryItemReaderBuilder}
88-
* @see BigQueryInteractiveQueryItemReader#setJobConfiguration(QueryJobConfiguration)
88+
* @return {@link BigQueryQueryItemReaderBuilder}
89+
* @see BigQueryQueryItemReader#setJobConfiguration(QueryJobConfiguration)
8990
*/
90-
public BigQueryInteractiveQueryItemReaderBuilder<T> jobConfiguration(QueryJobConfiguration jobConfiguration) {
91+
public BigQueryQueryItemReaderBuilder<T> jobConfiguration(QueryJobConfiguration jobConfiguration) {
9192
this.jobConfiguration = jobConfiguration;
9293
return this;
9394
}
9495

9596
/**
96-
* Please do not forget about {@link BigQueryInteractiveQueryItemReader#afterPropertiesSet()}.
97+
* Please do not forget about {@link BigQueryQueryItemReader#afterPropertiesSet()}.
9798
*
98-
* @return {@link BigQueryInteractiveQueryItemReader}
99+
* @return {@link BigQueryQueryItemReader}
99100
*/
100-
public BigQueryInteractiveQueryItemReader<T> build() {
101-
BigQueryInteractiveQueryItemReader<T> reader = new BigQueryInteractiveQueryItemReader<>();
101+
public BigQueryQueryItemReader<T> build() {
102+
BigQueryQueryItemReader<T> reader = new BigQueryQueryItemReader<>();
102103

103104
reader.setBigQuery(this.bigQuery);
104105
reader.setRowMapper(this.rowMapper);

‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/reader/batch/BigQueryBatchQueryCsvItemReaderTest.java

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
/*
2+
* Copyright 2002-2023 the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.springframework.batch.extensions.bigquery.integration.reader.batch;
18+
19+
import com.google.cloud.bigquery.QueryJobConfiguration;
20+
import com.google.cloud.bigquery.TableId;
21+
import org.apache.commons.lang3.math.NumberUtils;
22+
import org.junit.jupiter.api.Assertions;
23+
import org.junit.jupiter.api.Tag;
24+
import org.junit.jupiter.api.Test;
25+
import org.junit.jupiter.api.TestInfo;
26+
import org.springframework.batch.extensions.bigquery.common.BigQueryDataLoader;
27+
import org.springframework.batch.extensions.bigquery.common.PersonDto;
28+
import org.springframework.batch.extensions.bigquery.common.TestConstants;
29+
import org.springframework.batch.extensions.bigquery.integration.reader.base.BaseCsvJsonInteractiveQueryItemReaderTest;
30+
import org.springframework.batch.extensions.bigquery.reader.BigQueryQueryItemReader;
31+
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQueryItemReaderBuilder;
32+
import org.springframework.batch.item.Chunk;
33+
34+
@Tag("csv")
35+
public class BigQueryBatchQueryCsvItemReaderTest extends BaseCsvJsonInteractiveQueryItemReaderTest {
36+
37+
@Test
38+
void batchQueryTest1(TestInfo testInfo) throws Exception {
39+
String tableName = getTableName(testInfo);
40+
new BigQueryDataLoader(bigQuery).loadCsvSample(tableName);
41+
Chunk<PersonDto> chunk = BigQueryDataLoader.CHUNK;
42+
43+
QueryJobConfiguration jobConfiguration = QueryJobConfiguration
44+
.newBuilder("SELECT p.name, p.age FROM spring_batch_extensions.%s p ORDER BY p.name LIMIT 2")
45+
.setDestinationTable(TableId.of(TestConstants.DATASET, tableName))
46+
.setPriority(QueryJobConfiguration.Priority.BATCH)
47+
.build();
48+
49+
BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
50+
.bigQuery(bigQuery)
51+
.rowMapper(TestConstants.PERSON_MAPPER)
52+
.jobConfiguration(jobConfiguration)
53+
.build();
54+
55+
reader.afterPropertiesSet();
56+
57+
PersonDto actualFirstPerson = reader.read();
58+
PersonDto expectedFirstPerson = chunk.getItems().get(0);
59+
60+
PersonDto actualSecondPerson = reader.read();
61+
PersonDto expectedSecondPerson = chunk.getItems().get(1);
62+
63+
PersonDto actualThirdPerson = reader.read();
64+
65+
Assertions.assertNotNull(actualFirstPerson);
66+
Assertions.assertEquals(expectedFirstPerson.name(), actualFirstPerson.name());
67+
Assertions.assertEquals(expectedFirstPerson.age().compareTo(actualFirstPerson.age()), NumberUtils.INTEGER_ZERO);
68+
69+
Assertions.assertNotNull(actualSecondPerson);
70+
Assertions.assertEquals(expectedSecondPerson.name(), actualSecondPerson.name());
71+
Assertions.assertEquals(expectedSecondPerson.age().compareTo(actualSecondPerson.age()), NumberUtils.INTEGER_ZERO);
72+
73+
Assertions.assertNull(actualThirdPerson);
74+
}
75+
76+
}

‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/reader/batch/BigQueryBatchQueryJsonItemReaderTest.java

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
/*
2+
* Copyright 2002-2023 the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.springframework.batch.extensions.bigquery.integration.reader.batch;
18+
19+
import com.google.cloud.bigquery.QueryJobConfiguration;
20+
import com.google.cloud.bigquery.TableId;
21+
import org.apache.commons.lang3.math.NumberUtils;
22+
import org.junit.jupiter.api.Assertions;
23+
import org.junit.jupiter.api.Tag;
24+
import org.junit.jupiter.api.Test;
25+
import org.junit.jupiter.api.TestInfo;
26+
import org.springframework.batch.extensions.bigquery.common.BigQueryDataLoader;
27+
import org.springframework.batch.extensions.bigquery.common.PersonDto;
28+
import org.springframework.batch.extensions.bigquery.common.TestConstants;
29+
import org.springframework.batch.extensions.bigquery.integration.reader.base.BaseCsvJsonInteractiveQueryItemReaderTest;
30+
import org.springframework.batch.extensions.bigquery.reader.BigQueryQueryItemReader;
31+
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQueryItemReaderBuilder;
32+
import org.springframework.batch.item.Chunk;
33+
34+
@Tag("json")
35+
public class BigQueryBatchQueryJsonItemReaderTest extends BaseCsvJsonInteractiveQueryItemReaderTest {
36+
37+
@Test
38+
void batchQueryTest1(TestInfo testInfo) throws Exception {
39+
String tableName = getTableName(testInfo);
40+
new BigQueryDataLoader(bigQuery).loadJsonSample(tableName);
41+
Chunk<PersonDto> chunk = BigQueryDataLoader.CHUNK;
42+
43+
QueryJobConfiguration jobConfiguration = QueryJobConfiguration
44+
.newBuilder("SELECT p.name, p.age FROM spring_batch_extensions.%s p ORDER BY p.name LIMIT 2")
45+
.setDestinationTable(TableId.of(TestConstants.DATASET, tableName))
46+
.setPriority(QueryJobConfiguration.Priority.BATCH)
47+
.build();
48+
49+
BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
50+
.bigQuery(bigQuery)
51+
.rowMapper(TestConstants.PERSON_MAPPER)
52+
.jobConfiguration(jobConfiguration)
53+
.build();
54+
55+
reader.afterPropertiesSet();
56+
57+
PersonDto actualFirstPerson = reader.read();
58+
PersonDto expectedFirstPerson = chunk.getItems().get(0);
59+
60+
PersonDto actualSecondPerson = reader.read();
61+
PersonDto expectedSecondPerson = chunk.getItems().get(1);
62+
63+
PersonDto actualThirdPerson = reader.read();
64+
65+
Assertions.assertNotNull(actualFirstPerson);
66+
Assertions.assertEquals(expectedFirstPerson.name(), actualFirstPerson.name());
67+
Assertions.assertEquals(expectedFirstPerson.age().compareTo(actualFirstPerson.age()), NumberUtils.INTEGER_ZERO);
68+
69+
Assertions.assertNotNull(actualSecondPerson);
70+
Assertions.assertEquals(expectedSecondPerson.name(), actualSecondPerson.name());
71+
Assertions.assertEquals(expectedSecondPerson.age().compareTo(actualSecondPerson.age()), NumberUtils.INTEGER_ZERO);
72+
73+
Assertions.assertNull(actualThirdPerson);
74+
}
75+
76+
}

‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/reader/BigQueryInteractiveQueryCsvItemReaderTest.java renamed to ‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/reader/interactive/BigQueryInteractiveQueryCsvItemReaderTest.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
* limitations under the License.
1515
*/
1616

17-
package org.springframework.batch.extensions.bigquery.integration.reader;
17+
package org.springframework.batch.extensions.bigquery.integration.reader.interactive;
1818

1919
import org.apache.commons.lang3.math.NumberUtils;
2020
import org.junit.jupiter.api.Assertions;
@@ -25,8 +25,8 @@
2525
import org.springframework.batch.extensions.bigquery.common.PersonDto;
2626
import org.springframework.batch.extensions.bigquery.common.TestConstants;
2727
import org.springframework.batch.extensions.bigquery.integration.reader.base.BaseCsvJsonInteractiveQueryItemReaderTest;
28-
import org.springframework.batch.extensions.bigquery.reader.BigQueryInteractiveQueryItemReader;
29-
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryInteractiveQueryItemReaderBuilder;
28+
import org.springframework.batch.extensions.bigquery.reader.BigQueryQueryItemReader;
29+
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQueryItemReaderBuilder;
3030
import org.springframework.batch.item.Chunk;
3131

3232
@Tag("csv")
@@ -38,7 +38,7 @@ void interactiveQueryTest1(TestInfo testInfo) throws Exception {
3838
new BigQueryDataLoader(bigQuery).loadCsvSample(tableName);
3939
Chunk<PersonDto> chunk = BigQueryDataLoader.CHUNK;
4040

41-
BigQueryInteractiveQueryItemReader<PersonDto> reader = new BigQueryInteractiveQueryItemReaderBuilder<PersonDto>()
41+
BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
4242
.bigQuery(bigQuery)
4343
.query(String.format("SELECT p.name, p.age FROM spring_batch_extensions.%s p ORDER BY p.name LIMIT 2", tableName))
4444
.rowMapper(TestConstants.PERSON_MAPPER)

‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/reader/BigQueryInteractiveQueryJsonItemReaderTest.java renamed to ‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/reader/interactive/BigQueryInteractiveQueryJsonItemReaderTest.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
* limitations under the License.
1515
*/
1616

17-
package org.springframework.batch.extensions.bigquery.integration.reader;
17+
package org.springframework.batch.extensions.bigquery.integration.reader.interactive;
1818

1919
import org.apache.commons.lang3.math.NumberUtils;
2020
import org.junit.jupiter.api.Assertions;
@@ -25,8 +25,8 @@
2525
import org.springframework.batch.extensions.bigquery.common.PersonDto;
2626
import org.springframework.batch.extensions.bigquery.common.TestConstants;
2727
import org.springframework.batch.extensions.bigquery.integration.reader.base.BaseCsvJsonInteractiveQueryItemReaderTest;
28-
import org.springframework.batch.extensions.bigquery.reader.BigQueryInteractiveQueryItemReader;
29-
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryInteractiveQueryItemReaderBuilder;
28+
import org.springframework.batch.extensions.bigquery.reader.BigQueryQueryItemReader;
29+
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQueryItemReaderBuilder;
3030
import org.springframework.batch.item.Chunk;
3131

3232
@Tag("json")
@@ -38,7 +38,7 @@ void interactiveQueryTest1(TestInfo testInfo) throws Exception {
3838
new BigQueryDataLoader(bigQuery).loadJsonSample(tableName);
3939
Chunk<PersonDto> chunk = BigQueryDataLoader.CHUNK;
4040

41-
BigQueryInteractiveQueryItemReader<PersonDto> reader = new BigQueryInteractiveQueryItemReaderBuilder<PersonDto>()
41+
BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
4242
.bigQuery(bigQuery)
4343
.query(String.format("SELECT p.name, p.age FROM spring_batch_extensions.%s p ORDER BY p.name LIMIT 2", tableName))
4444
.rowMapper(TestConstants.PERSON_MAPPER)

‎spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/reader/builder/BigQueryBatchQueryItemReaderBuilderTests.java

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/*
2+
* Copyright 2002-2023 the original author or authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package org.springframework.batch.extensions.bigquery.unit.reader.builder;
18+
19+
import com.google.cloud.bigquery.BigQuery;
20+
import com.google.cloud.bigquery.QueryJobConfiguration;
21+
import com.google.cloud.bigquery.TableId;
22+
import org.junit.jupiter.api.Assertions;
23+
import org.junit.jupiter.api.Test;
24+
import org.springframework.batch.extensions.bigquery.common.PersonDto;
25+
import org.springframework.batch.extensions.bigquery.common.TestConstants;
26+
import org.springframework.batch.extensions.bigquery.reader.BigQueryQueryItemReader;
27+
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQueryItemReaderBuilder;
28+
import org.springframework.batch.extensions.bigquery.unit.base.AbstractBigQueryTest;
29+
30+
class BigQueryBatchQueryItemReaderBuilderTests extends AbstractBigQueryTest {
31+
32+
@Test
33+
void testCustomQueryItemReader() {
34+
BigQuery mockedBigQuery = prepareMockedBigQuery();
35+
36+
QueryJobConfiguration jobConfiguration = QueryJobConfiguration
37+
.newBuilder("SELECT p.name, p.age FROM spring_batch_extensions.persons p LIMIT 2")
38+
.setDestinationTable(TableId.of(TestConstants.DATASET, "persons_duplicate"))
39+
.setPriority(QueryJobConfiguration.Priority.BATCH)
40+
.build();
41+
42+
BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
43+
.bigQuery(mockedBigQuery)
44+
.jobConfiguration(jobConfiguration)
45+
.rowMapper(TestConstants.PERSON_MAPPER)
46+
.build();
47+
48+
reader.afterPropertiesSet();
49+
50+
Assertions.assertNotNull(reader);
51+
}
52+
53+
}

0 commit comments

Comments
 (0)