Skip to content

Adding batchManager() method into code generated sync and async clients #2666

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
56bf8ed
Moving protected APIs out of internal package
Aug 17, 2021
a1a5022
Refactoring test import
Aug 17, 2021
3e10d3c
Adding configurable option for users to specify batchKey and batchBuf…
Aug 17, 2021
19ac4a0
Modifying batchBuffer and batchingMap to handle limits
Aug 17, 2021
91a9bc0
Returning exception on batch entry failure
Aug 17, 2021
e56da96
Modifying test to expect exception on batch entry failure
Aug 17, 2021
3c60b32
Refactoring core DefaultBatchManager
Aug 17, 2021
9a67cb0
Modifying SqsBatchManager and associated files to accept generated cl…
Aug 17, 2021
cd6ea6b
Adding batchManager configuration class
Aug 17, 2021
8ff9aa1
Adding batchManager config into CustomizationConfig
Aug 17, 2021
7a15a33
Adding batchManager() method
Aug 17, 2021
18ffd5a
Adding batchManager config into sqs customization.config
Aug 17, 2021
9445a55
Removed need for a hasExecutor and hasScheduledExecutor configuration
Aug 17, 2021
63a83df
Removed need to check and sepcify instance type
Aug 17, 2021
fd2b869
Adding .batchManager() method to async client
Aug 17, 2021
82dd2cd
Removing create() method in AsyncBatchManager and adding async batch …
Aug 17, 2021
d47456e
Merge branch 'feature/master/automatic-request-batching-codegenChange…
Aug 18, 2021
74b057b
Adding expected generated test files
Aug 18, 2021
1388fed
Adding tests for codegen batchManager() method
Aug 18, 2021
9300a44
Modifying SqsBatchManager tests to use client's batchManager() method
Aug 18, 2021
2545bfa
Modifying codegen to use a scheduledExecutor (sync clients don't supp…
Aug 18, 2021
000288a
Addressing PR comments
Aug 19, 2021
31b7e4f
Addressing more PR comments
Aug 19, 2021
b83022e
Only create default executor if builder.executor is null. Adding test…
Aug 19, 2021
676d5e6
Fixing checkstyle issues
Aug 19, 2021
419833d
Reverting to using Executor instead of ExecutorService
Aug 19, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/*
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

package software.amazon.awssdk.codegen.model.config.customization;

/**
* Config required to generate a batchManager method that returns an instance of a BatchManager in addition to any required
* executors or scheduledExecutors.
*/
public class BatchManagerMethod {

public static final String METHOD_NAME = "batchManager";

/** Fqcn of the return type of the operation for the sync client */
private String returnType;

/** Fqcn of the return type of the operation for the async client */
private String asyncReturnType;

public String getReturnType() {
return returnType;
}

public void setReturnType(String returnType) {
this.returnType = returnType;
}

public String getAsyncReturnType() {
return asyncReturnType;
}

public void setAsyncReturnType(String asyncReturnType) {
this.asyncReturnType = asyncReturnType;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,8 @@ public class CustomizationConfig {

private String userAgent;

private BatchManagerMethod batchManagerMethod;

private CustomizationConfig() {
}

Expand Down Expand Up @@ -485,4 +487,12 @@ public CustomizationConfig withUserAgent(String userAgent) {
this.userAgent = userAgent;
return this;
}

public BatchManagerMethod getBatchManagerMethod() {
return batchManagerMethod;
}

public void setBatchManagerMethod(BatchManagerMethod batchManagerMethod) {
this.batchManagerMethod = batchManagerMethod;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import static software.amazon.awssdk.codegen.poet.client.ClientClassUtils.addS3ArnableFieldCode;
import static software.amazon.awssdk.codegen.poet.client.ClientClassUtils.applyPaginatorUserAgentMethod;
import static software.amazon.awssdk.codegen.poet.client.ClientClassUtils.applySignerOverrideMethod;
import static software.amazon.awssdk.codegen.poet.client.ClientClassUtils.batchMangerMethod;
import static software.amazon.awssdk.codegen.poet.client.SyncClientClass.getProtocolSpecs;

import com.squareup.javapoet.ClassName;
Expand Down Expand Up @@ -83,13 +84,15 @@ public final class AsyncClientClass extends AsyncClientInterface {
private final PoetExtensions poetExtensions;
private final ClassName className;
private final ProtocolSpec protocolSpec;
private boolean hasScheduledExecutor;

public AsyncClientClass(GeneratorTaskParams dependencies) {
super(dependencies.getModel());
this.model = dependencies.getModel();
this.poetExtensions = dependencies.getPoetExtensions();
this.className = poetExtensions.getClientClass(model.getMetadata().getAsyncClient());
this.protocolSpec = getProtocolSpecs(poetExtensions, model);
this.hasScheduledExecutor = false;
}

@Override
Expand Down Expand Up @@ -141,12 +144,16 @@ public TypeSpec poetSpec() {
protocolSpec.createErrorResponseHandler().ifPresent(classBuilder::addMethod);

if (model.hasWaiters()) {
classBuilder.addField(FieldSpec.builder(ClassName.get(ScheduledExecutorService.class), "executorService")
.addModifiers(PRIVATE, FINAL)
.build());
classBuilder.addMethod(waiterImplMethod());
addScheduledExecutorIfNeeded(classBuilder);
}

if (model.getCustomizationConfig().getBatchManagerMethod() != null) {
classBuilder.addMethod(batchMangerMethod(model, false));
addScheduledExecutorIfNeeded(classBuilder);
}


return classBuilder.build();
}

Expand Down Expand Up @@ -195,7 +202,7 @@ private MethodSpec constructor(Builder classBuilder) {
builder.endControlFlow();
}

if (model.hasWaiters()) {
if (model.hasWaiters() || model.getCustomizationConfig().getBatchManagerMethod() != null) {
builder.addStatement("this.executorService = clientConfiguration.option($T.SCHEDULED_EXECUTOR_SERVICE)",
SdkClientOption.class);
}
Expand Down Expand Up @@ -485,4 +492,13 @@ private boolean hasStreamingV4AuthOperations() {
return model.getOperations().values().stream()
.anyMatch(this::shouldUseAsyncWithBodySigner);
}

private void addScheduledExecutorIfNeeded(Builder classBuilder) {
if (!hasScheduledExecutor) {
classBuilder.addField(FieldSpec.builder(ClassName.get(ScheduledExecutorService.class), "executorService")
.addModifiers(PRIVATE, FINAL)
.build());
hasScheduledExecutor = true;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import software.amazon.awssdk.codegen.docs.DocConfiguration;
import software.amazon.awssdk.codegen.docs.SimpleMethodOverload;
import software.amazon.awssdk.codegen.docs.WaiterDocs;
import software.amazon.awssdk.codegen.model.config.customization.BatchManagerMethod;
import software.amazon.awssdk.codegen.model.config.customization.UtilitiesMethod;
import software.amazon.awssdk.codegen.model.intermediate.IntermediateModel;
import software.amazon.awssdk.codegen.model.intermediate.OperationModel;
Expand Down Expand Up @@ -108,6 +109,10 @@ public TypeSpec poetSpec() {
result.addMethod(waiterMethod());
}

if (model.getCustomizationConfig().getBatchManagerMethod() != null) {
result.addMethod(batchManagerMethod());
}

return result.build();
}

Expand Down Expand Up @@ -467,4 +472,17 @@ private MethodSpec waiterMethod() {
.addJavadoc(WaiterDocs.waiterMethodInClient(poetExtensions.getAsyncWaiterInterface()))
.build();
}

private MethodSpec batchManagerMethod() {
BatchManagerMethod config = model.getCustomizationConfig().getBatchManagerMethod();
ClassName returnType = PoetUtils.classNameFromFqcn(config.getAsyncReturnType());

return MethodSpec.methodBuilder("batchManager")
.returns(returnType)
.addModifiers(Modifier.PUBLIC, Modifier.DEFAULT)
.addStatement("throw new $T()", UnsupportedOperationException.class)
.addJavadoc("Creates an instance of {@link $T} object with the "
+ "configuration set on this client.", returnType)
.build();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import software.amazon.awssdk.arns.Arn;
import software.amazon.awssdk.auth.signer.EventStreamAws4Signer;
import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration;
import software.amazon.awssdk.codegen.model.config.customization.BatchManagerMethod;
import software.amazon.awssdk.codegen.model.config.customization.S3ArnableFieldConfig;
import software.amazon.awssdk.codegen.model.intermediate.IntermediateModel;
import software.amazon.awssdk.codegen.model.intermediate.MemberModel;
Expand Down Expand Up @@ -268,6 +269,25 @@ static Optional<CodeBlock> addS3ArnableFieldCode(OperationModel opModel, Interme
return Optional.empty();
}

static MethodSpec batchMangerMethod(IntermediateModel model, boolean isSync) {
String scheduledExecutor = "executorService";
BatchManagerMethod config = model.getCustomizationConfig().getBatchManagerMethod();
ClassName returnType;
if (isSync) {
returnType = PoetUtils.classNameFromFqcn(config.getReturnType());
} else {
returnType = PoetUtils.classNameFromFqcn(config.getAsyncReturnType());
}

return MethodSpec.methodBuilder(BatchManagerMethod.METHOD_NAME)
.addModifiers(Modifier.PUBLIC)
.addAnnotation(Override.class)
.returns(returnType)
.addStatement("return $T.builder().client(this).scheduledExecutor($N).build()",
returnType, scheduledExecutor)
.build();
}

/**
* Given operation and c2j name, returns the String that represents calling the
* c2j member's getter method in the opmodel input shape.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import static software.amazon.awssdk.codegen.poet.client.ClientClassUtils.addS3ArnableFieldCode;
import static software.amazon.awssdk.codegen.poet.client.ClientClassUtils.applyPaginatorUserAgentMethod;
import static software.amazon.awssdk.codegen.poet.client.ClientClassUtils.applySignerOverrideMethod;
import static software.amazon.awssdk.codegen.poet.client.ClientClassUtils.batchMangerMethod;

import com.squareup.javapoet.ClassName;
import com.squareup.javapoet.FieldSpec;
Expand All @@ -32,6 +33,7 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ScheduledExecutorService;
import java.util.stream.Collectors;
import javax.lang.model.element.Modifier;
import software.amazon.awssdk.annotations.SdkInternalApi;
Expand Down Expand Up @@ -116,6 +118,13 @@ public TypeSpec poetSpec() {
classBuilder.addMethod(utilitiesMethod());
}

if (model.getCustomizationConfig().getBatchManagerMethod() != null) {
classBuilder.addMethod(batchMangerMethod(model, true));
classBuilder.addField(FieldSpec.builder(ClassName.get(ScheduledExecutorService.class), "executorService")
.addModifiers(PRIVATE, FINAL)
.build());
}

model.getEndpointOperation().ifPresent(
o -> classBuilder.addField(EndpointDiscoveryRefreshCache.class, "endpointDiscoveryCache", PRIVATE));

Expand Down Expand Up @@ -181,6 +190,11 @@ private MethodSpec constructor() {
builder.endControlFlow();
}

if (model.getCustomizationConfig().getBatchManagerMethod() != null) {
builder.addStatement("this.executorService = clientConfiguration.option($T.SCHEDULED_EXECUTOR_SERVICE)",
SdkClientOption.class);
}

return builder.build();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import software.amazon.awssdk.codegen.docs.DocConfiguration;
import software.amazon.awssdk.codegen.docs.SimpleMethodOverload;
import software.amazon.awssdk.codegen.docs.WaiterDocs;
import software.amazon.awssdk.codegen.model.config.customization.BatchManagerMethod;
import software.amazon.awssdk.codegen.model.config.customization.UtilitiesMethod;
import software.amazon.awssdk.codegen.model.intermediate.IntermediateModel;
import software.amazon.awssdk.codegen.model.intermediate.OperationModel;
Expand Down Expand Up @@ -110,6 +111,10 @@ public TypeSpec poetSpec() {
result.addMethod(waiterMethod());
}

if (model.getCustomizationConfig().getBatchManagerMethod() != null) {
result.addMethod(batchManagerMethod());
}

return result.build();
}

Expand Down Expand Up @@ -498,4 +503,17 @@ private MethodSpec waiterMethod() {
.addJavadoc(WaiterDocs.waiterMethodInClient(poetExtensions.getSyncWaiterInterface()))
.build();
}

private MethodSpec batchManagerMethod() {
BatchManagerMethod config = model.getCustomizationConfig().getBatchManagerMethod();
ClassName returnType = PoetUtils.classNameFromFqcn(config.getReturnType());

return MethodSpec.methodBuilder("batchManager")
.returns(returnType)
.addModifiers(Modifier.PUBLIC, Modifier.DEFAULT)
.addStatement("throw new $T()", UnsupportedOperationException.class)
.addJavadoc("Creates an instance of {@link $T} object with the "
+ "configuration set on this client.", returnType)
.build();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,18 @@ public static IntermediateModel internalConfigModels() {
return new IntermediateModelBuilder(models).build();
}

public static IntermediateModel batchManagerModels() {
File serviceModel = new File(ClientTestModels.class.getResource("client/c2j/batchmanager/service-2.json").getFile());
File customizationModel = new File(ClientTestModels.class.getResource("client/c2j/batchmanager/customization.config").getFile());

C2jModels models = C2jModels.builder()
.serviceModel(getServiceModel(serviceModel))
.customizationConfig(getCustomizationConfig(customizationModel))
.build();

return new IntermediateModelBuilder(models).build();
}

private static ServiceModel getServiceModel(File file) {
return ModelLoaderUtils.loadModel(ServiceModel.class, file);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import software.amazon.awssdk.codegen.model.intermediate.IntermediateModel;
import software.amazon.awssdk.codegen.poet.ClassSpec;
import software.amazon.awssdk.codegen.poet.ClientTestModels;
import software.amazon.awssdk.codegen.poet.waiters.WaiterInterfaceSpec;

public class PoetClientFunctionalTests {

Expand Down Expand Up @@ -117,4 +118,29 @@ public void syncClientCustomServiceMetaData() throws Exception {
ClassSpec syncClientCustomServiceMetaData = createSyncClientClass(ClientTestModels.customContentTypeModels());
assertThat(syncClientCustomServiceMetaData, generatesTo("test-customservicemetadata-sync.java"));
}

@Test
public void syncClientBatchManager() throws Exception {
ClassSpec syncClientBatchManager = createSyncClientClass(ClientTestModels.batchManagerModels());
assertThat(syncClientBatchManager, generatesTo("test-batchmanager-sync-class.java"));
}

@Test
public void asyncClientBatchManager() throws Exception {
ClassSpec asyncClientBatchManager = new AsyncClientClass(
GeneratorTaskParams.create(ClientTestModels.batchManagerModels(), "sources/", "tests/"));
assertThat(asyncClientBatchManager, generatesTo("test-batchmanager-async-class.java"));
}

@Test
public void syncClientBatchManagerInterface() throws Exception {
ClassSpec syncClientBatchManagerInterface = new SyncClientInterface(ClientTestModels.batchManagerModels());
assertThat(syncClientBatchManagerInterface, generatesTo("test-batchmanager-sync-interface.java"));
}

@Test
public void asyncClientBatchManagerInterface() throws Exception {
ClassSpec asyncClientBatchManagerInterface = new AsyncClientInterface(ClientTestModels.batchManagerModels());
assertThat(asyncClientBatchManagerInterface, generatesTo("test-batchmanager-async-interface.java"));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"batchManagerMethod": {
"returnType": "software.amazon.awssdk.services.batchmanagertest.batchmanager.SyncBatchManagerTest",
"asyncReturnType": "software.amazon.awssdk.services.batchmanagertest.batchmanager.AsyncBatchManagerTest"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
{
"version":"2.0",
"metadata":{
"apiVersion":"2016-03-11",
"endpointPrefix":"batchmanager",
"jsonVersion":"1.1",
"protocol":"rest-json",
"serviceAbbreviation":"BatchManager",
"serviceFullName":"BatchManager",
"serviceId":"BatchManager",
"signatureVersion":"v4",
"uid":"batchmanager-2016-03-11"
},
"operations":{
},
"shapes": {
"String":{"type":"string"}
},
"documentation": "A service that implements the batchManager() method"
}
Loading