AWS/Query and EC2 marshaller support.

build-tools/src/main/resources/software/amazon/awssdk/spotbugs-suppressions.xml

@@ -27,14 +27,14 @@
     <Match>
         <Or>
             <!-- Container classes do not copy arrays for performance reasons at this time. -->
-            <Class name="software.amazon.awssdk.core.protocol.json.JsonContent" />
+            <Class name="software.amazon.awssdk.protocols.json.JsonContent" />
         </Or>
         <Bug pattern="EI_EXPOSE_REP,EI_EXPOSE_REP2" />
     </Match>
 
     <!-- Delegate closes input stream. -->
     <Match>
-        <Class name="software.amazon.awssdk.core.internal.protocol.json.IonFactory" />
+        <Class name="software.amazon.awssdk.protocols.ion.internal.IonFactory" />
         <Method name="createParser" />
         <Bug pattern="OBL_UNSATISFIED_OBLIGATION" />
     </Match>
@@ -44,7 +44,7 @@
     <Match>
         <Or>
             <Class name="software.amazon.awssdk.services.dynamodb.datamodeling.DynamoDBMapper$BatchGetItemException"/>
-            <Class name="software.amazon.awssdk.core.internal.protocol.json.IonFactory"/>
+            <Class name="software.amazon.awssdk.protocols.ion.internal.IonFactory"/>
         </Or>
         <Bug pattern="SE_TRANSIENT_FIELD_NOT_RESTORED" />
     </Match>

buildspecs/on-demand-integ-test.yml

@@ -0,0 +1,6 @@
+version: 0.2
+
+phases:
+  build:
+    commands:
+    - mvn clean verify -Dskip.unit.tests -P integration-tests -Dfindbugs.skip -Dcheckstyle.skip -pl !:dynamodbmapper-v1 -Dfailsafe.rerunFailingTestsCount=1 -Dmaven.wagon.httpconnectionManager.maxPerRoute=2 --fail-at-end

codegen/pom.xml

@@ -110,6 +110,31 @@
             <groupId>org.reactivestreams</groupId>
             <artifactId>reactive-streams</artifactId>
         </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>aws-json-protocol</artifactId>
+            <version>${awsjavasdk.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>aws-cbor-protocol</artifactId>
+            <version>${awsjavasdk.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>aws-ion-protocol</artifactId>
+            <version>${awsjavasdk.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>aws-query-protocol</artifactId>
+            <version>${awsjavasdk.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>protocol-core</artifactId>
+            <version>${awsjavasdk.version}</version>
+        </dependency>
 
         <dependency>
             <artifactId>junit</artifactId>

codegen/src/main/java/software/amazon/awssdk/codegen/AddShapes.java

@@ -221,18 +221,22 @@ private ParameterHttpMapping generateParameterHttpMapping(Shape parentShape,
         ParameterHttpMapping mapping = new ParameterHttpMapping();
 
         Shape memberShape = allC2jShapes.get(member.getShape());
-
         mapping.withLocation(Location.forValue(member.getLocation()))
-                .withPayload(member.isPayload()).withStreaming(member.isStreaming())
-                .withFlattened(member.isFlattened() || memberShape.isFlattened())
-                .withUnmarshallLocationName(deriveUnmarshallerLocationName(memberName, member))
-                .withMarshallLocationName(
-                        deriveMarshallerLocationName(memberName, member, protocol))
-                .withIsGreedy(isGreedy(parentShape, allC2jShapes, mapping));
+               .withPayload(member.isPayload()).withStreaming(member.isStreaming())
+               .withFlattened(isFlattened(member, memberShape))
+               .withUnmarshallLocationName(deriveUnmarshallerLocationName(memberShape, memberName, member))
+               .withMarshallLocationName(
+                        deriveMarshallerLocationName(memberShape, memberName, member, protocol))
+               .withIsGreedy(isGreedy(parentShape, allC2jShapes, mapping));
 
         return mapping;
     }
 
+    private boolean isFlattened(Member member, Shape memberShape) {
+        return member.isFlattened()
+               || memberShape.isFlattened();
+    }
+
     /**
      * @param parentShape  Shape containing the member in question.
      * @param allC2jShapes All shapes in the service model.
@@ -266,9 +270,14 @@ private String findRequestUri(Shape parentShape, Map<String, Shape> allC2jShapes
                 .findFirst().orElseThrow(() -> new RuntimeException("Could not find request URI for input shape"));
     }
 
-    private String deriveUnmarshallerLocationName(String memberName, Member member) {
-
-        String locationName = member.getLocationName();
+    private String deriveUnmarshallerLocationName(Shape memberShape, String memberName, Member member) {
+        String locationName;
+        if (memberShape.getListMember() != null && memberShape.isFlattened()) {
+            locationName = memberShape.getListMember().getLocationName() == null ?
+                           member.getLocationName() : memberShape.getListMember().getLocationName();
+        } else {
+            locationName = member.getLocationName();
+        }
 
         if (locationName != null && !locationName.trim().isEmpty()) {
             return locationName;
@@ -277,12 +286,13 @@ private String deriveUnmarshallerLocationName(String memberName, Member member)
         return memberName;
     }
 
-    private String deriveMarshallerLocationName(String memberName, Member member, String protocol) {
+    private String deriveMarshallerLocationName(Shape memberShape, String memberName, Member member, String protocol) {
         String queryName = member.getQueryName();
         if (queryName != null && !queryName.trim().isEmpty()) {
             return queryName;
         } else {
-            String locationName = member.getLocationName();
+            String locationName = memberShape.getListMember() != null && memberShape.isFlattened() && !protocol.equals("ec2") ?
+                                  memberShape.getListMember().getLocationName() : member.getLocationName();
             if (locationName != null && !locationName.trim().isEmpty()) {
                 if (protocol.equals(Protocol.EC2.getValue())) {
                     return StringUtils.upperCase(locationName.substring(0, 1)) +

codegen/src/main/java/software/amazon/awssdk/codegen/emitters/tasks/MarshallerGeneratorTasks.java

@@ -28,6 +28,7 @@
 import software.amazon.awssdk.codegen.emitters.GeneratorTask;
 import software.amazon.awssdk.codegen.emitters.GeneratorTaskParams;
 import software.amazon.awssdk.codegen.model.intermediate.Metadata;
+import software.amazon.awssdk.codegen.model.intermediate.Protocol;
 import software.amazon.awssdk.codegen.model.intermediate.ShapeModel;
 import software.amazon.awssdk.codegen.model.intermediate.ShapeType;
 import software.amazon.awssdk.codegen.poet.eventstream.EventStreamUtils;
@@ -68,8 +69,7 @@ private boolean shouldGenerate(ShapeModel shapeModel) {
     }
 
     private Stream<GeneratorTask> createTask(String javaShapeName, ShapeModel shapeModel) throws Exception {
-        if (metadata.isJsonProtocol()) {
-
+        if (metadata.isJsonProtocol() || metadata.getProtocol() == Protocol.QUERY || metadata.getProtocol() == Protocol.EC2) {
             return ShapeType.Request == shapeModel.getShapeType() ||
                    (ShapeType.Model == shapeModel.getShapeType() && shapeModel.isEvent()
                     && EventStreamUtils.isRequestEvent(model, shapeModel))

codegen/src/main/java/software/amazon/awssdk/codegen/internal/Utils.java

@@ -314,8 +314,10 @@ public static ShapeMarshaller createInputShapeMarshaller(ServiceMetadata service
                 marshaller.setXmlNameSpaceUri(xmlNamespace.getUri());
             }
         }
-        if (!StringUtils.isEmpty(service.getTargetPrefix()) && Metadata.isNotRestProtocol(service.getProtocol())) {
-            marshaller.setTarget(service.getTargetPrefix() + "." + operation.getName());
+        if (Metadata.isNotRestProtocol(service.getProtocol())) {
+            marshaller.setTarget(StringUtils.isEmpty(service.getTargetPrefix()) ?
+                                 operation.getName() :
+                                 service.getTargetPrefix() + "." + operation.getName());
         }
         return marshaller;
 

codegen/src/main/java/software/amazon/awssdk/codegen/model/intermediate/MemberModel.java

@@ -28,7 +28,6 @@
 import java.util.Map;
 import software.amazon.awssdk.codegen.internal.TypeUtils;
 import software.amazon.awssdk.core.SdkBytes;
-import software.amazon.awssdk.core.protocol.SdkField;
 import software.amazon.awssdk.core.runtime.transform.PathMarshaller;
 import software.amazon.awssdk.utils.StringUtils;
 

codegen/src/main/java/software/amazon/awssdk/codegen/naming/NamingStrategy.java

@@ -17,7 +17,6 @@
 
 import software.amazon.awssdk.codegen.model.intermediate.MemberModel;
 import software.amazon.awssdk.codegen.model.service.Shape;
-import software.amazon.awssdk.core.protocol.SdkField;
 
 /**
  * Strategy to name various Java constructs based on the naming in the model and potentially customizations.

codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/AsyncClientClass.java

@@ -41,7 +41,6 @@
 import software.amazon.awssdk.awscore.client.handler.AwsAsyncClientHandler;
 import software.amazon.awssdk.awscore.eventstream.EventStreamTaggedUnionJsonMarshaller;
 import software.amazon.awssdk.awscore.internal.client.handler.AwsClientHandlerUtils;
-import software.amazon.awssdk.awscore.protocol.json.AwsJsonProtocolFactory;
 import software.amazon.awssdk.codegen.emitters.GeneratorTaskParams;
 import software.amazon.awssdk.codegen.model.intermediate.IntermediateModel;
 import software.amazon.awssdk.codegen.model.intermediate.MemberModel;
@@ -56,6 +55,7 @@
 import software.amazon.awssdk.core.client.config.SdkAdvancedAsyncClientOption;
 import software.amazon.awssdk.core.client.config.SdkClientConfiguration;
 import software.amazon.awssdk.core.client.handler.AsyncClientHandler;
+import software.amazon.awssdk.protocols.json.AwsJsonProtocolFactory;
 import software.amazon.awssdk.utils.CompletableFutureUtils;
 import software.amazon.awssdk.utils.FunctionalUtils;
 
@@ -70,7 +70,7 @@ public AsyncClientClass(GeneratorTaskParams dependencies) {
         this.model = dependencies.getModel();
         this.poetExtensions = dependencies.getPoetExtensions();
         this.className = poetExtensions.getClientClass(model.getMetadata().getAsyncClient());
-        this.protocolSpec = getProtocolSpecs(poetExtensions, model.getMetadata().getProtocol());
+        this.protocolSpec = getProtocolSpecs(poetExtensions, model);
     }
 
     @Override
@@ -94,6 +94,7 @@ public TypeSpec poetSpec() {
                     .addMethods(operations())
                     .addMethod(closeMethod())
                     .addMethods(protocolSpec.additionalMethods())
+                    .addFields(protocolSpec.additionalFields())
                     .addMethod(protocolSpec.initProtocolFactory(model));
 
         // Kinesis doesn't support CBOR for STS yet so need another protocol factory for JSON
@@ -120,14 +121,15 @@ private MethodSpec constructor(Builder classBuilder) {
                                                .addParameter(SdkClientConfiguration.class, "clientConfiguration")
                                                .addStatement("this.clientHandler = new $T(clientConfiguration)",
                                                              AwsAsyncClientHandler.class);
+        FieldSpec protocolFactoryField = protocolSpec.protocolFactory(model);
         if (model.getMetadata().isJsonProtocol()) {
-            builder.addStatement("this.$N = init($L)", protocolSpec.protocolFactory(model).name,
-                                 model.getMetadata().isCborProtocol());
+            builder.addStatement("this.$N = init($T.builder()).build()", protocolFactoryField.name,
+                                protocolFactoryField.type);
         } else {
-            builder.addStatement("this.$N = init()", protocolSpec.protocolFactory(model).name);
+            builder.addStatement("this.$N = init()", protocolFactoryField.name);
         }
         if (model.getMetadata().isCborProtocol()) {
-            builder.addStatement("this.jsonProtocolFactory = init(false)");
+            builder.addStatement("this.jsonProtocolFactory = init($T.builder()).build()", AwsJsonProtocolFactory.class);
         }
         if (hasOperationWithEventStreamOutput()) {
             classBuilder.addField(FieldSpec.builder(ClassName.get(Executor.class), "executor",

codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/SyncClientClass.java

@@ -22,6 +22,7 @@
 import static software.amazon.awssdk.codegen.poet.client.ClientClassUtils.getCustomResponseHandler;
 
 import com.squareup.javapoet.ClassName;
+import com.squareup.javapoet.FieldSpec;
 import com.squareup.javapoet.MethodSpec;
 import com.squareup.javapoet.TypeSpec;
 import com.squareup.javapoet.TypeSpec.Builder;
@@ -42,6 +43,7 @@
 import software.amazon.awssdk.codegen.poet.client.specs.JsonProtocolSpec;
 import software.amazon.awssdk.codegen.poet.client.specs.ProtocolSpec;
 import software.amazon.awssdk.codegen.poet.client.specs.QueryXmlProtocolSpec;
+import software.amazon.awssdk.codegen.poet.client.specs.XmlProtocolSpec;
 import software.amazon.awssdk.codegen.utils.PaginatorUtils;
 import software.amazon.awssdk.core.client.config.SdkClientConfiguration;
 import software.amazon.awssdk.core.client.handler.SyncClientHandler;
@@ -58,7 +60,7 @@ public SyncClientClass(GeneratorTaskParams taskParams) {
         this.model = taskParams.getModel();
         this.poetExtensions = taskParams.getPoetExtensions();
         this.className = poetExtensions.getClientClass(model.getMetadata().getSyncClient());
-        this.protocolSpec = getProtocolSpecs(poetExtensions, model.getMetadata().getProtocol());
+        this.protocolSpec = getProtocolSpecs(poetExtensions, model);
     }
 
     @Override
@@ -76,6 +78,8 @@ public TypeSpec poetSpec() {
                                         .addField(SdkClientConfiguration.class, "clientConfiguration", PRIVATE, FINAL)
                                         .addMethod(constructor())
                                         .addMethod(nameMethod())
+                                        .addFields(protocolSpec.additionalFields())
+                                        .addMethods(protocolSpec.additionalMethods())
                                         .addMethods(operations());
 
         protocolSpec.createErrorResponseHandler().ifPresent(classBuilder::addMethod);
@@ -116,11 +120,12 @@ private MethodSpec constructor() {
                                                .addStatement("this.clientHandler = new $T(clientConfiguration)",
                                                              protocolSpec.getClientHandlerClass())
                                                .addStatement("this.clientConfiguration = clientConfiguration");
+        FieldSpec protocolFactoryField = protocolSpec.protocolFactory(model);
         if (model.getMetadata().isJsonProtocol()) {
-            builder.addStatement("this.$N = init($L)", protocolSpec.protocolFactory(model).name,
-                                 model.getMetadata().isCborProtocol());
+            builder.addStatement("this.$N = init($T.builder()).build()", protocolFactoryField.name,
+                                 protocolFactoryField.type);
         } else {
-            builder.addStatement("this.$N = init()", protocolSpec.protocolFactory(model).name);
+            builder.addStatement("this.$N = init()", protocolFactoryField.name);
         }
         return builder.build();
     }
@@ -182,18 +187,20 @@ private MethodSpec closeMethod() {
                          .build();
     }
 
-    static ProtocolSpec getProtocolSpecs(PoetExtensions poetExtensions, Protocol protocol) {
+    static ProtocolSpec getProtocolSpecs(PoetExtensions poetExtensions, IntermediateModel model) {
+        Protocol protocol = model.getMetadata().getProtocol();
         switch (protocol) {
             case QUERY:
-            case REST_XML:
                 return new QueryXmlProtocolSpec(poetExtensions);
+            case REST_XML:
+                return new XmlProtocolSpec(poetExtensions);
             case EC2:
                 return new Ec2ProtocolSpec(poetExtensions);
             case AWS_JSON:
             case REST_JSON:
             case CBOR:
             case ION:
-                return new JsonProtocolSpec(poetExtensions);
+                return new JsonProtocolSpec(poetExtensions, model);
             case API_GATEWAY:
                 throw new UnsupportedOperationException("Not yet supported.");
             default:

codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/specs/Ec2ProtocolSpec.java

@@ -19,6 +19,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import software.amazon.awssdk.codegen.poet.PoetExtensions;
+import software.amazon.awssdk.protocols.query.AwsEc2ProtocolFactory;
 
 public class Ec2ProtocolSpec extends QueryXmlProtocolSpec {
 
@@ -36,6 +37,11 @@ public List<MethodSpec> additionalMethods() {
         return additionalMethods;
     }
 
+    @Override
+    protected Class<?> protocolFactoryClass() {
+        return AwsEc2ProtocolFactory.class;
+    }
+
     /*
     private MethodSpec dryRunMethod() {
         TypeVariableName typeVariableName = TypeVariableName.get("X", AmazonWebServiceRequest.class);