Refactoring for JSON marshalling/unmarshalling, support for JSON value trait

Author: shorea

codegen/src/main/java/software/amazon/awssdk/codegen/AddShapes.java

@@ -162,7 +162,8 @@ private MemberModel generateMemberModel(String c2jMemberName, Member c2jMemberDe
                    .withVariable(new VariableModel(variableName, variableType, variableDeclarationType)
                                          .withDocumentation(c2jMemberDefinition.getDocumentation()))
                    .withSetterModel(new VariableModel(variableName, variableType, variableDeclarationType))
-                   .withGetterModel(new ReturnTypeModel(variableType));
+                   .withGetterModel(new ReturnTypeModel(variableType))
+                   .withJsonValue(c2jMemberDefinition.getJsonValue());
         memberModel.setDocumentation(c2jMemberDefinition.getDocumentation());
         memberModel.setDeprecated(c2jMemberDefinition.isDeprecated());
         memberModel

codegen/src/main/java/software/amazon/awssdk/codegen/IntermediateModelBuilder.java

@@ -90,7 +90,7 @@ private List<IntermediateModelShapeProcessor> createShapeProcessors() {
         return processors;
     }
 
-    public IntermediateModel build() throws IOException {
+    public IntermediateModel build() {
         // Note: This needs to come before any pre/post processing of the
         // models, as the transformer must have access to the original shapes,
         // before any customizations have been applied (which modifies them).
@@ -119,7 +119,7 @@ public IntermediateModel build() throws IOException {
 
         IntermediateModel fullModel = new IntermediateModel(
             constructMetadata(service, codeGenConfig, customConfig), operations, shapes,
-            customConfig, examples, authorizers, paginators.getPaginators());
+            customConfig, examples, authorizers, paginators.getPaginators(), namingStrategy);
 
         customization.postprocess(fullModel);
 
@@ -135,7 +135,8 @@ public IntermediateModel build() throws IOException {
                                                                fullModel.getCustomizationConfig(),
                                                                fullModel.getExamples(),
                                                                fullModel.getCustomAuthorizers(),
-                                                               fullModel.getPaginators());
+                                                               fullModel.getPaginators(),
+                                                               namingStrategy);
 
         linkMembersToShapes(trimmedModel);
         linkOperationsToInputOutputShapes(trimmedModel);

codegen/src/main/java/software/amazon/awssdk/codegen/emitters/tasks/MarshallerGeneratorTasks.java

@@ -30,7 +30,6 @@
 import software.amazon.awssdk.codegen.model.intermediate.Metadata;
 import software.amazon.awssdk.codegen.model.intermediate.ShapeModel;
 import software.amazon.awssdk.codegen.model.intermediate.ShapeType;
-import software.amazon.awssdk.codegen.poet.transform.JsonModelMarshallerSpec;
 import software.amazon.awssdk.codegen.poet.transform.MarshallerSpec;
 import software.amazon.awssdk.utils.ImmutableMap;
 
@@ -48,7 +47,7 @@ public MarshallerGeneratorTasks(GeneratorTaskParams dependencies) {
     }
 
     @Override
-    protected List<GeneratorTask> createTasks() throws Exception {
+    protected List<GeneratorTask> createTasks() {
         info("Emitting marshaller classes");
         return model.getShapes().entrySet().stream()
                     .filter(e -> shouldGenerate(e.getValue()))
@@ -70,9 +69,8 @@ private boolean shouldGenerate(ShapeModel shapeModel) {
     private Stream<GeneratorTask> createTask(String javaShapeName, ShapeModel shapeModel) throws Exception {
         if (metadata.isJsonProtocol()) {
             return ShapeType.Request == shapeModel.getShapeType() ?
-                   Stream.of(createPoetGeneratorTask(new JsonModelMarshallerSpec(model, shapeModel, "ModelMarshaller")),
-                             createPoetGeneratorTask(new MarshallerSpec(model, shapeModel))) :
-                   Stream.of(createPoetGeneratorTask(new JsonModelMarshallerSpec(model, shapeModel, "Marshaller")));
+                   Stream.of(createPoetGeneratorTask(new MarshallerSpec(model, shapeModel))) :
+                   Stream.empty();
         }
 
         return Stream.of(

codegen/src/main/java/software/amazon/awssdk/codegen/emitters/tasks/UnmarshallerGeneratorTasks.java

@@ -18,6 +18,7 @@
 import static software.amazon.awssdk.utils.FunctionalUtils.safeFunction;
 
 import freemarker.template.Template;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -43,6 +44,9 @@ public UnmarshallerGeneratorTasks(GeneratorTaskParams dependencies) {
     @Override
     protected List<GeneratorTask> createTasks() throws Exception {
         info("Emitting unmarshaller classes");
+        if (metadata.isJsonProtocol()) {
+            return Collections.emptyList();
+        }
         return model.getShapes().entrySet().stream()
                 .filter(e -> shouldGenerate(e.getValue()))
                 .map(safeFunction(e -> createTask(e.getKey(), e.getValue())))

codegen/src/main/java/software/amazon/awssdk/codegen/model/intermediate/IntermediateModel.java

@@ -31,6 +31,8 @@
 import software.amazon.awssdk.codegen.internal.Utils;
 import software.amazon.awssdk.codegen.model.config.customization.CustomizationConfig;
 import software.amazon.awssdk.codegen.model.service.PaginatorDefinition;
+import software.amazon.awssdk.codegen.naming.DefaultNamingStrategy;
+import software.amazon.awssdk.codegen.naming.NamingStrategy;
 import software.amazon.awssdk.core.SdkResponseMetadata;
 import software.amazon.awssdk.utils.IoUtils;
 
@@ -56,6 +58,9 @@ public final class IntermediateModel {
     @JsonIgnore
     private final Map<String, PaginatorDefinition> paginators;
 
+    @JsonIgnore
+    private final NamingStrategy namingStrategy;
+
     @JsonCreator
     public IntermediateModel(
         @JsonProperty("metadata") Metadata metadata,
@@ -64,7 +69,7 @@ public IntermediateModel(
         @JsonProperty("customizationConfig") CustomizationConfig customizationConfig,
         @JsonProperty("serviceExamples") ServiceExamples examples) {
 
-        this(metadata, operations, shapes, customizationConfig, examples, Collections.emptyMap(), Collections.emptyMap());
+        this(metadata, operations, shapes, customizationConfig, examples, Collections.emptyMap(), Collections.emptyMap(), null);
     }
 
     public IntermediateModel(
@@ -74,14 +79,16 @@ public IntermediateModel(
         CustomizationConfig customizationConfig,
         ServiceExamples examples,
         Map<String, AuthorizerModel> customAuthorizers,
-        Map<String, PaginatorDefinition> paginators) {
+        Map<String, PaginatorDefinition> paginators,
+        NamingStrategy namingStrategy) {
         this.metadata = metadata;
         this.operations = operations;
         this.shapes = shapes;
         this.customizationConfig = customizationConfig;
         this.examples = examples;
         this.customAuthorizers = customAuthorizers;
         this.paginators = paginators;
+        this.namingStrategy = namingStrategy;
     }
 
     public Metadata getMetadata() {
@@ -116,6 +123,10 @@ public Map<String, PaginatorDefinition> getPaginators() {
         return paginators;
     }
 
+    public NamingStrategy getNamingStrategy() {
+        return namingStrategy;
+    }
+
     /**
      * @return Exception unmarshaller implementation to use. Currently only needed by XML based
      * protocols.

codegen/src/main/java/software/amazon/awssdk/codegen/model/intermediate/MemberModel.java

@@ -74,6 +74,8 @@ public class MemberModel extends DocumentationModel {
 
     private String beanStyleSetterName;
 
+    private boolean isJsonValue;
+
     public String getName() {
         return name;
     }
@@ -451,6 +453,19 @@ public boolean getIsBinary() {
         return http.getIsStreaming() || (http.getIsPayload() && isSdkBytesType());
     }
 
+    public boolean isJsonValue() {
+        return isJsonValue;
+    }
+
+    public void setJsonValue(boolean jsonValue) {
+        isJsonValue = jsonValue;
+    }
+
+    public MemberModel withJsonValue(boolean jsonValue) {
+        setJsonValue(jsonValue);
+        return this;
+    }
+
     /**
      * @return Implementation of {@link PathMarshaller} to use if this member is bound the the URI.
      * @throws IllegalStateException If this member is not bound to the URI. Templates should first check
@@ -508,7 +523,7 @@ public String getMarshallingType() {
         } else if (isMap()) {
             return "MAP";
         } else if (!isSimple()) {
-            return "STRUCTURED";
+            return "SDK_POJO";
         } else {
             return TypeUtils.getMarshallingType(variable.getSimpleType());
         }
@@ -525,7 +540,7 @@ public String getMarshallingTargetClass() {
         } else if (isMap()) {
             return "Map";
         } else if (!isSimple()) {
-            return "StructuredPojo";
+            return "SdkPojo";
         } else {
             return variable.getVariableType();
         }

codegen/src/main/java/software/amazon/awssdk/codegen/model/service/Member.java

@@ -15,6 +15,8 @@
 
 package software.amazon.awssdk.codegen.model.service;
 
+import com.fasterxml.jackson.annotation.JsonProperty;
+
 public class Member {
 
     private String shape;
@@ -39,6 +41,9 @@ public class Member {
 
     private boolean deprecated;
 
+    @JsonProperty("jsonvalue")
+    private boolean jsonValue;
+
     public String getShape() {
         return shape;
     }
@@ -126,4 +131,12 @@ public boolean isDeprecated() {
     public void setDeprecated(boolean deprecated) {
         this.deprecated = deprecated;
     }
+
+    public boolean getJsonValue() {
+        return jsonValue;
+    }
+
+    public void setJsonValue(boolean jsonValue) {
+        this.jsonValue = jsonValue;
+    }
 }

codegen/src/main/java/software/amazon/awssdk/codegen/naming/DefaultNamingStrategy.java

@@ -22,11 +22,13 @@
 import static software.amazon.awssdk.codegen.internal.Constant.REQUEST_CLASS_SUFFIX;
 import static software.amazon.awssdk.codegen.internal.Constant.RESPONSE_CLASS_SUFFIX;
 import static software.amazon.awssdk.codegen.internal.Constant.VARIABLE_NAME_SUFFIX;
+import static software.amazon.awssdk.codegen.internal.Utils.capitalize;
 import static software.amazon.awssdk.codegen.internal.Utils.unCapitalize;
 
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
+import java.util.Locale;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
@@ -35,6 +37,7 @@
 import software.amazon.awssdk.codegen.internal.Constant;
 import software.amazon.awssdk.codegen.internal.Utils;
 import software.amazon.awssdk.codegen.model.config.customization.CustomizationConfig;
+import software.amazon.awssdk.codegen.model.intermediate.MemberModel;
 import software.amazon.awssdk.codegen.model.service.ServiceModel;
 import software.amazon.awssdk.codegen.model.service.Shape;
 import software.amazon.awssdk.utils.Logger;
@@ -153,6 +156,10 @@ private String concatServiceNameIfShareModel(String serviceName) {
         return serviceName;
     }
 
+    private String screamCase(String word) {
+        return Stream.of(splitOnWordBoundaries(word)).map(s -> s.toUpperCase(Locale.US)).collect(joining("_"));
+    }
+
     private String pascalCase(String word) {
         return Stream.of(splitOnWordBoundaries(word)).map(StringUtils::lowerCase).map(Utils::capitalize).collect(joining());
     }
@@ -292,6 +299,11 @@ public String getFluentEnumSetterMethodName(String memberName, Shape shape) {
         return Utils.unCapitalize(memberName);
     }
 
+    @Override
+    public String getSdkFieldFieldName(MemberModel memberModel) {
+        return screamCase(memberModel.getName()) + "_FIELD";
+    }
+
     private String[] splitOnWordBoundaries(String toSplit) {
         String result = toSplit;
 

codegen/src/main/java/software/amazon/awssdk/codegen/naming/NamingStrategy.java

@@ -15,7 +15,9 @@
 
 package software.amazon.awssdk.codegen.naming;
 
+import software.amazon.awssdk.codegen.model.intermediate.MemberModel;
 import software.amazon.awssdk.codegen.model.service.Shape;
+import software.amazon.awssdk.core.protocol.SdkField;
 
 /**
  * Strategy to name various Java constructs based on the naming in the model and potentially customizations.
@@ -143,4 +145,12 @@ public interface NamingStrategy {
      * @return Name of the getter method for an enum model class member.
      */
     String getFluentEnumSetterMethodName(String memberName, Shape shape);
+
+    /**
+     * Stuttering is intentional, returns the name of the {@link SdkField} field.
+     *
+     * @param memberModel Member to generate field name for.
+     * @return Name of field for {@link SdkField} pojo.
+     */
+    String getSdkFieldFieldName(MemberModel memberModel);
 }

codegen/src/main/java/software/amazon/awssdk/codegen/poet/client/specs/JsonProtocolSpec.java

@@ -104,7 +104,6 @@ public MethodSpec initProtocolFactory(IntermediateModel model) {
 
     @Override
     public CodeBlock responseHandler(IntermediateModel model, OperationModel opModel) {
-        ClassName unmarshaller = getUnmarshallerType(opModel);
         // TODO for rest-json we need to use the real response handler since response members will be bound to headers, for
         // aws-json we need to have a dummy response handler since the response members will be in the initial-response
         // event message
@@ -116,14 +115,14 @@ public CodeBlock responseHandler(IntermediateModel model, OperationModel opModel
             .builder()
             .add("\n\n$T<$T> responseHandler = $L.createResponseHandler(new $T()" +
                  "                                   .withPayloadJson($L)" +
-                 "                                   .withHasStreamingSuccessResponse($L), new $T());",
+                 "                                   .withHasStreamingSuccessResponse($L), $T::builder);",
                  HttpResponseHandler.class,
                  pojoResponseType,
                  protocolFactory,
                  JsonOperationMetadata.class,
                  !opModel.getHasBlobMemberAsPayload(),
                  opModel.hasStreamingOutput(),
-                 unmarshaller);
+                 pojoResponseType);
         if (opModel.hasEventStreamOutput()) {
             builder.add("\n\n$T<$T> voidResponseHandler = $L.createResponseHandler(new $T()" +
                         "                                   .withPayloadJson(false)" +
@@ -314,10 +313,6 @@ private String eventStreamOutputWhenComplete(String responseHandlerName) {
                              + "})", responseHandlerName);
     }
 
-    private ClassName getUnmarshallerType(OperationModel opModel) {
-        return poetExtensions.getTransformClass(opModel.getReturnType().getReturnType() + "Unmarshaller");
-    }
-
     /**
      * Gets the POJO response type for the operation.
      *