Merge pull request #651 from aws/h2

Merging H2 changes to master

Author: shorea

build-tools/src/main/resources/software/amazon/awssdk/checkstyle-suppressions.xml

@@ -23,6 +23,10 @@
     <suppress checks=".*"
               files=".*[\\/](test|it)[\\/]java[\\/].+\.java$"/>
 
+    <!-- TODO: Disable these suppressions. -->
+    <suppress checks=".*"
+              files=".(flow)[\\/].+\.java$"/>
+
     <!-- ignore missing annotation checks under test/codegen directory -->
     <suppress checks="MissingSdkAnnotationCheck"
               files=".(codegen|test)[\\/].+\.java$"/>
@@ -31,6 +35,14 @@
     <suppress checks="MissingSdkAnnotationCheck"
               files=".(services)[\\/].+\.java$"/>
 
+    <!-- TODO want to contribute this back to Netty -->
+    <suppress checks=".*"
+              files=".*BetterFixedChannelPool\.java$"/>
+
+    <!-- TODO want to contribute this back to Netty -->
+    <suppress checks=".*"
+              files=".*[\\/]io[\\/]netty[\\/]handler[\\/]codec[\\/]http2[\\/].*\.java$"/>
+
     <!-- Assumes getter/setter match JSON property -->
     <suppress checks="AbbreviationAsWordInName"
               files=".*[\\/]software[\\/]amazon[\\/]awssdk[\\/]services[\\/]s3[\\/]event[\\/]S3EventNotification.java$"/>

build-tools/src/main/resources/software/amazon/awssdk/spotbugs-suppressions.xml

@@ -61,13 +61,6 @@
         <Bug pattern="URF_UNREAD_FIELD,DLS_DEAD_LOCAL_STORE,REC_CATCH_EXCEPTION" />
     </Match>
 
-    <!-- NP_NONNULL_PARAM_VIOLATION false postive - https://github.com/spotbugs/spotbugs/issues/484 -->
-    <Match>
-        <Class name="software.amazon.awssdk.core.pagination.async.SequentialSubscriber" />
-        <Method name="onComplete" />
-        <Bug pattern="NP_NONNULL_PARAM_VIOLATION" />
-    </Match>
-
     <!-- False positive -->
     <Match>
         <Class name="software.amazon.awssdk.core.ResponseBytes" />
@@ -103,4 +96,12 @@
         <Bug pattern="RI_REDUNDANT_INTERFACES" />
     </Match>
 
+    <Match>
+        <Or>
+            <Package name="io.netty.handler.codec.http2"/>
+            <Class name="software.amazon.awssdk.http.nio.netty.internal.utils.BetterFixedChannelPool" />
+        </Or>
+        <Bug pattern="BC_UNCONFIRMED_CAST,SIC_INNER_SHOULD_BE_STATIC_ANON,DLS_DEAD_LOCAL_STORE,DM_CONVERT_CASE,NM_CLASS_NOT_EXCEPTION,NP_NULL_ON_SOME_PATH"/>
+    </Match>
+
 </FindBugsFilter>

bundle/pom.xml

@@ -74,6 +74,7 @@
                             <include>commons-codec:commons-codec</include>
                             <include>software.amazon.ion:ion-java</include>
                             <include>software.amazon.awssdk:*</include>
+                            <include>software.amazon:*</include>
                             <include>commons-logging:*</include>
                         </includes>
                     </artifactSet>

codegen/pom.xml

@@ -40,6 +40,11 @@
             <artifactId>annotations</artifactId>
             <version>${awsjavasdk.version}</version>
         </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>http-client-spi</artifactId>
+            <version>${awsjavasdk.version}</version>
+        </dependency>
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>regions</artifactId>

codegen/src/main/java/software/amazon/awssdk/codegen/AddMetadata.java

@@ -97,7 +97,8 @@ public static Metadata constructMetadata(ServiceModel serviceModel,
                 .withSigningName(serviceMetadata.getSigningName())
                 .withAuthType(AuthType.fromValue(serviceMetadata.getSignatureVersion()))
                 .withRequiresApiKey(requiresApiKey(serviceModel))
-                .withUid(serviceMetadata.getUid());
+                .withUid(serviceMetadata.getUid())
+                .withSupportsH2(supportsH2(serviceMetadata));
 
         final String jsonVersion = getJsonVersion(metadata, serviceMetadata);
         metadata.setJsonVersion(jsonVersion);
@@ -108,6 +109,10 @@ public static Metadata constructMetadata(ServiceModel serviceModel,
         return metadata;
     }
 
+    private static boolean supportsH2(ServiceMetadata serviceMetadata) {
+        return serviceMetadata.getProtocolSettings() != null && serviceMetadata.getProtocolSettings().containsKey("h2");
+    }
+
     private static String getJsonVersion(Metadata metadata, ServiceMetadata serviceMetadata) {
         // TODO this should be defaulted in the C2J build tool
         if (serviceMetadata.getJsonVersion() == null && metadata.isJsonProtocol()) {

codegen/src/main/java/software/amazon/awssdk/codegen/AddModelShapes.java

@@ -53,7 +53,7 @@ private Map<String, ShapeModel> constructModelShapes(Set<String> shapesToSkip) {
             ShapeType shapeType = getModelShapeType(shape);
 
             if (shapeType != null) {
-                final String javaClassName = getNamingStrategy().getJavaClassName(shapeName);
+                String javaClassName = getNamingStrategy().getJavaClassName(shapeName);
                 if (shapesToSkip.contains(javaClassName)) {
                     continue;
                 }
@@ -77,6 +77,9 @@ private Map<String, ShapeModel> constructModelShapes(Set<String> shapesToSkip) {
      * @return null if the given shape is neither a structure nor enum model.
      */
     private ShapeType getModelShapeType(final Shape shape) {
+        if (shape.isException()) {
+            return null;
+        }
         if (isStructure(shape)) {
             return ShapeType.Model;
 

codegen/src/main/java/software/amazon/awssdk/codegen/AddShapes.java

@@ -80,6 +80,8 @@ protected final ShapeModel generateShapeModel(String javaClassName, String shape
         shapeModel.setRequired(shape.getRequired());
         shapeModel.setDeprecated(shape.isDeprecated());
         shapeModel.setWrapper(shape.isWrapper());
+        shapeModel.withIsEventStream(shape.isEventStream());
+        shapeModel.withIsEvent(shape.isEvent());
 
         boolean hasHeaderMember = false;
         boolean hasStatusCodeMember = false;

codegen/src/main/java/software/amazon/awssdk/codegen/docs/PaginationDocs.java

@@ -25,7 +25,7 @@
 import software.amazon.awssdk.codegen.model.intermediate.OperationModel;
 import software.amazon.awssdk.codegen.poet.PoetExtensions;
 import software.amazon.awssdk.codegen.utils.PaginatorUtils;
-import software.amazon.awssdk.core.pagination.async.SequentialSubscriber;
+import software.amazon.awssdk.utils.async.SequentialSubscriber;
 
 public class PaginationDocs {
 

codegen/src/main/java/software/amazon/awssdk/codegen/emitters/tasks/AwsGeneratorTasks.java

@@ -35,7 +35,8 @@ public AwsGeneratorTasks(GeneratorTaskParams params) {
     private Iterable<GeneratorTask> createAwsTasks(GeneratorTaskParams params) {
         // TODO Move AsyncClientGeneratorTasks to common generic tasks (mostly CommonGeneratorTasks class)
         return new CompositeIterable<>(new AsyncClientGeneratorTasks(params),
-                                       new PaginatorsGeneratorTasks(params));
+                                       new PaginatorsGeneratorTasks(params),
+                                       new EventStreamGeneratorTasks(params));
     }
 
     @Override

codegen/src/main/java/software/amazon/awssdk/codegen/emitters/tasks/EventStreamGeneratorTasks.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright 2010-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.codegen.emitters.tasks;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import software.amazon.awssdk.codegen.emitters.GeneratorTask;
+import software.amazon.awssdk.codegen.emitters.GeneratorTaskParams;
+import software.amazon.awssdk.codegen.emitters.PoetGeneratorTask;
+import software.amazon.awssdk.codegen.model.intermediate.OperationModel;
+import software.amazon.awssdk.codegen.poet.ClassSpec;
+import software.amazon.awssdk.codegen.poet.eventstream.EventStreamResponseHandlerBuilderImplSpec;
+import software.amazon.awssdk.codegen.poet.eventstream.EventStreamResponseHandlerSpec;
+import software.amazon.awssdk.codegen.poet.eventstream.EventStreamUtils;
+import software.amazon.awssdk.codegen.poet.eventstream.EventStreamVisitorBuilderImplSpec;
+
+/**
+ * Generator tasks for event streaming operations.
+ */
+class EventStreamGeneratorTasks extends BaseGeneratorTasks {
+
+    private final GeneratorTaskParams params;
+
+    EventStreamGeneratorTasks(GeneratorTaskParams dependencies) {
+        super(dependencies);
+        this.params = dependencies;
+    }
+
+    @Override
+    protected List<GeneratorTask> createTasks() throws Exception {
+        info("Emitting event stream classes");
+
+        String fileHeader = model.getFileHeader();
+        String modelDirectory = params.getPathProvider().getModelDirectory();
+        return model.getOperations().values().stream()
+                    .filter(OperationModel::hasEventStreamOutput)
+                    .flatMap(this::eventStreamClassSpecs)
+                    .map(spec -> new PoetGeneratorTask(modelDirectory, fileHeader, spec))
+                    .collect(Collectors.toList());
+    }
+
+    private Stream<ClassSpec> eventStreamClassSpecs(OperationModel opModel) {
+        EventStreamUtils eventStreamUtils = EventStreamUtils.create(params.getPoetExtensions(), opModel);
+        return Stream.of(
+            new EventStreamResponseHandlerSpec(params, eventStreamUtils),
+            new EventStreamResponseHandlerBuilderImplSpec(params, eventStreamUtils),
+            new EventStreamVisitorBuilderImplSpec(params, eventStreamUtils));
+    }
+}

codegen/src/main/java/software/amazon/awssdk/codegen/emitters/tasks/MarshallerGeneratorTasks.java

@@ -61,11 +61,10 @@ private boolean shouldGenerate(ShapeModel shapeModel) {
             info("Skip generating marshaller class for " + shapeModel.getShapeName());
             return false;
         }
-        return shouldGenerate(shapeModel.getShapeType());
-    }
-
-    private boolean shouldGenerate(ShapeType shapeType) {
-        return ShapeType.Request == shapeType || (ShapeType.Model == shapeType && metadata.isJsonProtocol());
+        ShapeType shapeType = shapeModel.getShapeType();
+        return (ShapeType.Request == shapeType || (ShapeType.Model == shapeType && metadata.isJsonProtocol()))
+               // The event stream shape is a container for event subtypes and isn't something that needs to ever be marshalled
+               && !shapeModel.isEventStream();
     }
 
     private Stream<GeneratorTask> createTask(String javaShapeName, ShapeModel shapeModel) throws Exception {

codegen/src/main/java/software/amazon/awssdk/codegen/emitters/tasks/UnmarshallerGeneratorTasks.java

@@ -85,14 +85,11 @@ private boolean shouldGenerate(ShapeModel shapeModel) {
             info("Skip generating unmarshaller class for " + shapeModel.getShapeName());
             return false;
         }
-        return shouldGenerate(shapeModel.getShapeType());
-    }
-
-    private boolean shouldGenerate(ShapeType shapeType) {
-        switch (shapeType) {
+        switch (shapeModel.getShapeType()) {
             case Response:
             case Model:
-                return true;
+                // The event stream shape is a container for event subtypes and isn't something that needs to ever be unmarshalled
+                return !shapeModel.isEventStream();
             case Exception:
                 // Generating Exception Unmarshallers is not required for the JSON protocol
                 return !metadata.isJsonProtocol();

codegen/src/main/java/software/amazon/awssdk/codegen/model/intermediate/MemberModel.java

@@ -553,5 +553,4 @@ private void appendParagraph(StringBuilder builder, String content, Object... co
                .append("</p>")
                .append(LF);
     }
-
 }

codegen/src/main/java/software/amazon/awssdk/codegen/model/intermediate/Metadata.java

@@ -103,6 +103,8 @@ public class Metadata {
 
     private String baseResponseName;
 
+    private boolean supportsH2;
+
     public String getApiVersion() {
         return apiVersion;
     }
@@ -717,4 +719,12 @@ public Metadata withBaseResponseName(String baseResponseName) {
     private String joinPackageNames(String lhs, String rhs) {
         return StringUtils.isBlank(rhs) ? lhs : lhs + '.' + rhs;
     }
+
+    public boolean supportsH2() {
+        return supportsH2;
+    }
+
+    public void withSupportsH2(boolean supportsH2) {
+        this.supportsH2 = supportsH2;
+    }
 }

codegen/src/main/java/software/amazon/awssdk/codegen/model/intermediate/OperationModel.java

@@ -188,4 +188,16 @@ public boolean isPaginated() {
     public void setPaginated(boolean paginated) {
         isPaginated = paginated;
     }
+
+    /**
+     * @return True if the operation has an event stream member in the output shape. False otherwise.
+     */
+    public boolean hasEventStreamOutput() {
+        return outputShape != null
+               && outputShape.getMembers() != null
+               && outputShape.getMembers().stream()
+                             .filter(m -> m.getShape() != null)
+                             .anyMatch(m -> m.getShape().isEventStream());
+    }
+
 }

codegen/src/main/java/software/amazon/awssdk/codegen/model/intermediate/ShapeModel.java

@@ -59,6 +59,10 @@ public class ShapeModel extends DocumentationModel implements HasDeprecation {
 
     private ShapeCustomizationInfo customization = new ShapeCustomizationInfo();
 
+    private boolean isEventStream;
+
+    private boolean isEvent;
+
     public ShapeModel(@JsonProperty("c2jName") String c2jName) {
         this.c2jName = c2jName;
     }
@@ -272,7 +276,10 @@ public List<MemberModel> getMembers() {
      */
     public List<MemberModel> getNonStreamingMembers() {
         return getMembers().stream()
+                           // Filter out binary streaming members
                            .filter(m -> !m.getHttp().getIsStreaming())
+                           // Filter out event stream members (if shape is null then it's primitive and we should include it).
+                           .filter(m -> m.getShape() == null || !m.getShape().isEventStream)
                            .collect(Collectors.toList());
     }
 
@@ -449,4 +456,29 @@ public void setRequestSignerClassFqcn(String authorizerClass) {
         this.requestSignerClassFqcn = authorizerClass;
     }
 
+    /**
+     * @return True if the shape is an 'eventstream' shape. The eventstream shape is the tagged union like
+     * container that holds individual 'events'.
+     */
+    public boolean isEventStream() {
+        return this.isEventStream;
+    }
+
+    public ShapeModel withIsEventStream(boolean isEventStream) {
+        this.isEventStream = isEventStream;
+        return this;
+    }
+
+    /**
+     * @return True if the shape is an 'event'. I.E. It is a member of the eventstream and represents one logical event
+     * that can be delivered on the event stream.
+     */
+    public boolean isEvent() {
+        return this.isEvent;
+    }
+
+    public ShapeModel withIsEvent(boolean isEvent) {
+        this.isEvent = isEvent;
+        return this;
+    }
 }

codegen/src/main/java/software/amazon/awssdk/codegen/model/service/ServiceMetadata.java

@@ -15,6 +15,8 @@
 
 package software.amazon.awssdk.codegen.model.service;
 
+import java.util.Map;
+
 public class ServiceMetadata {
 
     private String apiVersion;
@@ -43,6 +45,8 @@ public class ServiceMetadata {
 
     private String uid;
 
+    private Map<String, String> protocolSettings;
+
     public String getApiVersion() {
         return apiVersion;
     }
@@ -142,6 +146,14 @@ public void setUid(String uid) {
         this.uid = uid;
     }
 
+    public Map<String, String> getProtocolSettings() {
+        return protocolSettings;
+    }
+
+    public void setProtocolSettings(Map<String, String> protocolSettings) {
+        this.protocolSettings = protocolSettings;
+    }
+
     public String getServiceId() {
         return serviceId;
     }