Skip to content

Commit fd413c5

Browse files
kuheAllanZhengYPtrivikr
authored
Stream improvement serde (#593)
* generate util functions to consume response stream * inject sdk stream utility function to the stream response * add SDKStreamSerdeContext interface * add internal trait to the sdkStreamMixin config * add missing new line for default modes config * revert SdkStream type in SymbolVisitor, add to command generator * only mixin stream utils in client SDK * feat(serde): use type-mapping to apply stream mixin instead of omit * feat(serde): use ternary Co-authored-by: Trivikram Kamat <[email protected]> * Update AddSdkStreamMixinDependency.java Co-authored-by: AllanZhengYP <[email protected]> Co-authored-by: Trivikram Kamat <[email protected]>
1 parent ab6066c commit fd413c5

File tree

9 files changed

+198
-31
lines changed

9 files changed

+198
-31
lines changed

‎smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/CodegenUtils.java

Lines changed: 49 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import software.amazon.smithy.model.shapes.Shape;
2929
import software.amazon.smithy.model.shapes.StructureShape;
3030
import software.amazon.smithy.model.traits.StreamingTrait;
31+
import software.amazon.smithy.typescript.codegen.integration.AddSdkStreamMixinDependency;
3132
import software.amazon.smithy.utils.SmithyUnstableApi;
3233

3334
/**
@@ -75,12 +76,14 @@ public static String getOperationSerializerContextType(
7576

7677
/**
7778
* Get context type for command deserializer function.
79+
* @param settings The TypeScript settings
7880
* @param writer The code writer.
7981
* @param model The model for the service containing the given command.
8082
* @param operation The operation shape for given command.
8183
* @return The TypeScript type for the deserializer context
8284
*/
8385
public static String getOperationDeserializerContextType(
86+
TypeScriptSettings settings,
8487
TypeScriptWriter writer,
8588
Model model,
8689
OperationShape operation
@@ -90,9 +93,15 @@ public static String getOperationDeserializerContextType(
9093
// If event stream trait exists, add corresponding serde context type to the intersection type.
9194
EventStreamIndex eventStreamIndex = EventStreamIndex.of(model);
9295
if (eventStreamIndex.getOutputInfo(operation).isPresent()) {
93-
writer.addImport("EventStreamSerdeContext", "__EventStreamSerdeContext", "@aws-sdk/types");
96+
writer.addImport("EventStreamSerdeContext", "__EventStreamSerdeContext",
97+
TypeScriptDependency.AWS_SDK_TYPES.packageName);
9498
contextInterfaceList.add("__EventStreamSerdeContext");
9599
}
100+
if (AddSdkStreamMixinDependency.hasStreamingBlobDeser(settings, model, operation)) {
101+
writer.addImport("SdkStreamSerdeContext", "__SdkStreamSerdeContext",
102+
TypeScriptDependency.AWS_SDK_TYPES.packageName);
103+
contextInterfaceList.add("__SdkStreamSerdeContext");
104+
}
96105
return String.join(" & ", contextInterfaceList);
97106
}
98107

@@ -108,37 +117,67 @@ static List<MemberShape> getBlobStreamingMembers(Model model, StructureShape sha
108117
return shape.getAllMembers().values().stream()
109118
.filter(memberShape -> {
110119
// Streaming blobs need to have their types modified
111-
// See `writeStreamingMemberType`
120+
// See `writeClientCommandStreamingInputType`
112121
Shape target = model.expectShape(memberShape.getTarget());
113122
return target.isBlobShape() && target.hasTrait(StreamingTrait.class);
114123
})
115124
.collect(Collectors.toList());
116125
}
117126

118127
/**
119-
* Ease the input streaming member restriction so that users don't need to construct a stream every time.
120-
* This type decoration is allowed in Smithy because it makes input type more permissive than output type
121-
* for the same member.
128+
* Generate the type of the command input of the client sdk given the streaming blob
129+
* member of the shape. The generated type eases the streaming member requirement so that users don't need to
130+
* construct a stream every time.
131+
* This type decoration is allowed in Smithy because it makes, for the same member, the type to be serialized is
132+
* more permissive than the type to be deserialized.
122133
* Refer here for more rationales: https://github.com/aws/aws-sdk-js-v3/issues/843
123134
*/
124-
static void writeStreamingMemberType(
135+
static void writeClientCommandStreamingInputType(
125136
TypeScriptWriter writer,
126137
Symbol containerSymbol,
127138
String typeName,
128139
MemberShape streamingMember
129140
) {
130141
String memberName = streamingMember.getMemberName();
131142
String optionalSuffix = streamingMember.isRequired() ? "" : "?";
132-
writer.openBlock("type $LType = Omit<$T, $S> & {", "};", typeName, containerSymbol, memberName, () -> {
133-
writer.writeDocs(String.format("For *`%1$s[\"%2$s\"]`*, see {@link %1$s.%2$s}.",
134-
containerSymbol.getName(), memberName));
135-
writer.write("$1L$2L: $3T[$1S]|string|Uint8Array|Buffer;", memberName, optionalSuffix, containerSymbol);
143+
writer.openBlock("type $LType = Omit<$T, $S> & {", "};", typeName,
144+
containerSymbol, memberName, () -> {
145+
writer.writeDocs(String.format("For *`%1$s[\"%2$s\"]`*, see {@link %1$s.%2$s}.",
146+
containerSymbol.getName(), memberName));
147+
writer.write("$1L$2L: $3T[$1S]|string|Uint8Array|Buffer;", memberName, optionalSuffix,
148+
containerSymbol);
136149
});
137150
writer.writeDocs(String.format("This interface extends from `%1$s` interface. There are more parameters than"
138151
+ " `%2$s` defined in {@link %1$s}", containerSymbol.getName(), memberName));
139152
writer.write("export interface $1L extends $1LType {}", typeName);
140153
}
141154

155+
/**
156+
* Generate the type of the command output of the client sdk given the streaming blob
157+
* member of the shape. The type marks the streaming blob member to contain the utility methods to transform the
158+
* stream to string, buffer or WHATWG stream API.
159+
*/
160+
static void writeClientCommandStreamingOutputType(
161+
TypeScriptWriter writer,
162+
Symbol containerSymbol,
163+
String typeName,
164+
MemberShape streamingMember
165+
) {
166+
String memberName = streamingMember.getMemberName();
167+
String optionalSuffix = streamingMember.isRequired() ? "" : "?";
168+
writer.addImport("MetadataBearer", "__MetadataBearer", TypeScriptDependency.AWS_SDK_TYPES.packageName);
169+
writer.addImport("SdkStream", "__SdkStream", TypeScriptDependency.AWS_SDK_TYPES.packageName);
170+
writer.addImport("WithSdkStreamMixin", "__WithSdkStreamMixin", TypeScriptDependency.AWS_SDK_TYPES.packageName);
171+
172+
173+
writer.write(
174+
"export interface $L extends __WithSdkStreamMixin<$T, $S>, __MetadataBearer {}",
175+
typeName,
176+
containerSymbol,
177+
memberName
178+
);
179+
}
180+
142181
/**
143182
* Returns the list of function parameter key-value pairs to be written for
144183
* provided parameters map.

‎smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/CommandGenerator.java

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@
1616
package software.amazon.smithy.typescript.codegen;
1717

1818
import static software.amazon.smithy.typescript.codegen.CodegenUtils.getBlobStreamingMembers;
19-
import static software.amazon.smithy.typescript.codegen.CodegenUtils.writeStreamingMemberType;
19+
import static software.amazon.smithy.typescript.codegen.CodegenUtils.writeClientCommandStreamingInputType;
20+
import static software.amazon.smithy.typescript.codegen.CodegenUtils.writeClientCommandStreamingOutputType;
2021

2122
import java.nio.file.Paths;
2223
import java.util.List;
@@ -314,7 +315,8 @@ private void writeInputType(String typeName, Optional<StructureShape> inputShape
314315
if (blobStreamingMembers.isEmpty()) {
315316
writer.write("export interface $L extends $T {}", typeName, symbolProvider.toSymbol(input));
316317
} else {
317-
writeStreamingMemberType(writer, symbolProvider.toSymbol(input), typeName, blobStreamingMembers.get(0));
318+
writeClientCommandStreamingInputType(writer, symbolProvider.toSymbol(input), typeName,
319+
blobStreamingMembers.get(0));
318320
}
319321
} else {
320322
// If the input is non-existent, then use an empty object.
@@ -327,8 +329,15 @@ private void writeOutputType(String typeName, Optional<StructureShape> outputSha
327329
// to a defined output shape.
328330
writer.addImport("MetadataBearer", "__MetadataBearer", TypeScriptDependency.AWS_SDK_TYPES.packageName);
329331
if (outputShape.isPresent()) {
330-
writer.write("export interface $L extends $T, __MetadataBearer {}",
331-
typeName, symbolProvider.toSymbol(outputShape.get()));
332+
StructureShape output = outputShape.get();
333+
List<MemberShape> blobStreamingMembers = getBlobStreamingMembers(model, output);
334+
if (blobStreamingMembers.isEmpty()) {
335+
writer.write("export interface $L extends $T, __MetadataBearer {}",
336+
typeName, symbolProvider.toSymbol(outputShape.get()));
337+
} else {
338+
writeClientCommandStreamingOutputType(writer, symbolProvider.toSymbol(output), typeName,
339+
blobStreamingMembers.get(0));
340+
}
332341
} else {
333342
writer.write("export interface $L extends __MetadataBearer {}", typeName);
334343
}
@@ -371,7 +380,8 @@ private void writeSerde() {
371380
.write("private deserialize(")
372381
.indent()
373382
.write("output: $T,", applicationProtocol.getResponseType())
374-
.write("context: $L", CodegenUtils.getOperationDeserializerContextType(writer, model, operation))
383+
.write("context: $L",
384+
CodegenUtils.getOperationDeserializerContextType(settings, writer, model, operation))
375385
.dedent()
376386
.openBlock("): Promise<$T> {", "}", outputType, () -> writeSerdeDispatcher(false))
377387
.write("");

‎smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/ServerCommandGenerator.java

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,6 @@
1515

1616
package software.amazon.smithy.typescript.codegen;
1717

18-
import static software.amazon.smithy.typescript.codegen.CodegenUtils.getBlobStreamingMembers;
19-
import static software.amazon.smithy.typescript.codegen.CodegenUtils.writeStreamingMemberType;
20-
2118
import java.nio.file.Paths;
2219
import java.util.Collections;
2320
import java.util.Iterator;
@@ -31,7 +28,6 @@
3128
import software.amazon.smithy.model.Model;
3229
import software.amazon.smithy.model.knowledge.OperationIndex;
3330
import software.amazon.smithy.model.knowledge.TopDownIndex;
34-
import software.amazon.smithy.model.shapes.MemberShape;
3531
import software.amazon.smithy.model.shapes.OperationShape;
3632
import software.amazon.smithy.model.shapes.ServiceShape;
3733
import software.amazon.smithy.model.shapes.StructureShape;
@@ -98,16 +94,10 @@ private void addInputAndOutputTypes() {
9894
writer.write("");
9995
}
10096

101-
// TODO: Flip these so that metadata is attached to input and streaming customization is attached to output.
10297
private void writeInputType(String typeName, Optional<StructureShape> inputShape) {
10398
if (inputShape.isPresent()) {
10499
StructureShape input = inputShape.get();
105-
List<MemberShape> blobStreamingMembers = getBlobStreamingMembers(model, input);
106-
if (blobStreamingMembers.isEmpty()) {
107-
writer.write("export interface $L extends $T {}", typeName, symbolProvider.toSymbol(input));
108-
} else {
109-
writeStreamingMemberType(writer, symbolProvider.toSymbol(input), typeName, blobStreamingMembers.get(0));
110-
}
100+
writer.write("export interface $L extends $T {}", typeName, symbolProvider.toSymbol(inputShape.get()));
111101
renderNamespace(typeName, input);
112102
} else {
113103
// If the input is non-existent, then use an empty object.

‎smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/TypeScriptDependency.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,10 @@ public enum TypeScriptDependency implements SymbolDependencyContainer {
100100
XML_PARSER("dependencies", "fast-xml-parser", "4.0.11", false),
101101
HTML_ENTITIES("dependencies", "entities", "2.2.0", false),
102102

103+
// Conditionally added when streaming blob response payload exists.
104+
UTIL_STREAM_NODE("dependencies", "@aws-sdk/util-stream-node", false),
105+
UTIL_STREAM_BROWSER("dependencies", "@aws-sdk/util-stream-browser", false),
106+
103107
// Server dependency for SSDKs
104108
SERVER_COMMON("dependencies", "@aws-smithy/server-common", "1.0.0-alpha.6", false);
105109

‎smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/integration/AddDefaultsModeDependency.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,6 @@ public void addConfigInterfaceFields(
4141
writer.addImport("Provider", "Provider", TypeScriptDependency.AWS_SDK_TYPES.packageName);
4242
writer.writeDocs("The {@link DefaultsMode} that will be used to determine how certain default configuration "
4343
+ "options are resolved in the SDK.");
44-
writer.write("defaultsMode?: DefaultsMode | Provider<DefaultsMode>;");
44+
writer.write("defaultsMode?: DefaultsMode | Provider<DefaultsMode>;\n");
4545
}
4646
}

‎smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/integration/AddSdkStreamMixinDependency.java

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
/*
2+
* Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.smithy.typescript.codegen.integration;
17+
18+
import java.util.Collections;
19+
import java.util.Map;
20+
import java.util.Set;
21+
import java.util.function.Consumer;
22+
import software.amazon.smithy.codegen.core.SymbolProvider;
23+
import software.amazon.smithy.model.Model;
24+
import software.amazon.smithy.model.knowledge.TopDownIndex;
25+
import software.amazon.smithy.model.shapes.BlobShape;
26+
import software.amazon.smithy.model.shapes.MemberShape;
27+
import software.amazon.smithy.model.shapes.OperationShape;
28+
import software.amazon.smithy.model.shapes.ServiceShape;
29+
import software.amazon.smithy.model.shapes.Shape;
30+
import software.amazon.smithy.model.shapes.StructureShape;
31+
import software.amazon.smithy.model.traits.StreamingTrait;
32+
import software.amazon.smithy.typescript.codegen.LanguageTarget;
33+
import software.amazon.smithy.typescript.codegen.TypeScriptDependency;
34+
import software.amazon.smithy.typescript.codegen.TypeScriptSettings;
35+
import software.amazon.smithy.typescript.codegen.TypeScriptWriter;
36+
import software.amazon.smithy.utils.MapUtils;
37+
import software.amazon.smithy.utils.SmithyInternalApi;
38+
39+
/**
40+
* Add runtime config for injecting utility functions to consume the JavaScript
41+
* runtime-specific stream implementations.
42+
*/
43+
@SmithyInternalApi
44+
public final class AddSdkStreamMixinDependency implements TypeScriptIntegration {
45+
46+
@Override
47+
public void addConfigInterfaceFields(
48+
TypeScriptSettings settings,
49+
Model model,
50+
SymbolProvider symbolProvider,
51+
TypeScriptWriter writer
52+
) {
53+
if (!hasStreamingBlobDeser(settings, model)) {
54+
return;
55+
}
56+
57+
writer.addImport("SdkStreamMixinInjector", "__SdkStreamMixinInjector",
58+
TypeScriptDependency.AWS_SDK_TYPES.packageName);
59+
writer.writeDocs("The internal function that inject utilities to runtime-specific stream to help users"
60+
+ " consume the data\n@internal");
61+
writer.write("sdkStreamMixin?: __SdkStreamMixinInjector;\n");
62+
}
63+
64+
@Override
65+
public Map<String, Consumer<TypeScriptWriter>> getRuntimeConfigWriters(
66+
TypeScriptSettings settings,
67+
Model model,
68+
SymbolProvider symbolProvider,
69+
LanguageTarget target
70+
) {
71+
if (!hasStreamingBlobDeser(settings, model)) {
72+
return Collections.emptyMap();
73+
}
74+
switch (target) {
75+
case NODE:
76+
return MapUtils.of("sdkStreamMixin", writer -> {
77+
writer.addDependency(TypeScriptDependency.UTIL_STREAM_NODE);
78+
writer.addImport("sdkStreamMixin", "sdkStreamMixin",
79+
TypeScriptDependency.UTIL_STREAM_NODE.packageName);
80+
writer.write("sdkStreamMixin");
81+
});
82+
case BROWSER:
83+
return MapUtils.of("sdkStreamMixin", writer -> {
84+
writer.addDependency(TypeScriptDependency.UTIL_STREAM_BROWSER);
85+
writer.addImport("sdkStreamMixin", "sdkStreamMixin",
86+
TypeScriptDependency.UTIL_STREAM_BROWSER.packageName);
87+
writer.write("sdkStreamMixin");
88+
});
89+
default:
90+
return Collections.emptyMap();
91+
}
92+
}
93+
94+
private static boolean hasStreamingBlobDeser(TypeScriptSettings settings, Model model) {
95+
ServiceShape serviceShape = settings.getService(model);
96+
TopDownIndex topDownIndex = TopDownIndex.of(model);
97+
Set<OperationShape> operations = topDownIndex.getContainedOperations(serviceShape);
98+
for (OperationShape operation : operations) {
99+
if (hasStreamingBlobDeser(settings, model, operation)) {
100+
return true;
101+
}
102+
}
103+
return false;
104+
}
105+
106+
public static boolean hasStreamingBlobDeser(TypeScriptSettings settings, Model model, OperationShape operation) {
107+
StructureShape ioShapeToDeser = (settings.generateServerSdk())
108+
? model.expectShape(operation.getInputShape()).asStructureShape().get()
109+
: model.expectShape(operation.getOutputShape()).asStructureShape().get();
110+
for (MemberShape member : ioShapeToDeser.members()) {
111+
Shape shape = model.expectShape(member.getTarget());
112+
if (shape instanceof BlobShape && shape.hasTrait(StreamingTrait.class)) {
113+
return true;
114+
}
115+
}
116+
return false;
117+
}
118+
}

‎smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/integration/HttpBindingProtocolGenerator.java

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2024,7 +2024,8 @@ private void generateOperationResponseDeserializer(
20242024
String errorMethodName = methodName + "Error";
20252025
// Add the normalized output type.
20262026
Symbol outputType = symbol.expectProperty("outputType", Symbol.class);
2027-
String contextType = CodegenUtils.getOperationDeserializerContextType(writer, context.getModel(), operation);
2027+
String contextType = CodegenUtils.getOperationDeserializerContextType(context.getSettings(), writer,
2028+
context.getModel(), operation);
20282029

20292030
// Handle the general response.
20302031
writer.openBlock("export const $L = async(\n"
@@ -2320,14 +2321,18 @@ private HttpBinding readPayload(
23202321
HttpBinding binding
23212322
) {
23222323
TypeScriptWriter writer = context.getWriter();
2324+
boolean isClientSdk = context.getSettings().generateClient();
23232325

23242326
// There can only be one payload binding.
23252327
Shape target = context.getModel().expectShape(binding.getMember().getTarget());
23262328

23272329
// Handle streaming shapes differently.
23282330
if (target.hasTrait(StreamingTrait.class)) {
2329-
// If payload is streaming, return raw low-level stream directly.
23302331
writer.write("const data: any = output.body;");
2332+
// If payload is streaming blob, return low-level stream with the stream utility functions mixin.
2333+
if (isClientSdk && target instanceof BlobShape) {
2334+
writer.write("context.sdkStreamMixin(data);");
2335+
}
23312336
} else if (target instanceof BlobShape) {
23322337
// If payload is non-streaming Blob, only need to collect stream to binary data (Uint8Array).
23332338
writer.write("const data: any = await collectBody(output.body, context);");

‎smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/integration/HttpRpcProtocolGenerator.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -380,8 +380,8 @@ private void generateOperationDeserializer(GenerationContext context, OperationS
380380
// e.g., deserializeAws_restJson1_1ExecuteStatement
381381
String methodName = ProtocolGenerator.getDeserFunctionName(symbol, getName());
382382
String errorMethodName = methodName + "Error";
383-
String serdeContextType = CodegenUtils.getOperationDeserializerContextType(writer, context.getModel(),
384-
operation);
383+
String serdeContextType = CodegenUtils.getOperationDeserializerContextType(context.getSettings(), writer,
384+
context.getModel(), operation);
385385
// Add the normalized output type.
386386
Symbol outputType = symbol.expectProperty("outputType", Symbol.class);
387387

‎smithy-typescript-codegen/src/main/resources/META-INF/services/software.amazon.smithy.typescript.codegen.integration.TypeScriptIntegration

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@ software.amazon.smithy.typescript.codegen.integration.AddChecksumRequiredDepende
33
software.amazon.smithy.typescript.codegen.integration.AddDefaultsModeDependency
44
software.amazon.smithy.typescript.codegen.integration.AddHttpApiKeyAuthPlugin
55
software.amazon.smithy.typescript.codegen.integration.AddBaseServiceExceptionClass
6+
software.amazon.smithy.typescript.codegen.integration.AddSdkStreamMixinDependency

0 commit comments

Comments
 (0)