s3: handle unrecognized values for Expires in responses (#2653)

Author: lucix-aws

.changelog/384878677f9a4885825e6a4cc2b1012f.json

@@ -0,0 +1,8 @@
+{
+    "id": "38487867-7f9a-4885-825e-6a4cc2b1012f",
+    "type": "bugfix",
+    "description": "Prevent parsing failures for nonstandard `Expires` values in responses. If the SDK cannot parse the value set in the response header for this field it will now be returned as `nil`. A new field, `ExpiresString`, has been added that will retain the unparsed value from the response (regardless of whether it came back in a format recognized by the SDK).",
+    "modules": [
+        "service/s3"
+    ]
+}

codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/customization/s3/S3ExpiresShapeCustomization.java

@@ -0,0 +1,128 @@
+/*
+ * Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.smithy.aws.go.codegen.customization.s3;
+
+import static software.amazon.smithy.aws.go.codegen.customization.S3ModelUtils.isServiceS3;
+import static software.amazon.smithy.go.codegen.GoWriter.goTemplate;
+import static software.amazon.smithy.go.codegen.SymbolUtils.buildPackageSymbol;
+
+import java.util.List;
+import software.amazon.smithy.codegen.core.SymbolProvider;
+import software.amazon.smithy.go.codegen.GoDelegator;
+import software.amazon.smithy.go.codegen.GoSettings;
+import software.amazon.smithy.go.codegen.GoWriter;
+import software.amazon.smithy.go.codegen.SmithyGoDependency;
+import software.amazon.smithy.go.codegen.integration.GoIntegration;
+import software.amazon.smithy.go.codegen.integration.RuntimeClientPlugin;
+import software.amazon.smithy.model.Model;
+import software.amazon.smithy.model.shapes.MemberShape;
+import software.amazon.smithy.model.shapes.Shape;
+import software.amazon.smithy.model.shapes.ShapeId;
+import software.amazon.smithy.model.shapes.StringShape;
+import software.amazon.smithy.model.traits.DeprecatedTrait;
+import software.amazon.smithy.model.traits.DocumentationTrait;
+import software.amazon.smithy.model.traits.HttpHeaderTrait;
+import software.amazon.smithy.model.traits.OutputTrait;
+import software.amazon.smithy.model.transform.ModelTransformer;
+import software.amazon.smithy.utils.MapUtils;
+
+/**
+ * Restrictions around timestamp formatting for the 'Expires' value in some S3 responses has never been standardized and
+ * thus many non-conforming values for the field (unsupported formats, arbitrary strings, etc.) exist in the wild. This
+ * customization makes the response parsing forgiving for this field in responses and adds an ExpiresString field that
+ * contains the unparsed value.
+ */
+public class S3ExpiresShapeCustomization implements GoIntegration {
+    private static final ShapeId S3_EXPIRES = ShapeId.from("com.amazonaws.s3#Expires");
+    private static final ShapeId S3_EXPIRES_STRING = ShapeId.from("com.amazonaws.s3#ExpiresString");
+    private static final String DESERIALIZE_S3_EXPIRES = "deserializeS3Expires";
+
+    @Override
+    public List<RuntimeClientPlugin> getClientPlugins() {
+        return List.of(RuntimeClientPlugin.builder()
+                .addShapeDeserializer(S3_EXPIRES, buildPackageSymbol(DESERIALIZE_S3_EXPIRES))
+                .build());
+    }
+
+    @Override
+    public Model preprocessModel(Model model, GoSettings settings) {
+        if (!isServiceS3(model, settings.getService(model))) {
+            return model;
+        }
+
+        var withExpiresString = model.toBuilder()
+                .addShape(StringShape.builder()
+                        .id(S3_EXPIRES_STRING)
+                        .build())
+                .build();
+        return ModelTransformer.create().mapShapes(withExpiresString, this::addExpiresString);
+    }
+
+    @Override
+    public void writeAdditionalFiles(GoSettings settings, Model model, SymbolProvider symbolProvider, GoDelegator goDelegator) {
+        goDelegator.useFileWriter("deserializers.go", settings.getModuleName(), deserializeS3Expires());
+    }
+
+    private Shape addExpiresString(Shape shape) {
+        if (!shape.hasTrait(OutputTrait.class)) {
+            return shape;
+        }
+
+        var expires = shape.getMember(S3_EXPIRES.getName());
+        if (expires.isEmpty()) {
+            return shape;
+        }
+
+        if (!expires.get().getTarget().equals(S3_EXPIRES)) {
+            return shape;
+        }
+
+        var deprecated = DeprecatedTrait.builder()
+                .message("This field is handled inconsistently across AWS SDKs. Prefer using the ExpiresString field " +
+                        "which contains the unparsed value from the service response.")
+                .build();
+        var stringDocs = new DocumentationTrait("The unparsed value of the Expires field from the service " +
+                "response. Prefer use of this value over the normal Expires response field where possible.");
+        return Shape.shapeToBuilder(shape)
+                .addMember(expires.get().toBuilder()
+                        .addTrait(deprecated)
+                        .build())
+                .addMember(MemberShape.builder()
+                        .id(shape.getId().withMember(S3_EXPIRES_STRING.getName()))
+                        .target(S3_EXPIRES_STRING)
+                        .addTrait(expires.get().expectTrait(HttpHeaderTrait.class)) // copies header name
+                        .addTrait(stringDocs)
+                        .build())
+                .build();
+    }
+
+    private GoWriter.Writable deserializeS3Expires() {
+        return goTemplate("""
+                func $name:L(v string) ($time:P, error) {
+                    t, err := $parseHTTPDate:T(v)
+                    if err != nil {
+                        return nil, nil
+                    }
+                    return &t, nil
+                }
+                """,
+                MapUtils.of(
+                        "name", DESERIALIZE_S3_EXPIRES,
+                        "time", SmithyGoDependency.TIME.struct("Time"),
+                        "parseHTTPDate", SmithyGoDependency.SMITHY_TIME.func("ParseHTTPDate")
+                ));
+    }
+}

codegen/smithy-aws-go-codegen/src/main/resources/META-INF/services/software.amazon.smithy.go.codegen.integration.GoIntegration

@@ -75,3 +75,4 @@ software.amazon.smithy.aws.go.codegen.customization.auth.GlobalAnonymousOption
 software.amazon.smithy.aws.go.codegen.customization.CloudFrontKVSSigV4a
 software.amazon.smithy.aws.go.codegen.customization.BackfillProtocolTestServiceTrait
 software.amazon.smithy.go.codegen.integration.MiddlewareStackSnapshotTests
+software.amazon.smithy.aws.go.codegen.customization.s3.S3ExpiresShapeCustomization

service/s3/api_op_GetObject.go

@@ -0,0 +1,75 @@
+package s3
+
+import (
+	"context"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+)
+
+type mockHeadObject struct {
+	expires string
+}
+
+func (m *mockHeadObject) Do(r *http.Request) (*http.Response, error) {
+	return &http.Response{
+		StatusCode: 200,
+		Header: http.Header{
+			"Expires": {m.expires},
+		},
+		Body: http.NoBody,
+	}, nil
+}
+
+func TestInvalidExpires(t *testing.T) {
+	expires := "2023-11-01"
+	svc := New(Options{
+		HTTPClient: &mockHeadObject{expires},
+		Region:     "us-east-1",
+	})
+
+	out, err := svc.HeadObject(context.Background(), &HeadObjectInput{
+		Bucket: aws.String("bucket"),
+		Key:    aws.String("key"),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if out.Expires != nil {
+		t.Errorf("out.Expires should be nil, is %s", *out.Expires)
+	}
+	if aws.ToString(out.ExpiresString) != expires {
+		t.Errorf("out.ExpiresString should be %s, is %s", expires, *out.ExpiresString)
+	}
+}
+
+func TestValidExpires(t *testing.T) {
+	exs := "Mon, 02 Jan 2006 15:04:05 GMT"
+	ext, err := time.Parse(exs, exs)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	svc := New(Options{
+		HTTPClient: &mockHeadObject{exs},
+		Region:     "us-east-1",
+	})
+
+	out, err := svc.HeadObject(context.Background(), &HeadObjectInput{
+		Bucket: aws.String("bucket"),
+		Key:    aws.String("key"),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if aws.ToTime(out.Expires) != ext {
+		t.Errorf("out.Expires should be %s, is %s", ext, *out.Expires)
+	}
+	if aws.ToString(out.ExpiresString) != exs {
+		t.Errorf("out.ExpiresString should be %s, is %s", exs, *out.ExpiresString)
+	}
+}