add configuration to preserve path seperator in URIs

sbiscigl · sbiscigl · commit 8ff1bec9b683 · 2024-03-26T10:57:23.000-04:00
diff --git a/src/aws-cpp-sdk-core/include/aws/core/Aws.h b/src/aws-cpp-sdk-core/include/aws/core/Aws.h
@@ -105,6 +105,12 @@ namespace Aws
          * Disable legacy URL encoding that leaves `$&,:@=` unescaped for legacy purposes.
          */
         bool compliantRfc3986Encoding;
+        /**
+         * When constructing Path segments in a URI preserve path separators instead of collapsing
+         * slashes. This is useful for aligning with other SDKs and tools on key path for S3 objects
+         * as currently the C++ SDK sanitizes the path.
+         */
+        bool preservePathSeparators = false;
     };
 
     /**
diff --git a/src/aws-cpp-sdk-core/include/aws/core/http/URI.h b/src/aws-cpp-sdk-core/include/aws/core/http/URI.h
@@ -24,6 +24,9 @@ namespace Aws
         extern bool s_compliantRfc3986Encoding;
         AWS_CORE_API void SetCompliantRfc3986Encoding(bool compliant);
 
+        extern bool s_preservePathSeparators;
+        AWS_CORE_API void SetPreservePathSeparators(bool preservePathSeparators);
+
         //per https://tools.ietf.org/html/rfc3986#section-3.4 there is nothing preventing servers from allowing
         //multiple values for the same key. So use a multimap instead of a map.
         typedef Aws::MultiMap<Aws::String, Aws::String> QueryStringParameterCollection;
@@ -135,7 +138,10 @@ namespace Aws
                 Aws::StringStream ss;
                 ss << pathSegments;
                 Aws::String segments = ss.str();
-                for (const auto& segment : Aws::Utils::StringUtils::Split(segments, '/'))
+                const auto splitOption = s_preservePathSeparators
+                                           ? Utils::StringUtils::SplitOptions::PRESERVE_DELIMITERS
+                                           : Utils::StringUtils::SplitOptions::NOT_SET;
+                for (const auto& segment : Aws::Utils::StringUtils::Split(segments, '/', splitOption))
                 {
                     m_pathSegments.push_back(segment);
                 }
diff --git a/src/aws-cpp-sdk-core/include/aws/core/utils/StringUtils.h b/src/aws-cpp-sdk-core/include/aws/core/utils/StringUtils.h
@@ -81,7 +81,12 @@ namespace Aws
                 /**
                  * Includes empty entries in the vector returned by Split()
                  */
-                INCLUDE_EMPTY_ENTRIES
+                INCLUDE_EMPTY_ENTRIES,
+                /**
+                 * Include delimeter in vector returned, however removing leading and trailing
+                 * delimiters.
+                 */
+                PRESERVE_DELIMITERS,
             };
 
             /**
@@ -116,6 +121,14 @@ namespace Aws
              */
             static Aws::Vector<Aws::String> Split(const Aws::String& toSplit, char splitOn, size_t numOfTargetParts, SplitOptions option);
 
+            /**
+             * Splits a string on delimeter, keeping the delimeter in the vector returned.
+             * @param toSplit, the original string to split
+             * @param splitOn, the delimiter you want to use.
+             * @param numOfTargetParts, how many target parts you want to get, if it is 0, as many as possible.
+             */
+            static Aws::Vector<Aws::String> SplitWithDelimiters(const Aws::String& toSplit, char splitOn, size_t numOfTargetParts);
+
             /**
             * Splits a string on new line characters.
             */
diff --git a/src/aws-cpp-sdk-core/source/Aws.cpp b/src/aws-cpp-sdk-core/source/Aws.cpp
@@ -155,6 +155,7 @@ namespace Aws
         Aws::Http::SetInitCleanupCurlFlag(options.httpOptions.initAndCleanupCurl);
         Aws::Http::SetInstallSigPipeHandlerFlag(options.httpOptions.installSigPipeHandler);
         Aws::Http::SetCompliantRfc3986Encoding(options.httpOptions.compliantRfc3986Encoding);
+        Aws::Http::SetPreservePathSeparators(options.httpOptions.preservePathSeparators);
         Aws::Http::InitHttp();
         Aws::InitializeEnumOverflowContainer();
         cJSON_AS4CPP_Hooks hooks;
diff --git a/src/aws-cpp-sdk-core/source/http/URI.cpp b/src/aws-cpp-sdk-core/source/http/URI.cpp
@@ -27,6 +27,9 @@ const char* SEPARATOR = "://";
 bool s_compliantRfc3986Encoding = false;
 void SetCompliantRfc3986Encoding(bool compliant) { s_compliantRfc3986Encoding = compliant; }
 
+bool s_preservePathSeparators = false;
+void SetPreservePathSeparators(bool preservePathSeparators) { s_preservePathSeparators = preservePathSeparators; }
+
 Aws::String urlEncodeSegment(const Aws::String& segment, bool rfcEncoded = false)
 {
     // consolidates legacy escaping logic into one local method
@@ -199,8 +202,18 @@ Aws::String URI::GetPath() const
 
     for (auto const& segment : m_pathSegments)
     {
-        path.push_back('/');
-        path.append(segment);
+        if((s_preservePathSeparators && segment == "/"))
+        {
+            path.push_back('/');
+        }
+        else
+        {
+            if (!s_preservePathSeparators)
+            {
+                path.push_back('/');
+            }
+            path.append(segment);
+        }
     }
 
     if (m_pathSegments.empty() || m_pathHasTrailingSlash)
@@ -217,7 +230,18 @@ Aws::String URI::GetURLEncodedPath() const
 
     for (auto const& segment : m_pathSegments)
     {
-        ss << '/' << StringUtils::URLEncode(segment.c_str());
+        if((s_preservePathSeparators && segment == "/"))
+        {
+            ss << "/";
+        }
+        else
+        {
+            if (!s_preservePathSeparators)
+            {
+                ss << "/";
+            }
+            ss << StringUtils::URLEncode(segment.c_str());
+        }
     }
 
     if (m_pathSegments.empty() || m_pathHasTrailingSlash)
@@ -237,7 +261,18 @@ Aws::String URI::GetURLEncodedPathRFC3986() const
     // (mostly; there is some non-standards legacy support that can be disabled)
     for (const auto& segment : m_pathSegments)
     {
-        ss << '/' << urlEncodeSegment(segment, m_useRfcEncoding);
+        if((s_preservePathSeparators && segment == "/"))
+        {
+            ss << "/";
+        }
+        else
+        {
+            if (!s_preservePathSeparators)
+            {
+                ss << "/";
+            }
+            ss << urlEncodeSegment(segment, m_useRfcEncoding);
+        }
     }
 
     if (m_pathSegments.empty() || m_pathHasTrailingSlash)
diff --git a/src/aws-cpp-sdk-core/source/utils/StringUtils.cpp b/src/aws-cpp-sdk-core/source/utils/StringUtils.cpp
@@ -90,6 +90,11 @@ Aws::Vector<Aws::String> StringUtils::Split(const Aws::String& toSplit, char spl
 
 Aws::Vector<Aws::String> StringUtils::Split(const Aws::String& toSplit, char splitOn, size_t numOfTargetParts, SplitOptions option)
 {
+    if (option == SplitOptions::PRESERVE_DELIMITERS)
+    {
+        return StringUtils::SplitWithDelimiters(toSplit, splitOn, numOfTargetParts);
+    }
+
     Aws::Vector<Aws::String> returnValues;
     Aws::StringStream input(toSplit);
     Aws::String item;
@@ -128,6 +133,28 @@ Aws::Vector<Aws::String> StringUtils::Split(const Aws::String& toSplit, char spl
     return returnValues;
 }
 
+Aws::Vector<Aws::String> StringUtils::SplitWithDelimiters(const Aws::String& toSplit, char splitOn, size_t numOfTargetParts)
+{
+    Aws::Vector<Aws::String> returnValues;
+    size_t start = 0;
+    size_t found = toSplit.find_first_of(splitOn, start);
+
+    while (returnValues.size() < numOfTargetParts && found != std::string::npos) {
+        if (found > start) {
+            returnValues.push_back(toSplit.substr(start, found - start));
+        }
+        returnValues.push_back(toSplit.substr(found, 1));
+        start = found + 1;
+        found = toSplit.find_first_of(splitOn, start);
+    }
+
+    if (start < toSplit.length()) {
+        returnValues.push_back(toSplit.substr(start));
+    }
+
+    return returnValues;
+}
+
 Aws::Vector<Aws::String> StringUtils::SplitOnLine(const Aws::String& toSplit)
 {
     Aws::StringStream input(toSplit);
diff --git a/tests/aws-cpp-sdk-s3-unit-tests/S3UnitTests.cpp b/tests/aws-cpp-sdk-s3-unit-tests/S3UnitTests.cpp
@@ -124,3 +124,60 @@ TEST_F(S3UnitTest, S3UriMiddleDots) {
   const auto seenRequest = _mockHttpClient->GetMostRecentHttpRequest();
   EXPECT_EQ("https://bluerev.s3.us-east-1.amazonaws.com/belinda/../says", seenRequest.GetUri().GetURIString());
 }
+
+TEST_F(S3UnitTest, S3UriPathPreservationOff) {
+  auto putObjectRequest = PutObjectRequest()
+      .WithBucket("vu")
+      .WithKey("////stephanie////says////////////that////////she//wants///////to/know.txt");
+
+  std::shared_ptr<IOStream> body = Aws::MakeShared<StringStream>(ALLOCATION_TAG,
+    "What country shall I say is calling From across the world?",
+    std::ios_base::in | std::ios_base::binary);
+
+  putObjectRequest.SetBody(body);
+
+  //We have to mock requset because it is used to create the return body, it actually isnt used.
+  auto mockRequest = Aws::MakeShared<Standard::StandardHttpRequest>(ALLOCATION_TAG, "mockuri", HttpMethod::HTTP_GET);
+  mockRequest->SetResponseStreamFactory([]() -> IOStream* {
+    return Aws::New<StringStream>(ALLOCATION_TAG, "response-string", std::ios_base::in | std::ios_base::binary);
+  });
+  auto mockResponse = Aws::MakeShared<Standard::StandardHttpResponse>(ALLOCATION_TAG, mockRequest);
+  mockResponse->SetResponseCode(HttpResponseCode::OK);
+  _mockHttpClient->AddResponseToReturn(mockResponse);
+
+  const auto response = _s3Client->PutObject(putObjectRequest);
+  AWS_EXPECT_SUCCESS(response);
+
+  const auto seenRequest = _mockHttpClient->GetMostRecentHttpRequest();
+  EXPECT_EQ("https://s3.us-east-1.amazonaws.com/vu/stephanie/says/that/she/wants/to/know.txt", seenRequest.GetUri().GetURIString());
+}
+
+TEST_F(S3UnitTest, S3UriPathPreservationOn) {
+  //Turn on path preservation
+  Aws::Http::SetPreservePathSeparators(true);
+
+  auto putObjectRequest = PutObjectRequest()
+      .WithBucket("vu")
+      .WithKey("////stephanie////says////////////that////////she//wants///////to/know.txt");
+
+  std::shared_ptr<IOStream> body = Aws::MakeShared<StringStream>(ALLOCATION_TAG,
+    "What country shall I say is calling From across the world?",
+    std::ios_base::in | std::ios_base::binary);
+
+  putObjectRequest.SetBody(body);
+
+  //We have to mock requset because it is used to create the return body, it actually isnt used.
+  auto mockRequest = Aws::MakeShared<Standard::StandardHttpRequest>(ALLOCATION_TAG, "mockuri", HttpMethod::HTTP_GET);
+  mockRequest->SetResponseStreamFactory([]() -> IOStream* {
+    return Aws::New<StringStream>(ALLOCATION_TAG, "response-string", std::ios_base::in | std::ios_base::binary);
+  });
+  auto mockResponse = Aws::MakeShared<Standard::StandardHttpResponse>(ALLOCATION_TAG, mockRequest);
+  mockResponse->SetResponseCode(HttpResponseCode::OK);
+  _mockHttpClient->AddResponseToReturn(mockResponse);
+
+  const auto response = _s3Client->PutObject(putObjectRequest);
+  AWS_EXPECT_SUCCESS(response);
+
+  const auto seenRequest = _mockHttpClient->GetMostRecentHttpRequest();
+  EXPECT_EQ("https://s3.us-east-1.amazonaws.com/vu////stephanie////says////////////that////////she//wants///////to/know.txt", seenRequest.GetUri().GetURIString());
+}
