s3-zip file support (#687)

* s3-zip file support

* Addresses build issue and implements review comment

* Rebase

* Rebase and resolve merge conflicts

* Rebasing and fixing merge conflicts-3

* Rebase and fix merge conflicts

* Comments out the test causing build failure on Windows

* Rebase and fix merge conflicts - 4

* Removes the forgotten comment

* Cosmetic cleanups

* Rebase and fix merge conflicts-2

* Rebase and fix merge conflicts-3

* Rebase and fix merge conflicts-4

Author: ebozduman

Minio.Functional.Tests/FunctionalTest.cs

@@ -33,6 +33,8 @@
 using System.Threading.Tasks;
 using System.Web;
 using System.Xml;
+using ICSharpCode.SharpZipLib.Core;
+using ICSharpCode.SharpZipLib.Zip;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Minio.DataModel;
 using Minio.DataModel.ILM;
@@ -2393,6 +2395,129 @@ internal static async Task ObjectRetentionAsync_Test1(MinioClient minio)
 
     #endregion
 
+    internal static MemoryStream CreateZipFile(string prefix, int nFiles)
+    {
+        var outputMemStream = new MemoryStream();
+        var zipStream = new ZipOutputStream(outputMemStream);
+
+        zipStream.SetLevel(3); //0-9, 9 being the highest level of compression
+        byte[] bytes = null;
+
+        for (var i = 0; i <= nFiles; i++)
+        {
+            // Make one large, compressible file.
+            if (i == nFiles) i = 1000000;
+
+            var fileName = prefix + "file-" + i + ".bin";
+            Directory.CreateDirectory(prefix);
+            var newEntry = new ZipEntry(fileName);
+            newEntry.DateTime = DateTime.Now;
+            zipStream.PutNextEntry(newEntry);
+
+            bytes = rsg.GenerateStreamFromSeed(i).ToArray();
+            var inStream = new MemoryStream(bytes);
+            if (i == 0) StreamUtils.Copy(inStream, zipStream, new byte[128]);
+            else StreamUtils.Copy(inStream, zipStream, new byte[i * 128]);
+
+            inStream.Close();
+            zipStream.CloseEntry();
+        }
+
+        zipStream.IsStreamOwner = false; // False stops the Close also Closing the underlying stream.
+        zipStream.Close(); // Must finish the ZipOutputStream before using outputMemStream.
+
+        outputMemStream.Position = 0;
+        outputMemStream.Seek(0, SeekOrigin.Begin);
+
+        return outputMemStream;
+    }
+
+    internal static async Task GetObjectS3Zip_Test1(MinioClient minio)
+    {
+        var path = "test/small/";
+        var startTime = DateTime.Now;
+        var bucketName = GetRandomName(15);
+        var randomFileName = GetRandomName(15) + ".zip";
+        var objectName = GetRandomObjectName(15) + ".zip";
+
+        var args = new Dictionary<string, string>
+        {
+            { "bucketName", bucketName },
+            { "objectName", objectName }
+        };
+        try
+        {
+            await Setup_Test(minio, bucketName);
+            const int nFiles = 500;
+            var memStream = CreateZipFile(path, nFiles);
+
+            var putObjectArgs = new PutObjectArgs()
+                .WithBucket(bucketName)
+                .WithObject(objectName)
+                .WithStreamData(memStream)
+                .WithObjectSize(memStream.Length);
+            await minio.PutObjectAsync(putObjectArgs).ConfigureAwait(false);
+
+            var getObjectArgs = new GetObjectArgs()
+                .WithBucket(bucketName)
+                .WithFile(randomFileName)
+                .WithObject(objectName);
+
+            var resp = await minio.GetObjectAsync(getObjectArgs).ConfigureAwait(false);
+
+            var statArgs = new StatObjectArgs()
+                .WithBucket(bucketName)
+                .WithObject(objectName);
+            var stat = await minio.StatObjectAsync(statArgs).ConfigureAwait(false);
+
+            var lOpts = new Dictionary<string, string>();
+            lOpts.Add("x-minio-extract", "true");
+
+            // Test with different prefix values
+            // prefix value="", expected number of files listed=1
+            var prefix = "";
+            ListObjects_Test(minio, bucketName, prefix, 1, true, headers: lOpts);
+
+            // prefix value="/", expected number of files listed=nFiles+1
+            prefix = objectName + "/";
+            ListObjects_Test(minio, bucketName, prefix, nFiles + 1, true, headers: lOpts);
+
+            // prefix value="/test", expected number of files listed=nFiles + 1
+            prefix = objectName + "/test";
+            ListObjects_Test(minio, bucketName, prefix, nFiles + 1, true, headers: lOpts);
+
+            // prefix value="/test/", expected number of files listed=nFiles+1
+            prefix = objectName + "/test/";
+            ListObjects_Test(minio, bucketName, prefix, nFiles + 1, true, headers: lOpts);
+
+            // prefix value="/test", expected number of files listed=nFiles+1
+            prefix = objectName + "/test/small";
+            ListObjects_Test(minio, bucketName, prefix, nFiles + 1, true, headers: lOpts);
+
+            // prefix value="/test", expected number of files listed=nFiles+1
+            prefix = objectName + "/test/small/";
+            ListObjects_Test(minio, bucketName, prefix, nFiles + 1, true, headers: lOpts);
+
+            // prefix value="/test", expected number of files listed=1
+            prefix = objectName + "/test/small/" + "file-1.bin";
+            ListObjects_Test(minio, bucketName, prefix, 1, true, headers: lOpts);
+
+            new MintLogger("GetObjectS3Zip_Test1", getObjectSignature, "Tests s3Zip files", TestStatus.PASS,
+                DateTime.Now - startTime, args: args).Log();
+        }
+        catch (Exception ex)
+        {
+            new MintLogger("GetObjectS3Zip_Test1", getObjectSignature, "Tests s3Zip files", TestStatus.FAIL,
+                DateTime.Now - startTime, ex.Message, ex.ToString(), args: args).Log();
+            throw;
+        }
+        finally
+        {
+            File.Delete(randomFileName);
+            Directory.Delete(path.Split("/")[0], true);
+            await TearDown(minio, bucketName);
+        }
+    }
 
     #region Bucket Notifications
 
@@ -4976,13 +5101,14 @@ internal static async Task ListObjectVersions_Test1(MinioClient minio)
 
 
     internal static void ListObjects_Test(MinioClient minio, string bucketName, string prefix, int numObjects,
-        bool recursive = true, bool versions = false)
+        bool recursive = true, bool versions = false, Dictionary<string, string> headers = null)
     {
         var startTime = DateTime.Now;
         var count = 0;
         var args = new ListObjectsArgs()
             .WithBucket(bucketName)
             .WithPrefix(prefix)
+            .WithHeaders(headers)
             .WithRecursive(recursive)
             .WithVersions(versions);
         if (!versions)
@@ -4991,11 +5117,11 @@ internal static void ListObjects_Test(MinioClient minio, string bucketName, stri
             var subscription = observable.Subscribe(
                 item =>
                 {
-                    Assert.IsTrue(item.Key.StartsWith(prefix));
-                    count += 1;
+                    if (!string.IsNullOrEmpty(prefix)) Assert.IsTrue(item.Key.StartsWith(prefix));
+                    count++;
                 },
                 ex => throw ex,
-                () => { Assert.AreEqual(count, numObjects); });
+                () => { ; });
         }
         else
         {
@@ -5007,8 +5133,11 @@ internal static void ListObjects_Test(MinioClient minio, string bucketName, stri
                     count += 1;
                 },
                 ex => throw ex,
-                () => { Assert.AreEqual(count, numObjects); });
+                () => { ; });
         }
+
+        Thread.Sleep(1000);
+        Assert.AreEqual(numObjects, count);
     }
 
     #endregion

Minio.Functional.Tests/Minio.Functional.Tests.csproj

@@ -7,6 +7,7 @@
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="MSTest.TestFramework" Version="1.3.2" />
+    <PackageReference Include="SharpZipLib" Version="1.3.3" />
     <PackageReference Include="System.Net.Http" Version="4.3.4" />
     <ProjectReference Include="..\Minio\Minio.csproj" />
   </ItemGroup>

Minio.Functional.Tests/Program.cs

@@ -165,6 +165,9 @@ public static void Main(string[] args)
         // Test SetPolicyAsync function
         FunctionalTest.SetBucketPolicy_Test1(minioClient).Wait();
 
+        // Test S3Zip function
+        FunctionalTest.GetObjectS3Zip_Test1(minioClient).Wait();
+
         // Test Presigned Get/Put operations
         FunctionalTest.PresignedGetObject_Test1(minioClient).Wait();
         FunctionalTest.PresignedGetObject_Test2(minioClient).Wait();

Minio/ApiEndpoints/BucketOperations.cs

@@ -218,6 +218,7 @@ public IObservable<Item> ListObjectsAsync(ListObjectsArgs args, CancellationToke
                             .WithContinuationToken(nextContinuationToken)
                             .WithMarker(marker)
                             .WithListObjectsV1(!args.UseV2)
+                            .WithHeaders(args.Headers)
                             .WithVersionIdMarker(versionIdMarker);
                         if (args.Versions)
                         {

Minio/DataModel/BucketOperationsArgs.cs

@@ -187,12 +187,8 @@ public GetObjectListArgs WithListObjectsV1(bool useV1)
 
     internal override HttpRequestMessageBuilder BuildRequest(HttpRequestMessageBuilder requestMessageBuilder)
     {
-        // using System.Web; Not sure if we need to add query parameters like this vs. requestMessageBuilder.AddQueryParameter()
-        // var query = HttpUtility.ParseQueryString(string.Empty);
-        // query["foo"] = "bar<>&-baz";
-        // query["bar"] = "bazinga";
-        // string queryString = query.ToString();        {
-
+        foreach (var h in Headers)
+            requestMessageBuilder.AddOrUpdateHeaderParameter(h.Key, h.Value);
 
         requestMessageBuilder.AddQueryParameter("delimiter", Delimiter);
         requestMessageBuilder.AddQueryParameter("max-keys", "1000");