Write directly to the ODB when possible

LibGit2Sharp.Tests/ObjectDatabaseFixture.cs

@@ -28,7 +28,7 @@ public void CanTellIfObjectsExists(string sha, bool shouldExists)
         [Fact]
         public void CanCreateABlobFromAFileInTheWorkingDirectory()
         {
-            string path = CloneStandardTestRepo();
+            string path = InitNewRepository();
             using (var repo = new Repository(path))
             {
                 Assert.Equal(FileStatus.Nonexistent, repo.Index.RetrieveStatus("hello.txt"));
@@ -51,7 +51,7 @@ public void CanCreateABlobFromAFileInTheWorkingDirectory()
         [Fact]
         public void CanCreateABlobIntoTheDatabaseOfABareRepository()
         {
-            string path = CloneBareTestRepo();
+            string path = InitNewRepository();
 
             SelfCleaningDirectory directory = BuildSelfCleaningDirectory();
 
@@ -83,7 +83,7 @@ public void CanCreateABlobIntoTheDatabaseOfABareRepository()
         [InlineData("e9671e138a780833cb689753570fd10a55be84fb", "dummy.guess")]
         public void CanCreateABlobFromAStream(string expectedSha, string hintPath)
         {
-            string path = CloneBareTestRepo();
+            string path = InitNewRepository();
 
             var sb = new StringBuilder();
             for (int i = 0; i < 6; i++)
@@ -103,42 +103,66 @@ public void CanCreateABlobFromAStream(string expectedSha, string hintPath)
             }
         }
 
+        Stream PrepareMemoryStream(int contentSize)
+        {
+            var sb = new StringBuilder();
+            for (int i = 0; i < contentSize; i++)
+            {
+                sb.Append(i % 10);
+            }
+
+            return new MemoryStream(Encoding.UTF8.GetBytes(sb.ToString()));
+        }
+
         [Theory]
-        [InlineData(16, 32)]
         [InlineData(34, 8)]
         [InlineData(7584, 5879)]
         [InlineData(7854, 1247)]
-        [InlineData(7854, 9785)]
         [InlineData(8192, 4096)]
         [InlineData(8192, 4095)]
         [InlineData(8192, 4097)]
         public void CanCreateABlobFromAStreamWithANumberOfBytesToConsume(int contentSize, int numberOfBytesToConsume)
         {
-            string path = CloneBareTestRepo();
+            string path = InitNewRepository();
 
-            var sb = new StringBuilder();
-            for (int i = 0; i < contentSize; i++)
+
+            using (var repo = new Repository(path))
             {
-                sb.Append(i % 10);
+                using (var stream = PrepareMemoryStream(contentSize))
+                {
+                    Blob blob = repo.ObjectDatabase.CreateBlob(stream, numberOfBytesToConsume: numberOfBytesToConsume);
+                    Assert.Equal(numberOfBytesToConsume, blob.Size);
+                }
             }
+        }
+
+        [Theory]
+        [InlineData(16, 32, null)]
+        [InlineData(7854, 9785, null)]
+        [InlineData(16, 32, "binary.bin")]
+        [InlineData(7854, 9785, "binary.bin")]
+        public void CreatingABlobFromTooShortAStreamThrows(int contentSize, int numberOfBytesToConsume, string hintpath)
+        {
+            string path = InitNewRepository();
 
             using (var repo = new Repository(path))
             {
-                using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(sb.ToString())))
+                using (var stream = PrepareMemoryStream(contentSize))
                 {
-                    Blob blob = repo.ObjectDatabase.CreateBlob(stream, numberOfBytesToConsume: numberOfBytesToConsume);
-                    Assert.Equal(Math.Min(numberOfBytesToConsume, contentSize), blob.Size);
+                    Assert.Throws<EndOfStreamException>(() => repo.ObjectDatabase.CreateBlob(stream, hintpath, numberOfBytesToConsume));
                 }
             }
         }
 
         [Fact]
         public void CreatingABlobFromANonReadableStreamThrows()
         {
-            string path = CloneStandardTestRepo();
+            string path = InitNewRepository();
 
-            using (var stream = new FileStream(Path.Combine(path, "file.txt"), FileMode.CreateNew, FileAccess.Write))
             using (var repo = new Repository(path))
+            using (var stream = new FileStream(
+                Path.Combine(repo.Info.WorkingDirectory, "file.txt"),
+                FileMode.CreateNew, FileAccess.Write))
             {
                 Assert.Throws<ArgumentException>(() => repo.ObjectDatabase.CreateBlob(stream));
             }
@@ -218,7 +242,7 @@ public void RemovingANonExistingEntryFromATreeDefinitionHasNoSideEffect()
         [Fact]
         public void CanCreateAnEmptyTree()
         {
-            string path = CloneBareTestRepo();
+            string path = InitNewRepository();
             using (var repo = new Repository(path))
             {
                 var td = new TreeDefinition();
@@ -364,7 +388,7 @@ public void CanCreateABinaryBlobFromAStream()
         {
             var binaryContent = new byte[] { 0, 1, 2, 3, 4, 5 };
 
-            string path = CloneBareTestRepo();
+            string path = InitNewRepository();
             using (var repo = new Repository(path))
             {
                 using (var stream = new MemoryStream(binaryContent))

LibGit2Sharp/Core/Handles/OdbStreamSafeHandle.cs

@@ -0,0 +1,11 @@
+namespace LibGit2Sharp.Core.Handles
+{
+    internal class OdbStreamSafeHandle : SafeHandleBase
+    {
+        protected override bool ReleaseHandleImpl()
+        {
+            Proxy.git_odb_stream_free(handle);
+            return true;
+        }
+    }
+}

LibGit2Sharp/Core/NativeMethods.cs

@@ -712,12 +712,24 @@ internal static extern int git_odb_foreach(
             git_odb_foreach_cb cb,
             IntPtr payload);
 
+        [DllImport(libgit2)]
+        internal static extern int git_odb_open_wstream(out OdbStreamSafeHandle stream, ObjectDatabaseSafeHandle odb, UIntPtr size, GitObjectType type);
+
         [DllImport(libgit2)]
         internal static extern void git_odb_free(IntPtr odb);
 
         [DllImport(libgit2)]
         internal static extern void git_object_free(IntPtr obj);
 
+        [DllImport(libgit2)]
+        internal static extern int git_odb_stream_write(OdbStreamSafeHandle Stream, IntPtr Buffer, UIntPtr len);
+
+        [DllImport(libgit2)]
+        internal static extern int git_odb_stream_finalize_write(out GitOid id, OdbStreamSafeHandle stream);
+
+        [DllImport(libgit2)]
+        internal static extern void git_odb_stream_free(IntPtr stream);
+
         [DllImport(libgit2)]
         internal static extern OidSafeHandle git_object_id(GitObjectSafeHandle obj);
 

LibGit2Sharp/Core/Proxy.cs

@@ -69,6 +69,12 @@ public static ObjectId git_blob_create_fromchunks(RepositorySafeHandle repo, Fil
             {
                 var oid = new GitOid();
                 int res = NativeMethods.git_blob_create_fromchunks(ref oid, repo, hintpath, fileCallback, IntPtr.Zero);
+
+                if (res == (int)GitErrorCode.User)
+                {
+                    throw new EndOfStreamException("The stream ended unexpectedly"); 
+                }
+
                 Ensure.ZeroResult(res);
 
                 return oid;
@@ -1290,11 +1296,57 @@ public static ICollection<TResult> git_odb_foreach<TResult>(
                     IntPtr.Zero));
         }
 
+        public static OdbStreamSafeHandle git_odb_open_wstream(ObjectDatabaseSafeHandle odb, UIntPtr size, GitObjectType type)
+        {
+            using (ThreadAffinity())
+            {
+                OdbStreamSafeHandle stream;
+                int res = NativeMethods.git_odb_open_wstream(out stream, odb, size, type);
+                Ensure.ZeroResult(res);
+
+                return stream;
+            }
+        }
+
         public static void git_odb_free(IntPtr odb)
         {
             NativeMethods.git_odb_free(odb);
         }
 
+        public static void git_odb_stream_write(OdbStreamSafeHandle stream, byte[] data, int len)
+        {
+            using (ThreadAffinity())
+            {
+                int res;
+                unsafe
+                {
+                    fixed (byte *p = data)
+                    {
+                        res = NativeMethods.git_odb_stream_write(stream, (IntPtr) p, (UIntPtr) len);
+                    }
+                }
+
+                Ensure.ZeroResult(res);
+            }
+        }
+
+        public static ObjectId git_odb_stream_finalize_write(OdbStreamSafeHandle stream)
+        {
+            using (ThreadAffinity())
+            {
+                GitOid id;
+                int res = NativeMethods.git_odb_stream_finalize_write(out id, stream);
+                Ensure.ZeroResult(res);
+
+                return id;
+            }
+        }
+
+        public static void git_odb_stream_free(IntPtr stream)
+        {
+            NativeMethods.git_odb_stream_free(stream);
+        }
+
         #endregion
 
         #region git_patch_

LibGit2Sharp/LibGit2Sharp.csproj

@@ -312,6 +312,7 @@
     <Compile Include="UsernamePasswordCredentials.cs" />
     <Compile Include="VoidReference.cs" />
     <Compile Include="Core\RawContentStream.cs" />
+    <Compile Include="Core\Handles\OdbStreamSafeHandle.cs" />
   </ItemGroup>
   <ItemGroup>
     <CodeAnalysisDictionary Include="CustomDictionary.xml" />

LibGit2Sharp/ObjectDatabase.cs

@@ -138,7 +138,19 @@ public int Provider(IntPtr content, int max_length, IntPtr data)
                     }
                 }
 
+                if (bytesToRead == 0)
+                {
+                    return 0;
+                }
+
                 int numberOfReadBytes = stream.Read(local, 0, bytesToRead);
+
+                if (numberOfBytesToConsume.HasValue
+                    && numberOfReadBytes == 0)
+                {
+                    return (int)GitErrorCode.User;
+                }
+
                 totalNumberOfReadBytes += numberOfReadBytes;
 
                 Marshal.Copy(local, 0, content, numberOfReadBytes);
@@ -148,7 +160,8 @@ public int Provider(IntPtr content, int max_length, IntPtr data)
         }
 
         /// <summary>
-        /// Inserts a <see cref="Blob"/> into the object database, created from the content of a data provider.
+        /// Inserts a <see cref="Blob"/> into the object database, created from the content of a stream.
+        /// <para>Optionally, git filters will be applied to the content before storing it.</para>
         /// </summary>
         /// <param name="stream">The stream from which will be read the content of the blob to be created.</param>
         /// <param name="hintpath">The hintpath is used to determine what git filters should be applied to the object before it can be placed to the object database.</param>
@@ -158,6 +171,12 @@ public virtual Blob CreateBlob(Stream stream, string hintpath = null, int? numbe
         {
             Ensure.ArgumentNotNull(stream, "stream");
 
+            // there's no need to buffer the file for filtering, so simply use a stream
+            if (hintpath == null && numberOfBytesToConsume.HasValue)
+            {
+                return CreateBlob(stream, numberOfBytesToConsume.Value);
+            }
+
             if (!stream.CanRead)
             {
                 throw new ArgumentException("The stream cannot be read from.", "stream");
@@ -169,6 +188,47 @@ public virtual Blob CreateBlob(Stream stream, string hintpath = null, int? numbe
             return repo.Lookup<Blob>(id);
         }
 
+        /// <summary>
+        /// Inserts a <see cref="Blob"/> into the object database created from the content of the stream.
+        /// </summary>
+        /// <param name="stream">The stream from which will be read the content of the blob to be created.</param>
+        /// <param name="numberOfBytesToConsume">Number of bytes to consume from the stream.</param>
+        /// <returns>The created <see cref="Blob"/>.</returns>
+        public virtual Blob CreateBlob(Stream stream, int numberOfBytesToConsume)
+        {
+            Ensure.ArgumentNotNull(stream, "stream");
+
+            if (!stream.CanRead)
+            {
+                throw new ArgumentException("The stream cannot be read from.", "stream");
+            }
+
+            using (var odbStream = Proxy.git_odb_open_wstream(handle, (UIntPtr)numberOfBytesToConsume, GitObjectType.Blob))
+            {
+                var buffer = new byte[4*1024];
+                int totalRead = 0;
+
+                while (totalRead < numberOfBytesToConsume)
+                {
+                    var left = numberOfBytesToConsume - totalRead;
+                    var toRead = left < buffer.Length ? left : buffer.Length;
+                    var read = stream.Read(buffer, 0, toRead);
+
+                    if (read == 0)
+                    {
+                        throw new EndOfStreamException("The stream ended unexpectedly");
+                    }
+
+                    Proxy.git_odb_stream_write(odbStream, buffer, read);
+                    totalRead += read;
+                }
+
+                var id = Proxy.git_odb_stream_finalize_write(odbStream);
+
+                return repo.Lookup<Blob>(id);
+            }
+        }
+
         /// <summary>
         /// Inserts a <see cref="Tree"/> into the object database, created from a <see cref="TreeDefinition"/>.
         /// </summary>