fix!: encode::loose_header() now supports large objects even on 32 bit systems.

Previously, larger than 4GB files wouldn't be supported, which causes problems when
genrating hashes even when streaming data.

Author: Byron

gix-object/src/blob.rs

@@ -12,8 +12,8 @@ impl<'a> crate::WriteTo for BlobRef<'a> {
         Kind::Blob
     }
 
-    fn size(&self) -> usize {
-        self.data.len()
+    fn size(&self) -> u64 {
+        self.data.len() as u64
     }
 }
 
@@ -27,7 +27,7 @@ impl crate::WriteTo for Blob {
         Kind::Blob
     }
 
-    fn size(&self) -> usize {
+    fn size(&self) -> u64 {
         self.to_ref().size()
     }
 }

gix-object/src/commit/write.rs

@@ -27,9 +27,9 @@ impl crate::WriteTo for Commit {
         Kind::Commit
     }
 
-    fn size(&self) -> usize {
+    fn size(&self) -> u64 {
         let hash_in_hex = self.tree.kind().len_in_hex();
-        b"tree".len() + 1 /*space*/ + hash_in_hex + 1 /* nl */
+        (b"tree".len() + 1 /*space*/ + hash_in_hex + 1 /* nl */
         + self.parents.iter().count() * (b"parent".len() + 1 + hash_in_hex + 1)
             + b"author".len() + 1 /* space */ + self.author.size() + 1 /* nl */
             + b"committer".len() + 1 /* space */ + self.committer.size() + 1 /* nl */
@@ -46,7 +46,7 @@ impl crate::WriteTo for Commit {
                 })
                 .sum::<usize>()
             + 1 /* nl */
-            + self.message.len()
+            + self.message.len()) as u64
     }
 }
 
@@ -73,9 +73,9 @@ impl<'a> crate::WriteTo for CommitRef<'a> {
         Kind::Commit
     }
 
-    fn size(&self) -> usize {
+    fn size(&self) -> u64 {
         let hash_in_hex = self.tree().kind().len_in_hex();
-        b"tree".len() + 1 /* space */ + hash_in_hex + 1 /* nl */
+        (b"tree".len() + 1 /* space */ + hash_in_hex + 1 /* nl */
             + self.parents.iter().count() * (b"parent".len() + 1 /* space */ + hash_in_hex + 1 /* nl */)
             + b"author".len() + 1 /* space */ + self.author.size() + 1 /* nl */
             + b"committer".len() + 1 /* space */ + self.committer.size() + 1 /* nl */
@@ -92,6 +92,6 @@ impl<'a> crate::WriteTo for CommitRef<'a> {
                 })
                 .sum::<usize>()
             + 1 /* nl */
-            + self.message.len()
+            + self.message.len()) as u64
     }
 }

gix-object/src/encode.rs

@@ -19,7 +19,7 @@ macro_rules! check {
     };
 }
 /// Generates a loose header buffer
-pub fn loose_header(kind: crate::Kind, size: usize) -> smallvec::SmallVec<[u8; 28]> {
+pub fn loose_header(kind: crate::Kind, size: u64) -> smallvec::SmallVec<[u8; 28]> {
     let mut v = smallvec::SmallVec::new();
     check!(v.write_all(kind.as_bytes()));
     check!(v.write_all(SPACE));

gix-object/src/lib.rs

@@ -345,7 +345,7 @@ pub mod decode {
     /// ([`kind`](super::Kind), `size`, `consumed bytes`).
     ///
     /// `size` is the uncompressed size of the payload in bytes.
-    pub fn loose_header(input: &[u8]) -> Result<(super::Kind, usize, usize), LooseHeaderDecodeError> {
+    pub fn loose_header(input: &[u8]) -> Result<(super::Kind, u64, usize), LooseHeaderDecodeError> {
         use LooseHeaderDecodeError::*;
         let kind_end = input.find_byte(0x20).ok_or(InvalidHeader {
             message: "Expected '<type> <size>'",
@@ -366,7 +366,7 @@ pub mod decode {
 
 /// A standalone function to compute a hash of kind `hash_kind` for an object of `object_kind` and its `data`.
 pub fn compute_hash(hash_kind: gix_hash::Kind, object_kind: Kind, data: &[u8]) -> gix_hash::ObjectId {
-    let header = encode::loose_header(object_kind, data.len());
+    let header = encode::loose_header(object_kind, data.len() as u64);
 
     let mut hasher = gix_features::hash::hasher(hash_kind);
     hasher.update(&header);

gix-object/src/object/mod.rs

@@ -24,7 +24,7 @@ mod write {
             self.kind()
         }
 
-        fn size(&self) -> usize {
+        fn size(&self) -> u64 {
             use crate::ObjectRef::*;
             match self {
                 Tree(v) => v.size(),
@@ -52,7 +52,7 @@ mod write {
             self.kind()
         }
 
-        fn size(&self) -> usize {
+        fn size(&self) -> u64 {
             use crate::Object::*;
             match self {
                 Tree(v) => v.size(),
@@ -185,6 +185,8 @@ pub enum LooseDecodeError {
     InvalidHeader(#[from] LooseHeaderDecodeError),
     #[error(transparent)]
     InvalidContent(#[from] DecodeError),
+    #[error("Object sized {size} does not fit into memory - this can happen on 32 bit systems")]
+    OutOfMemory { size: u64 },
 }
 
 impl<'a> ObjectRef<'a> {
@@ -193,7 +195,7 @@ impl<'a> ObjectRef<'a> {
         let (kind, size, offset) = loose_header(data)?;
 
         let body = &data[offset..]
-            .get(..size)
+            .get(..size.try_into().map_err(|_| LooseDecodeError::OutOfMemory { size })?)
             .ok_or(LooseHeaderDecodeError::InvalidHeader {
                 message: "object data was shorter than its size declared in the header",
             })?;

gix-object/src/tag/write.rs

@@ -44,16 +44,16 @@ impl crate::WriteTo for Tag {
         Kind::Tag
     }
 
-    fn size(&self) -> usize {
-        b"object".len() + 1 /* space */ + self.target.kind().len_in_hex() + 1 /* nl */
+    fn size(&self) -> u64 {
+        (b"object".len() + 1 /* space */ + self.target.kind().len_in_hex() + 1 /* nl */
             + b"type".len() + 1 /* space */ + self.target_kind.as_bytes().len() + 1 /* nl */
             + b"tag".len() + 1 /* space */ + self.name.len() + 1 /* nl */
             + self
             .tagger
             .as_ref()
             .map_or(0, |t| b"tagger".len() + 1 /* space */ + t.size() + 1 /* nl */)
             + 1 /* nl */ + self.message.len()
-            + self.pgp_signature.as_ref().map_or(0, |m| 1 /* nl */ + m.len())
+            + self.pgp_signature.as_ref().map_or(0, |m| 1 /* nl */ + m.len())) as u64
     }
 }
 
@@ -81,16 +81,16 @@ impl<'a> crate::WriteTo for TagRef<'a> {
         Kind::Tag
     }
 
-    fn size(&self) -> usize {
-        b"object".len() + 1 /* space */ + self.target().kind().len_in_hex() + 1 /* nl */
+    fn size(&self) -> u64 {
+        (b"object".len() + 1 /* space */ + self.target().kind().len_in_hex() + 1 /* nl */
             + b"type".len() + 1 /* space */ + self.target_kind.as_bytes().len() + 1 /* nl */
             + b"tag".len() + 1 /* space */ + self.name.len() + 1 /* nl */
             + self
                 .tagger
                 .as_ref()
                 .map_or(0, |t| b"tagger".len() + 1 /* space */ + t.size() + 1 /* nl */)
             + 1 /* nl */ + self.message.len()
-            + self.pgp_signature.as_ref().map_or(0, |m| 1 /* nl */ + m.len())
+            + self.pgp_signature.as_ref().map_or(0, |m| 1 /* nl */ + m.len())) as u64
     }
 }
 

gix-object/src/traits.rs

@@ -17,7 +17,7 @@ pub trait WriteTo {
     /// the object, as such it's possible for [`size`](Self::size) to
     /// return a sensible value but [`write_to`](Self::write_to) to
     /// fail because the object was not actually valid in some way.
-    fn size(&self) -> usize;
+    fn size(&self) -> u64;
 
     /// Returns a loose object header based on the object's data
     fn loose_header(&self) -> smallvec::SmallVec<[u8; 28]> {
@@ -37,7 +37,7 @@ where
         <T as WriteTo>::kind(self)
     }
 
-    fn size(&self) -> usize {
+    fn size(&self) -> u64 {
         <T as WriteTo>::size(self)
     }
 }

gix-object/src/tree/write.rs

@@ -57,10 +57,12 @@ impl crate::WriteTo for Tree {
         Kind::Tree
     }
 
-    fn size(&self) -> usize {
+    fn size(&self) -> u64 {
         self.entries
             .iter()
-            .map(|Entry { mode, filename, oid }| mode.as_bytes().len() + 1 + filename.len() + 1 + oid.as_bytes().len())
+            .map(|Entry { mode, filename, oid }| {
+                (mode.as_bytes().len() + 1 + filename.len() + 1 + oid.as_bytes().len()) as u64
+            })
             .sum()
     }
 }
@@ -100,11 +102,11 @@ impl<'a> crate::WriteTo for TreeRef<'a> {
         Kind::Tree
     }
 
-    fn size(&self) -> usize {
+    fn size(&self) -> u64 {
         self.entries
             .iter()
             .map(|EntryRef { mode, filename, oid }| {
-                mode.as_bytes().len() + 1 + filename.len() + 1 + oid.as_bytes().len()
+                (mode.as_bytes().len() + 1 + filename.len() + 1 + oid.as_bytes().len()) as u64
             })
             .sum()
     }