format: packfile based on ObjectStorage and CRC32 calculation

Author: mcuadros

common_test.go

@@ -126,7 +126,6 @@ func unpackFixtures(c *C, fixtures ...[]packedFixture) map[string]*Repository {
 			d := packfile.NewDecoder(r, repos[fixture.url].s.ObjectStorage())
 			_, err = d.Decode()
 			c.Assert(err, IsNil, comment)
-
 			c.Assert(f.Close(), IsNil, comment)
 		}
 	}

core/storage.go

@@ -54,6 +54,7 @@ type ObjectIter interface {
 // A transaction must end with a call to Commit or Rollback.
 type TxObjectStorage interface {
 	Set(Object) (Hash, error)
+	Get(ObjectType, Hash) (Object, error)
 	Commit() error
 	Rollback() error
 }

formats/idxfile/fixtures/git-fixture.idx renamed to fixtures/data/pack-a3fed42da1e8189a077c0e6846c040dcf73fc9dd.idx

@@ -0,0 +1,102 @@
+package fixtures
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	check "gopkg.in/check.v1"
+	"gopkg.in/src-d/go-git.v4/core"
+)
+
+var RootFolder = ""
+
+const DataFolder = "data"
+
+var fixtures = []*Fixture{{
+	Tags:         []string{"ofs-delta"},
+	URL:          "https://github.com/git-fixtures/basic",
+	Head:         core.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5"),
+	PackfileHash: core.NewHash("a3fed42da1e8189a077c0e6846c040dcf73fc9dd"),
+}, {
+	Tags:         []string{"ref-delta"},
+	URL:          "https://github.com/git-fixtures/basic",
+	Head:         core.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5"),
+	PackfileHash: core.NewHash("c544593473465e6315ad4182d04d366c4592b829"),
+}, {
+	URL:          "https://github.com/spinnaker/spinnaker",
+	Head:         core.NewHash("06ce06d0fc49646c4de733c45b7788aabad98a6f"),
+	PackfileHash: core.NewHash("f2e0a8889a746f7600e07d2246a2e29a72f696be"),
+}}
+
+func All() Fixtures {
+	return fixtures
+}
+
+func Basic() Fixtures {
+	return ByURL("https://github.com/git-fixtures/basic")
+}
+
+func ByURL(url string) Fixtures {
+	r := make(Fixtures, 0)
+	for _, f := range fixtures {
+		if f.URL == url {
+			r = append(r, f)
+		}
+	}
+
+	return r
+}
+
+type Fixture struct {
+	URL          string
+	Tags         []string
+	Head         core.Hash
+	PackfileHash core.Hash
+}
+
+func (f *Fixture) Packfile() io.ReadSeeker {
+	fn := filepath.Join(RootFolder, DataFolder, fmt.Sprintf("pack-%s.pack", f.PackfileHash))
+	file, err := os.Open(fn)
+	if err != nil {
+		panic(err)
+	}
+
+	return file
+}
+
+func (f *Fixture) Idx() io.ReadSeeker {
+	fn := filepath.Join(RootFolder, DataFolder, fmt.Sprintf("pack-%s.idx", f.PackfileHash))
+	file, err := os.Open(fn)
+	if err != nil {
+		panic(err)
+	}
+
+	return file
+}
+
+type Fixtures []*Fixture
+
+func (g Fixtures) Test(c *check.C, test func(*Fixture)) {
+	for _, f := range g {
+		c.Logf("executing test at %s", f.URL)
+		test(f)
+	}
+}
+
+func (g Fixtures) One() *Fixture {
+	return g[0]
+}
+
+func (g Fixtures) ByTag(tag string) *Fixture {
+	for _, f := range g {
+		for _, t := range f.Tags {
+			if t == tag {
+				return f
+			}
+		}
+	}
+
+	return nil
+}

fixtures/data/pack-a3fed42da1e8189a077c0e6846c040dcf73fc9dd.pack

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/binary"
 	"errors"
+	"fmt"
 	"io"
 
 	"gopkg.in/src-d/go-git.v4/core"
@@ -95,7 +96,7 @@ func readFanout(idx *Idxfile, r io.Reader) error {
 	}
 
 	idx.ObjectCount, err = readInt32(r)
-
+	fmt.Println(idx.ObjectCount)
 	return err
 }
 
@@ -116,7 +117,7 @@ func readObjectNames(idx *Idxfile, r io.Reader) error {
 func readCRC32(idx *Idxfile, r io.Reader) error {
 	c := int(idx.ObjectCount)
 	for i := 0; i < c; i++ {
-		if _, err := r.Read(idx.Entries[i].CRC32[:]); err != nil {
+		if err := binary.Read(r, binary.BigEndian, &idx.Entries[i].CRC32); err != nil {
 			return err
 		}
 	}

fixtures/data/pack-c544593473465e6315ad4182d04d366c4592b829.idx

@@ -1,11 +1,14 @@
 package idxfile
 
 import (
+	"bytes"
 	"fmt"
-	"os"
 	"testing"
 
 	. "gopkg.in/check.v1"
+	"gopkg.in/src-d/go-git.v4/fixtures"
+	"gopkg.in/src-d/go-git.v4/formats/packfile"
+	"gopkg.in/src-d/go-git.v4/storage/memory"
 )
 
 func Test(t *testing.T) { TestingT(t) }
@@ -15,26 +18,50 @@ type IdxfileSuite struct{}
 var _ = Suite(&IdxfileSuite{})
 
 func (s *IdxfileSuite) TestDecode(c *C) {
-	f, err := os.Open("fixtures/git-fixture.idx")
-	c.Assert(err, IsNil)
+	f := fixtures.Basic().One()
 
-	d := NewDecoder(f)
+	d := NewDecoder(f.Idx())
 	idx := &Idxfile{}
-	err = d.Decode(idx)
+	err := d.Decode(idx)
 	c.Assert(err, IsNil)
 
-	err = f.Close()
-	c.Assert(err, IsNil)
-
-	c.Assert(int(idx.ObjectCount), Equals, 31)
 	c.Assert(idx.Entries, HasLen, 31)
-	c.Assert(idx.Entries[0].Hash.String(), Equals,
-		"1669dce138d9b841a518c64b10914d88f5e488ea")
+	c.Assert(idx.Entries[0].Hash.String(), Equals, "1669dce138d9b841a518c64b10914d88f5e488ea")
 	c.Assert(idx.Entries[0].Offset, Equals, uint64(615))
+	c.Assert(idx.Entries[0].CRC32, Equals, uint32(3645019190))
+
+	c.Assert(fmt.Sprintf("%x", idx.IdxChecksum), Equals, "fb794f1ec720b9bc8e43257451bd99c4be6fa1c9")
+	c.Assert(fmt.Sprintf("%x", idx.PackfileChecksum), Equals, f.PackfileHash.String())
+}
+
+func (s *IdxfileSuite) TestDecodeCRCs(c *C) {
+	f := fixtures.Basic().ByTag("ofs-delta")
+
+	scanner := packfile.NewScanner(f.Packfile())
+	storage := memory.NewStorage()
+
+	pd := packfile.NewDecoder(scanner, storage.ObjectStorage())
+	checksum, err := pd.Decode()
+	c.Assert(err, IsNil)
+
+	i := &Idxfile{Version: VersionSupported}
+	i.PackfileChecksum = checksum
+
+	offsets := pd.Offsets()
+	for h, crc := range pd.CRCs() {
+		i.Add(h, uint64(offsets[h]), crc)
+	}
+
+	buf := bytes.NewBuffer(nil)
+	e := NewEncoder(buf)
+	_, err = e.Encode(i)
+	c.Assert(err, IsNil)
 
-	c.Assert(fmt.Sprintf("%x", idx.IdxChecksum), Equals,
-		"bba9b7a9895724819225a044c857d391bb9d61d9")
-	c.Assert(fmt.Sprintf("%x", idx.PackfileChecksum), Equals,
-		"54bb61360ab2dad1a3e344a8cd3f82b848518cba")
+	idx := &Idxfile{}
+
+	d := NewDecoder(buf)
+	err = d.Decode(idx)
+	c.Assert(err, IsNil)
 
+	c.Assert(idx, DeepEquals, i)
 }

fixtures/data/pack-c544593473465e6315ad4182d04d366c4592b829.pack

@@ -65,25 +65,24 @@ func (e *Encoder) encodeFanout(idx *Idxfile) (int, error) {
 }
 
 func (e *Encoder) encodeHashes(idx *Idxfile) (int, error) {
-	return e.encodeEntryField(idx, true)
-}
+	sz := 0
+	for _, ent := range idx.Entries {
+		i, err := e.Write(ent.Hash[:])
+		sz += i
 
-func (e *Encoder) encodeCRC32(idx *Idxfile) (int, error) {
-	return e.encodeEntryField(idx, false)
+		if err != nil {
+			return sz, err
+		}
+	}
+
+	return sz, nil
 }
 
-func (e *Encoder) encodeEntryField(idx *Idxfile, isHash bool) (int, error) {
+func (e *Encoder) encodeCRC32(idx *Idxfile) (int, error) {
 	sz := 0
 	for _, ent := range idx.Entries {
-		var data []byte
-		if isHash {
-			data = ent.Hash[:]
-		} else {
-			data = ent.CRC32[:]
-		}
-
-		i, err := e.Write(data)
-		sz += i
+		err := binary.Write(e, binary.BigEndian, ent.CRC32)
+		sz += 4
 
 		if err != nil {
 			return sz, err

fixtures/data/pack-f2e0a8889a746f7600e07d2246a2e29a72f696be.idx

@@ -2,46 +2,32 @@ package idxfile
 
 import (
 	"bytes"
-	"io"
-	"os"
+	"io/ioutil"
 
 	. "gopkg.in/check.v1"
+	"gopkg.in/src-d/go-git.v4/fixtures"
 )
 
-func (s *IdxfileSuite) TestEncode(c *C) {
-	for i, path := range [...]string{
-		"fixtures/git-fixture.idx",
-		"../packfile/fixtures/spinnaker-spinnaker.idx",
-	} {
-		com := Commentf("subtest %d: path = %s", i, path)
-
-		exp, idx, err := decode(path)
-		c.Assert(err, IsNil, com)
-
-		obt := new(bytes.Buffer)
-		e := NewEncoder(obt)
-		size, err := e.Encode(idx)
-		c.Assert(err, IsNil, com)
-
-		c.Assert(size, Equals, exp.Len(), com)
-		c.Assert(obt, DeepEquals, exp, com)
-	}
+func (s *IdxfileSuite) SetUpSuite(c *C) {
+	fixtures.RootFolder = "../../fixtures"
 }
 
-func decode(path string) (*bytes.Buffer, *Idxfile, error) {
-	f, err := os.Open(path)
-	if err != nil {
-		return nil, nil, err
-	}
+func (s *IdxfileSuite) TestEncode(c *C) {
+	fixtures.All().Test(c, func(f *fixtures.Fixture) {
+		expected, err := ioutil.ReadAll(f.Idx())
+		c.Assert(err, IsNil)
 
-	cont := new(bytes.Buffer)
-	tee := io.TeeReader(f, cont)
+		idx := &Idxfile{}
+		d := NewDecoder(bytes.NewBuffer(expected))
+		err = d.Decode(idx)
+		c.Assert(err, IsNil)
 
-	d := NewDecoder(tee)
-	idx := &Idxfile{}
-	if err = d.Decode(idx); err != nil {
-		return nil, nil, err
-	}
+		result := bytes.NewBuffer(nil)
+		e := NewEncoder(result)
+		size, err := e.Encode(idx)
+		c.Assert(err, IsNil)
 
-	return cont, idx, f.Close()
+		c.Assert(size, Equals, len(expected))
+		c.Assert(result.Bytes(), DeepEquals, expected)
+	})
 }

fixtures/data/pack-f2e0a8889a746f7600e07d2246a2e29a72f696be.pack

@@ -25,10 +25,18 @@ type Idxfile struct {
 // offset and CRC32 checksum.
 type Entry struct {
 	Hash   core.Hash
-	CRC32  [4]byte
+	CRC32  uint32
 	Offset uint64
 }
 
+func (idx *Idxfile) Add(h core.Hash, offset uint64, crc32 uint32) {
+	idx.Entries = append(idx.Entries, Entry{
+		Hash:   h,
+		Offset: offset,
+		CRC32:  crc32,
+	})
+}
+
 func (idx *Idxfile) isValid() bool {
 	fanout := idx.calculateFanout()
 	for k, c := range idx.Fanout {