format: packfile new interface (wip)

Author: mcuadros

formats/idxfile/encoder.go

@@ -81,6 +81,7 @@ func (e *Encoder) encodeEntryField(idx *Idxfile, isHash bool) (int, error) {
 		} else {
 			data = ent.CRC32[:]
 		}
+
 		i, err := e.Write(data)
 		sz += i
 

formats/packfile/decoder.go

@@ -1,7 +1,9 @@
 package packfile
 
 import (
+	"bytes"
 	"io"
+	"os"
 
 	"gopkg.in/src-d/go-git.v4/core"
 )
@@ -21,37 +23,46 @@ var (
 	// of objects in the packfile is higher than
 	// Decoder.MaxObjectsLimit.
 	ErrMaxObjectsLimitReached = NewError("max. objects limit reached")
-
 	// ErrInvalidObject is returned by Decode when an invalid object is
 	// found in the packfile.
 	ErrInvalidObject = NewError("invalid git object")
-
 	// ErrPackEntryNotFound is returned by Decode when a reference in
 	// the packfile references and unknown object.
 	ErrPackEntryNotFound = NewError("can't find a pack entry")
-
 	// ErrZLib is returned by Decode when there was an error unzipping
 	// the packfile contents.
 	ErrZLib = NewError("zlib reading error")
+	// ErrDuplicatedObject is returned by Remember if an object appears several
+	// times in a packfile.
+	ErrDuplicatedObject = NewError("duplicated object")
+	// ErrCannotRecall is returned by RecallByOffset or RecallByHash if the object
+	// to recall cannot be returned.
+	ErrCannotRecall = NewError("cannot recall object")
 )
 
 // Decoder reads and decodes packfiles from an input stream.
 type Decoder struct {
-	p *Parser
-	s core.ObjectStorage
+	p              *Parser
+	s              core.ObjectStorage
+	seeker         io.Seeker
+	offsetToObject map[int64]core.Object
+	hashToOffset   map[core.Hash]int64
 }
 
 // NewDecoder returns a new Decoder that reads from r.
-func NewDecoder(r ReadRecaller, s core.ObjectStorage) *Decoder {
+func NewDecoder(s core.ObjectStorage, p *Parser, seeker io.Seeker) *Decoder {
 	return &Decoder{
-		p: NewParser(r),
-		s: s,
+		p:              p,
+		s:              s,
+		seeker:         seeker,
+		offsetToObject: make(map[int64]core.Object, 0),
+		hashToOffset:   make(map[core.Hash]int64, 0),
 	}
 }
 
 // Decode reads a packfile and stores it in the value pointed to by s.
 func (d *Decoder) Decode() error {
-	count, err := d.p.ReadHeader()
+	_, count, err := d.p.Header()
 	if err != nil {
 		return err
 	}
@@ -74,21 +85,7 @@ func (d *Decoder) readObjects(tx core.TxObjectStorage, count uint32) error {
 	// That's 1 sec for ~2450 objects, ~4.20 MB, or ~250 ms per MB,
 	// of which 12-20 % is _not_ zlib inflation (ie. is our code).
 	for i := 0; i < int(count); i++ {
-		start, err := d.p.Offset()
-		if err != nil {
-			return err
-		}
-
-		obj := d.s.NewObject()
-		if err := d.p.FillObject(obj); err != nil {
-			if err == io.EOF {
-				break
-			}
-
-			return err
-		}
-
-		err = d.p.Remember(start, obj)
+		obj, err := d.readObject()
 		if err != nil {
 			return err
 		}
@@ -101,3 +98,134 @@ func (d *Decoder) readObjects(tx core.TxObjectStorage, count uint32) error {
 
 	return nil
 }
+
+func (d *Decoder) readObject() (core.Object, error) {
+	h, err := d.p.NextObjectHeader()
+	if err != nil {
+		return nil, err
+	}
+
+	obj := d.s.NewObject()
+	obj.SetSize(h.Length)
+	obj.SetType(h.Type)
+
+	switch h.Type {
+	case core.CommitObject, core.TreeObject, core.BlobObject, core.TagObject:
+		err = d.fillRegularObjectContent(obj)
+	case core.REFDeltaObject:
+		err = d.fillREFDeltaObjectContent(obj, h.Reference)
+	case core.OFSDeltaObject:
+		err = d.fillOFSDeltaObjectContent(obj, h.OffsetReference)
+	default:
+		err = ErrInvalidObject.AddDetails("type %q", h.Type)
+	}
+
+	return obj, d.remember(h.Offset, obj)
+}
+
+func (d *Decoder) fillRegularObjectContent(obj core.Object) error {
+	w, err := obj.Writer()
+	if err != nil {
+		return err
+	}
+
+	_, err = d.p.NextObject(w)
+	return err
+}
+
+func (d *Decoder) fillREFDeltaObjectContent(obj core.Object, ref core.Hash) error {
+	base, err := d.recallByHash(ref)
+	if err != nil {
+		return err
+	}
+	obj.SetType(base.Type())
+	if err := d.readAndApplyDelta(obj, base); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (d *Decoder) fillOFSDeltaObjectContent(obj core.Object, offset int64) error {
+	base, err := d.recallByOffset(offset)
+	if err != nil {
+		return err
+	}
+
+	obj.SetType(base.Type())
+	if err := d.readAndApplyDelta(obj, base); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ReadAndApplyDelta reads and apply the base patched with the contents
+// of a zlib compressed diff data in the delta portion of an object
+// entry in the packfile.
+func (d *Decoder) readAndApplyDelta(target, base core.Object) error {
+	buf := bytes.NewBuffer(nil)
+	if _, err := d.p.NextObject(buf); err != nil {
+		return err
+	}
+
+	return ApplyDelta(target, base, buf.Bytes())
+}
+
+// Remember stores the offset of the object and its hash, but not the
+// object itself.  This implementation does not check for already stored
+// offsets, as it is too expensive to build this information from an
+// index every time a get operation is performed on the SeekableReadRecaller.
+func (r *Decoder) remember(o int64, obj core.Object) error {
+	h := obj.Hash()
+	r.hashToOffset[h] = o
+	r.offsetToObject[o] = obj
+	return nil
+}
+
+// RecallByHash returns the object for a given hash by looking for it again in
+// the io.ReadeSeerker.
+func (r *Decoder) recallByHash(h core.Hash) (core.Object, error) {
+	o, ok := r.hashToOffset[h]
+	if !ok {
+		return nil, ErrCannotRecall.AddDetails("hash not found: %s", h)
+	}
+
+	return r.recallByOffset(o)
+}
+
+// RecallByOffset returns the object for a given offset by looking for it again in
+// the io.ReadeSeerker. For efficiency reasons, this method always find objects by
+// offset, even if they have not been remembered or if they have been forgetted.
+func (r *Decoder) recallByOffset(o int64) (obj core.Object, err error) {
+	obj, ok := r.offsetToObject[o]
+	if ok {
+		return obj, nil
+	}
+
+	if !ok && r.seeker == nil {
+		return nil, ErrCannotRecall.AddDetails("no object found at offset %d", o)
+	}
+
+	// remember current offset
+	beforeJump, err := r.seeker.Seek(0, os.SEEK_CUR)
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		// jump back
+		_, seekErr := r.seeker.Seek(beforeJump, os.SEEK_SET)
+		if err == nil {
+			err = seekErr
+		}
+	}()
+
+	// jump to requested offset
+	_, err = r.seeker.Seek(o, os.SEEK_SET)
+	if err != nil {
+		return nil, err
+	}
+
+	return r.readObject()
+}

formats/packfile/decoder_test.go

@@ -3,16 +3,12 @@ package packfile
 import (
 	"bytes"
 	"encoding/base64"
-	"fmt"
 	"os"
-	"runtime"
 	"testing"
-	"time"
 
 	"gopkg.in/src-d/go-git.v4/core"
 	"gopkg.in/src-d/go-git.v4/storage/memory"
 
-	"github.com/dustin/go-humanize"
 	. "gopkg.in/check.v1"
 )
 
@@ -27,9 +23,8 @@ var packFileWithEmptyObjects = "UEFDSwAAAAIAAAALnw54nKXMQWoDMQxA0b1PoX2hSLIm44FS
 func (s *ReaderSuite) TestReadPackfile(c *C) {
 	data, _ := base64.StdEncoding.DecodeString(packFileWithEmptyObjects)
 	f := bytes.NewReader(data)
-	r := NewStream(f)
 	sto := memory.NewStorage()
-	d := NewDecoder(r, sto.ObjectStorage())
+	d := NewDecoder(sto.ObjectStorage(), NewParser(f), nil)
 
 	err := d.Decode()
 	c.Assert(err, IsNil)
@@ -60,9 +55,8 @@ func (s *ReaderSuite) TestReadPackfileREFDelta(c *C) {
 func (s *ReaderSuite) testReadPackfileGitFixture(c *C, file string, format Format) {
 	f, err := os.Open(file)
 	c.Assert(err, IsNil)
-	r := NewSeekable(f)
 	sto := memory.NewStorage()
-	d := NewDecoder(r, sto.ObjectStorage())
+	d := NewDecoder(sto.ObjectStorage(), NewParser(f), f)
 
 	err = d.Decode()
 	c.Assert(err, IsNil)
@@ -109,72 +103,3 @@ func AssertObjects(c *C, s *memory.Storage, expects []string) {
 		c.Assert(obt.Hash().String(), Equals, exp)
 	}
 }
-
-func (s *ReaderSuite) BenchmarkFixtureRef(c *C) {
-	for i := 0; i < c.N; i++ {
-		readFromFile(c, "fixtures/git-fixture.ref-delta", REFDeltaFormat)
-	}
-}
-
-func (s *ReaderSuite) BenchmarkFixtureOfs(c *C) {
-	for i := 0; i < c.N; i++ {
-		readFromFile(c, "fixtures/git-fixture.ofs-delta", OFSDeltaFormat)
-	}
-}
-
-func (s *ReaderSuite) BenchmarkCandyJS(c *C) {
-	for i := 0; i < c.N; i++ {
-		readFromFile(c, "/tmp/go-candyjs", REFDeltaFormat)
-	}
-}
-
-func (s *ReaderSuite) BenchmarkSymfony(c *C) {
-	for i := 0; i < c.N; i++ {
-		readFromFile(c, "/tmp/symonfy", REFDeltaFormat)
-	}
-}
-
-func (s *ReaderSuite) BenchmarkGit(c *C) {
-	for i := 0; i < c.N; i++ {
-		readFromFile(c, "/tmp/git", REFDeltaFormat)
-	}
-}
-
-func (s *ReaderSuite) _testMemory(c *C, format Format) {
-	var b, a runtime.MemStats
-
-	start := time.Now()
-	runtime.ReadMemStats(&b)
-	p := readFromFile(c, "/tmp/symfony.ofs-delta", format)
-	runtime.ReadMemStats(&a)
-
-	fmt.Println("OFS--->")
-	fmt.Println("Alloc", a.Alloc-b.Alloc, humanize.Bytes(a.Alloc-b.Alloc))
-	fmt.Println("TotalAlloc", a.TotalAlloc-b.TotalAlloc, humanize.Bytes(a.TotalAlloc-b.TotalAlloc))
-	fmt.Println("HeapAlloc", a.HeapAlloc-b.HeapAlloc, humanize.Bytes(a.HeapAlloc-b.HeapAlloc))
-	fmt.Println("HeapSys", a.HeapSys, humanize.Bytes(a.HeapSys-b.HeapSys))
-
-	fmt.Println("objects", len(p.Objects))
-	fmt.Println("time", time.Since(start))
-}
-
-func (s *ReaderSuite) _TestMemoryOFS(c *C) {
-	s._testMemory(c, OFSDeltaFormat)
-}
-
-func (s *ReaderSuite) _TestMemoryREF(c *C) {
-	s._testMemory(c, REFDeltaFormat)
-}
-
-func readFromFile(c *C, file string, format Format) *memory.ObjectStorage {
-	f, err := os.Open(file)
-	c.Assert(err, IsNil)
-	r := NewSeekable(f)
-	sto := memory.NewStorage()
-	d := NewDecoder(r, sto.ObjectStorage())
-
-	err = d.Decode()
-	c.Assert(err, IsNil)
-
-	return sto.ObjectStorage().(*memory.ObjectStorage)
-}