storage: filessytem read multiple packfiles support and index decoding

Author: mcuadros

fixtures/data/git-0a00a25543e6d732dbf4e8e9fec55c8e65fc4e8d.tgz

@@ -6,24 +6,32 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/alcortesm/tgz"
+
 	check "gopkg.in/check.v1"
 	"gopkg.in/src-d/go-git.v4/core"
+	"gopkg.in/src-d/go-git.v4/utils/fs"
 )
 
 var RootFolder = ""
 
 const DataFolder = "data"
 
 var fixtures = []*Fixture{{
-	Tags:         []string{"ofs-delta"},
+	Tags:         []string{"ofs-delta", ".git"},
 	URL:          "https://github.com/git-fixtures/basic",
 	Head:         core.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5"),
 	PackfileHash: core.NewHash("a3fed42da1e8189a077c0e6846c040dcf73fc9dd"),
+	DotGitHash:   core.NewHash("0a00a25543e6d732dbf4e8e9fec55c8e65fc4e8d"),
 }, {
 	Tags:         []string{"ref-delta"},
 	URL:          "https://github.com/git-fixtures/basic",
 	Head:         core.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5"),
 	PackfileHash: core.NewHash("c544593473465e6315ad4182d04d366c4592b829"),
+}, {
+	Tags:       []string{".git", "unpacked", "multi-packfile"},
+	URL:        "https://github.com/src-d/go-git.git",
+	DotGitHash: core.NewHash("174be6bd4292c18160542ae6dc6704b877b8a01a"),
 }, {
 	URL:          "https://github.com/spinnaker/spinnaker",
 	Head:         core.NewHash("06ce06d0fc49646c4de733c45b7788aabad98a6f"),
@@ -49,11 +57,25 @@ func ByURL(url string) Fixtures {
 	return r
 }
 
+func ByTag(tag string) Fixtures {
+	r := make(Fixtures, 0)
+	for _, f := range fixtures {
+		for _, t := range f.Tags {
+			if t == tag {
+				r = append(r, f)
+			}
+		}
+	}
+
+	return r
+}
+
 type Fixture struct {
 	URL          string
 	Tags         []string
 	Head         core.Hash
 	PackfileHash core.Hash
+	DotGitHash   core.Hash
 }
 
 func (f *Fixture) Packfile() io.ReadSeeker {
@@ -76,6 +98,15 @@ func (f *Fixture) Idx() io.ReadSeeker {
 	return file
 }
 
+func (f *Fixture) DotGit() fs.Filesystem {
+	fn := filepath.Join(RootFolder, DataFolder, fmt.Sprintf("git-%s.tgz", f.DotGitHash))
+	path, err := tgz.Extract(fn)
+	if err != nil {
+		panic(err)
+	}
+	return fs.NewOS(path)
+}
+
 type Fixtures []*Fixture
 
 func (g Fixtures) Test(c *check.C, test func(*Fixture)) {

fixtures/data/git-174be6bd4292c18160542ae6dc6704b877b8a01a.tgz

@@ -188,7 +188,7 @@ func (d *Decoder) recallByOffset(o int64) (core.Object, error) {
 		return d.recallByHash(h)
 	}
 
-	return nil, ErrCannotRecall.AddDetails("no object found at offset %d", o)
+	return d.ReadObjectAt(o)
 }
 
 func (d *Decoder) recallByHash(h core.Hash) (core.Object, error) {

fixtures/fixtures.go

@@ -83,7 +83,21 @@ func (s *ReaderSuite) TestDecodeCRCs(c *C) {
 	}
 
 	c.Assert(int(sum), Equals, 78022211966)
+}
+
+func (s *ReaderSuite) TestReadObjectAt(c *C) {
+	f := fixtures.Basic().One()
+
+	scanner := NewScanner(f.Packfile())
+	storage := memory.NewStorage()
+
+	d := NewDecoder(scanner, storage.ObjectStorage())
 
+	// the objects at reference 186, is a delta, so should be recall, without
+	// being read before.
+	obj, err := d.ReadObjectAt(186)
+	c.Assert(err, IsNil)
+	c.Assert(obj.Hash().String(), Equals, "6ecf0ef2c2dffb796033e5a02219af86ec6584e5")
 }
 
 func AssertObjects(c *C, s *memory.Storage, expects []string) {

formats/packfile/decoder.go

@@ -11,7 +11,9 @@ import (
 	"time"
 
 	"gopkg.in/src-d/go-git.v4/core"
-	"gopkg.in/src-d/go-git.v4/storage/filesystem/internal/index"
+	"gopkg.in/src-d/go-git.v4/formats/idxfile"
+	"gopkg.in/src-d/go-git.v4/formats/packfile"
+	"gopkg.in/src-d/go-git.v4/storage/memory"
 	"gopkg.in/src-d/go-git.v4/utils/fs"
 )
 
@@ -34,8 +36,6 @@ var (
 	ErrIdxNotFound = errors.New("idx file not found")
 	// ErrPackfileNotFound is returned by Packfile when the packfile is not found
 	ErrPackfileNotFound = errors.New("packfile not found")
-	// ErrObjfileNotFound is returned by Objectfile when the objectffile is not found
-	ErrObjfileNotFound = errors.New("object file not found")
 	// ErrConfigNotFound is returned by Config when the config is not found
 	ErrConfigNotFound = errors.New("config file not found")
 )
@@ -77,12 +77,14 @@ func (d *DotGit) Refs() ([]*core.Reference, error) {
 	return refs, nil
 }
 
+// NewObjectPack return a writer for a new packfile, it saves the packfile to
+// disk and also generates and save the index for the given packfile.
 func (d *DotGit) NewObjectPack() (*PackWriter, error) {
 	return newPackWrite(d.fs)
 }
 
-// ObjectsPacks returns the list of availables packfiles
-func (d *DotGit) ObjectsPacks() ([]fs.FileInfo, error) {
+// ObjectPacks returns the list of availables packfiles
+func (d *DotGit) ObjectPacks() ([]core.Hash, error) {
 	packDir := d.fs.Join(objectsPath, packPath)
 	files, err := d.fs.ReadDir(packDir)
 	if err != nil {
@@ -93,43 +95,51 @@ func (d *DotGit) ObjectsPacks() ([]fs.FileInfo, error) {
 		return nil, err
 	}
 
-	var packs []fs.FileInfo
+	var packs []core.Hash
 	for _, f := range files {
-		if strings.HasSuffix(f.Name(), packExt) {
-			packs = append(packs, f)
+		if !strings.HasSuffix(f.Name(), packExt) {
+			continue
 		}
+
+		n := f.Name()
+		h := core.NewHash(n[5 : len(n)-5]) //pack-(hash).pack
+		packs = append(packs, h)
+
 	}
 
 	return packs, nil
 }
 
-// ObjectPack returns the requested packfile and his idx
-func (d *DotGit) ObjectPack(filename string) (pack, idx fs.File, err error) {
-	if !strings.HasSuffix(filename, packExt) {
-		return nil, nil, fmt.Errorf("a .pack file should be provided")
-	}
+// ObjectPack returns a fs.File of the given packfile
+func (d *DotGit) ObjectPack(hash core.Hash) (fs.File, error) {
+	file := d.fs.Join(objectsPath, packPath, fmt.Sprintf("pack-%s.pack", hash.String()))
 
-	pack, err = d.fs.Open(d.fs.Join(objectsPath, packPath, filename))
+	pack, err := d.fs.Open(file)
 	if err != nil {
 		if os.IsNotExist(err) {
-			return nil, nil, ErrPackfileNotFound
+			return nil, ErrPackfileNotFound
 		}
 
-		return
+		return nil, err
 	}
 
-	idxfile := filename[0:len(filename)-len(packExt)] + idxExt
-	idxpath := d.fs.Join(objectsPath, packPath, idxfile)
-	idx, err = d.fs.Open(idxpath)
+	return pack, nil
+}
+
+// ObjectPackIdx returns a fs.File of the index file for a given packfile
+func (d *DotGit) ObjectPackIdx(hash core.Hash) (fs.File, error) {
+	file := d.fs.Join(objectsPath, packPath, fmt.Sprintf("pack-%s.idx", hash.String()))
+
+	idx, err := d.fs.Open(file)
 	if err != nil {
 		if os.IsNotExist(err) {
-			return nil, nil, ErrIdxNotFound
+			return nil, ErrPackfileNotFound
 		}
 
-		return
+		return nil, err
 	}
 
-	return
+	return idx, nil
 }
 
 // Objects returns a slice with the hashes of objects found under the
@@ -194,14 +204,15 @@ func isHexAlpha(b byte) bool {
 }
 
 type PackWriter struct {
-	fs     fs.Filesystem
-	sr     io.ReadCloser
-	sw     io.WriteCloser
-	fw     fs.File
-	mw     io.Writer
-	hash   core.Hash
-	index  index.Index
-	result chan error
+	fs fs.Filesystem
+	sr io.ReadCloser
+	sw io.WriteCloser
+	fw fs.File
+	mw io.Writer
+
+	checksum core.Hash
+	index    idxfile.Idxfile
+	result   chan error
 }
 
 func newPackWrite(fs fs.Filesystem) (*PackWriter, error) {
@@ -230,12 +241,23 @@ func newPackWrite(fs fs.Filesystem) (*PackWriter, error) {
 
 func (w *PackWriter) buildIndex() {
 	defer w.sr.Close()
-	index, hash, err := index.NewFromPackfile(w.sr)
+	o := memory.NewStorage().ObjectStorage()
+	s := packfile.NewScannerFromReader(w.sr)
+	d := packfile.NewDecoder(s, o)
+
+	checksum, err := d.Decode()
+	if err != nil {
+		w.result <- err
+		return
+	}
 
-	w.index = index
-	w.hash = hash
+	w.checksum = checksum
+	w.index.PackfileChecksum = checksum
 
-	fmt.Println(hash, w.index)
+	offsets := d.Offsets()
+	for h, crc := range d.CRCs() {
+		w.index.Add(h, uint64(offsets[h]), crc)
+	}
 
 	w.result <- err
 }
@@ -265,9 +287,26 @@ func (w *PackWriter) Close() error {
 }
 
 func (w *PackWriter) save() error {
-	base := w.fs.Join(objectsPath, packPath, fmt.Sprintf("pack-%s", w.hash))
+	base := w.fs.Join(objectsPath, packPath, fmt.Sprintf("pack-%s", w.checksum))
+
+	idx, err := w.fs.Create(fmt.Sprintf("%s.idx", base))
+	if err != nil {
+		return err
+	}
 
-	//idx, err := w.fs.Create(fmt.Sprintf("%s.idx", base))
+	if err := w.encodeIdx(idx); err != nil {
+		return err
+	}
+
+	if err := idx.Close(); err != nil {
+		return err
+	}
 
 	return w.fs.Rename(w.fw.Filename(), fmt.Sprintf("%s.pack", base))
 }
+
+func (w *PackWriter) encodeIdx(writer io.Writer) error {
+	e := idxfile.NewEncoder(writer)
+	_, err := e.Encode(&w.index)
+	return err
+}