storage: filesystem idx generation (wip)

Author: mcuadros

storage/filesystem/config.go

@@ -52,18 +52,15 @@ func (c *ConfigStorage) DeleteRemote(name string) error {
 }
 
 func (c *ConfigStorage) read() (*ConfigFile, error) {
-	fs, path, err := c.dir.Config()
+	f, err := c.dir.Config()
 	if err != nil {
 		return nil, err
 	}
 
-	r, err := fs.Open(path)
-	if err != nil {
-		return nil, err
-	}
+	defer f.Close()
 
-	f := &ConfigFile{}
-	return f, f.Decode(r)
+	config := &ConfigFile{}
+	return config, config.Decode(f)
 }
 
 type ConfigFile struct {

storage/filesystem/internal/dotgit/dotgit.go

@@ -1,18 +1,30 @@
+// https://github.com/git/git/blob/master/Documentation/gitrepository-layout.txt
 package dotgit
 
 import (
+	"crypto/sha1"
 	"errors"
+	"fmt"
+	"io"
 	"os"
 	"strings"
+	"time"
 
 	"gopkg.in/src-d/go-git.v4/core"
+	"gopkg.in/src-d/go-git.v4/storage/filesystem/internal/index"
 	"gopkg.in/src-d/go-git.v4/utils/fs"
 )
 
 const (
 	suffix         = ".git"
 	packedRefsPath = "packed-refs"
 	configPath     = "config"
+
+	objectsPath = "objects"
+	packPath    = "pack"
+
+	packExt = ".pack"
+	idxExt  = ".idx"
 )
 
 var (
@@ -31,23 +43,19 @@ var (
 // The DotGit type represents a local git repository on disk. This
 // type is not zero-value-safe, use the New function to initialize it.
 type DotGit struct {
-	fs   fs.FS
-	path string
+	fs fs.Filesystem
 }
 
 // New returns a DotGit value ready to be used. The path argument must
 // be the absolute path of a git repository directory (e.g.
 // "/foo/bar/.git").
-func New(fs fs.FS, path string) (*DotGit, error) {
-	d := &DotGit{fs: fs, path: path}
-	if _, err := fs.Stat(path); err != nil {
-		if os.IsNotExist(err) {
-			return nil, ErrNotFound
-		}
-		return nil, err
-	}
+func New(fs fs.Filesystem) *DotGit {
+	return &DotGit{fs: fs}
+}
 
-	return d, nil
+// Config returns the path of the config file
+func (d *DotGit) Config() (fs.File, error) {
+	return d.fs.Open(configPath)
 }
 
 // Refs scans the git directory collecting references, which it returns.
@@ -69,96 +77,97 @@ func (d *DotGit) Refs() ([]*core.Reference, error) {
 	return refs, nil
 }
 
-// Packfile returns the path of the packfile (really, it returns the
-// path of the first file in the "objects/pack/" directory with a
-// ".pack" extension.
-func (d *DotGit) Packfile() (fs.FS, string, error) {
-	packDir := d.fs.Join(d.path, "objects", "pack")
+func (d *DotGit) NewObjectPack() (*PackWriter, error) {
+	return newPackWrite(d.fs)
+}
+
+// ObjectsPacks returns the list of availables packfiles
+func (d *DotGit) ObjectsPacks() ([]fs.FileInfo, error) {
+	packDir := d.fs.Join(objectsPath, packPath)
 	files, err := d.fs.ReadDir(packDir)
 	if err != nil {
 		if os.IsNotExist(err) {
-			return nil, "", ErrPackfileNotFound
+			return nil, nil
 		}
 
-		return nil, "", err
+		return nil, err
 	}
 
+	var packs []fs.FileInfo
 	for _, f := range files {
-		if strings.HasSuffix(f.Name(), ".pack") {
-			return d.fs, d.fs.Join(packDir, f.Name()), nil
+		if strings.HasSuffix(f.Name(), packExt) {
+			packs = append(packs, f)
 		}
 	}
 
-	return nil, "", ErrPackfileNotFound
+	return packs, nil
 }
 
-// Idxfile returns the path of the idx file (really, it returns the
-// path of the first file in the "objects/pack/" directory with an
-// ".idx" extension.
-func (d *DotGit) Idxfile() (fs.FS, string, error) {
-	packDir := d.fs.Join(d.path, "objects", "pack")
-	files, err := d.fs.ReadDir(packDir)
+// ObjectPack returns the requested packfile and his idx
+func (d *DotGit) ObjectPack(filename string) (pack, idx fs.File, err error) {
+	if !strings.HasSuffix(filename, packExt) {
+		return nil, nil, fmt.Errorf("a .pack file should be provided")
+	}
+
+	pack, err = d.fs.Open(d.fs.Join(objectsPath, packPath, filename))
 	if err != nil {
 		if os.IsNotExist(err) {
-			return nil, "", ErrIdxNotFound
+			return nil, nil, ErrPackfileNotFound
 		}
 
-		return nil, "", err
+		return
 	}
 
-	for _, f := range files {
-		if strings.HasSuffix(f.Name(), ".idx") {
-			return d.fs, d.fs.Join(packDir, f.Name()), nil
-		}
-	}
-
-	return nil, "", ErrIdxNotFound
-}
-
-// Config returns the path of the config file
-func (d *DotGit) Config() (fs.FS, string, error) {
-	configFile := d.fs.Join(d.path, configPath)
-	if _, err := d.fs.Stat(configFile); err != nil {
+	idxfile := filename[0:len(filename)-len(packExt)] + idxExt
+	idxpath := d.fs.Join(objectsPath, packPath, idxfile)
+	idx, err = d.fs.Open(idxpath)
+	if err != nil {
 		if os.IsNotExist(err) {
-			return nil, "", ErrNotFound
+			return nil, nil, ErrIdxNotFound
 		}
 
-		return nil, "", err
+		return
 	}
 
-	return d.fs, configFile, nil
+	return
 }
 
-// Objectfiles returns a slice with the hashes of objects found under the
+// Objects returns a slice with the hashes of objects found under the
 // .git/objects/ directory.
-func (dg *DotGit) Objectfiles() (fs.FS, []core.Hash, error) {
-	objsDir := dg.fs.Join(dg.path, "objects")
-
-	files, err := dg.fs.ReadDir(objsDir)
+func (d *DotGit) Objects() ([]core.Hash, error) {
+	files, err := d.fs.ReadDir(objectsPath)
 	if err != nil {
 		if os.IsNotExist(err) {
-			return nil, nil, ErrObjfileNotFound
+			return nil, nil
 		}
 
-		return nil, nil, err
+		return nil, err
 	}
 
 	var objects []core.Hash
 	for _, f := range files {
 		if f.IsDir() && len(f.Name()) == 2 && isHex(f.Name()) {
-			objDir := f.Name()
-			d, err := dg.fs.ReadDir(dg.fs.Join(objsDir, objDir))
+			base := f.Name()
+			d, err := d.fs.ReadDir(d.fs.Join(objectsPath, base))
 			if err != nil {
-				return nil, nil, err
+				return nil, err
 			}
 
 			for _, o := range d {
-				objects = append(objects, core.NewHash(objDir+o.Name()))
+				objects = append(objects, core.NewHash(base+o.Name()))
 			}
 		}
 	}
 
-	return dg.fs, objects, nil
+	return objects, nil
+}
+
+// Object return a fs.File poiting the object file, if exists
+func (d *DotGit) Object(h core.Hash) (fs.File, error) {
+	hash := h.String()
+	file := d.fs.Join(objectsPath, hash[0:2], hash[2:40])
+
+	return d.fs.Open(file)
 }
 
 func isHex(s string) bool {
@@ -184,19 +193,78 @@ func isHexAlpha(b byte) bool {
 	return b >= 'a' && b <= 'f' || b >= 'A' && b <= 'F'
 }
 
-// Objectfile returns the path of the object file for a given hash
-// *if the file exists*, otherwise returns an ErrObjfileNotFound error.
-func (d *DotGit) Objectfile(h core.Hash) (fs.FS, string, error) {
-	hash := h.String()
-	objFile := d.fs.Join(d.path, "objects", hash[0:2], hash[2:40])
+type PackWriter struct {
+	fs         fs.Filesystem
+	file       fs.File
+	writer     io.Writer
+	pipeReader io.ReadCloser
+	pipeWriter io.WriteCloser
+	hash       core.Hash
+	index      index.Index
+	result     chan error
+}
 
-	if _, err := d.fs.Stat(objFile); err != nil {
-		if os.IsNotExist(err) {
-			return nil, "", ErrObjfileNotFound
-		}
+func newPackWrite(fs fs.Filesystem) (*PackWriter, error) {
+	r, w := io.Pipe()
+
+	temp := sha1.Sum([]byte(time.Now().String()))
+	filename := fmt.Sprintf(".%x", temp)
+
+	file, err := fs.Create(fs.Join(objectsPath, packPath, filename))
+	if err != nil {
+		return nil, err
+	}
 
-		return nil, "", err
+	writer := &PackWriter{
+		fs:         fs,
+		file:       file,
+		writer:     io.MultiWriter(w, file),
+		pipeReader: r,
+		pipeWriter: w,
+		result:     make(chan error),
 	}
 
-	return d.fs, objFile, nil
+	go writer.buildIndex()
+	return writer, nil
+}
+
+func (w *PackWriter) buildIndex() {
+	defer w.pipeReader.Close()
+	index, hash, err := index.NewFromPackfileInMemory(w.pipeReader)
+	w.index = index
+	w.hash = hash
+
+	w.result <- err
+}
+
+func (w *PackWriter) Write(p []byte) (int, error) {
+	return w.writer.Write(p)
+}
+
+func (w *PackWriter) Close() error {
+	defer func() {
+		close(w.result)
+	}()
+
+	if err := w.file.Close(); err != nil {
+		return err
+	}
+
+	if err := w.pipeWriter.Close(); err != nil {
+		return err
+	}
+
+	if err := <-w.result; err != nil {
+		return err
+	}
+
+	return w.save()
+}
+
+func (w *PackWriter) save() error {
+	base := w.fs.Join(objectsPath, packPath, fmt.Sprintf("pack-%s", w.hash))
+
+	//idx, err := w.fs.Create(fmt.Sprintf("%s.idx", base))
+
+	return w.fs.Rename(w.file.Filename(), fmt.Sprintf("%s.pack", base))
 }