Adds support to open local repositories and to use file-based object storage (#55)

* remove some comments

* idx writer/reader

* Shut up ssh tests, they are annoying

* Add file scheme test to clients

* Add dummy file client

* Add test fot file client

* Make tests use fixture endpoint

* add parser for packed-refs format

* add parser for packed-refs format

* WIP adding dir.Refs() tests

* Add test for fixture refs

* refs parser for the refs directory

* Documentation

* Add Capabilities to file client

* tgz.Exatract now accpets a path instead of a Reader

* fix bug in idxfile fanout calculation

* remove dead code

* packfile documentation

* clean packfile parser code

* add core.Object.Content() and returns errors for core.ObjectStorage.Iter()

* add seekable storage

* add dir repos to NewRepository

* clean prints

* Add dir client documentation to README

* Organize the README

* README

* Clean tgz package

* Clean temp dirs after tgz tests

* Gometalinter on gitdir

* Clean pattern function

* metalinter tgz

* metalinter gitdir

* gitdir coverage and remove seekable packfile filedescriptor leak

* gitdir Idxfile tests and remove file descriptor leak

* gitdir Idxfile tests when no idx is found

* clean storage/seekable/internal/index and some formats/idxfile API issues

* clean storage/seekable

* clean formats/idx

* turn packfile/doc.go into packfile/doc.txt

* move formats/packfile/reader to decoder

* fix packfile decoder error names

* improve documentation

* comment packfile decoder errors

* comment public API (format/packfile)

* remve duplicated code in packfile decoder test

* move tracking_reader into an internal package and clean it

* use iota for packfile format

* rename packfile parse.go to packfile object_at.go

* clean packfile deltas

* fix delta header size bug

* improve delta documentation

* clean packfile deltas

* clean packfiles deltas

* clean repository.go

* Remove go 1.5 from Travis CI

Because go 1.5 does not suport internal packages.

* change local repo scheme to local://

* change "local://" to "file://" as the local scheme

* fix broken indentation

* shortens names of variables in short scopes

* more shortening of variable names

* more shortening of variable names

* Rename git dir client to "file", as the scheme used for it

* Fix file format ctor name, now that the package name has change

* Sortcut local repo constructor to not use remotes

The object storage is build directly in the repository ctor, instead
of creating a remote and waiting for the user to pull it.

* update README and fix some errors in it

* remove file scheme client

* Local respositories has now a new ctor

This is, they are no longer identified by the scheme of the URL, but are
created different from inception.

* remove unused URL field form Repository

* move all git dir logic to seekable sotrage ctor

* fix documentation

* Make formats/file/dir an internal package to storage/seekable

* change package storage/seekable to storage/fs

* clean storage/fs

* overall storage/fs clean

* more cleaning

* some metalinter fixes

* upgrade cshared to last changes

* remove dead code

* fix test error info

* remove file scheme check from clients

* fix test error message

* fix test error message

* fix error messages

* style changes

* fix comments everywhere

* style changes

* style changes

* scaffolding and tests for local packfiles without ifx files

* outsource index building from packfile to the packfile decoder

* refactor packfile header reading into a new function

* move code to generate index from packfile back to index package

* add header parsing

* fix documentation errata

* add undeltified and OFS delta support for index building from the packfile

* add tests for packfile with ref-deltas

* support for packfiles with ref-deltas and no idx

* refactor packfile format parser to reuse code

* refactor packfile format parser to reuse code

* refactor packfile format parser to reuse code

* refactor packfile format parser to reuse code

* refactor packfile format parser to reuse code

* WIP refactor packfile format parser to reuse code

* refactor packfile format parser to reuse code

* remove prints from tests

* remove prints from tests

* refactor packfile.core into packfile.parser

* rename packfile reader to something that shows it is a recaller

* rename cannot recall error

* rename packfile.Reader to packfile.ReadRecaller and document

* speed up test by using StreamReader instead of SeekableReader when possible

* clean packfile StreamReader

* stream_reader tests

* refactor packfile.StreamReader into packfile.StreamReadRecaller

* refactor packfile.SeekableReader into packfile.SeekableReadRecaller and document it

* generalize packfile.StreamReadRecaller test to all packfile.ReadRecaller implementations

* speed up storage/fs tests

* speed up tests in . by loading packfiles in memory

* speed up repository tests by using and smaller fixture

* restore doc.go files

* rename packfile.ReadRecaller implementations to shorter names

* update comments to type changes

* packfile.Parser test (WIP)

* packfile.Parser tests and add ForgetAll() to packfile.ReadRecaller

* add test for packfile.ReadRecaller.ForgetAll()

* clarify seekable being able to recallByOffset forgetted objects

* use better names for internal maps

* metalinter packfile package

* speed up some tests

* documentation fixes

* change storage.fs package name to storage.proxy to avoid confusion with new filesystem support

* New fs package and os transparent implementation

Now NewRepositoryFromFS receives a fs and a path and tests are
modified accordingly, but it is still not using for anything.

* add fs to gitdir and proxy.store

* reduce fs interface for easier implementation

* remove garbage dirs from tgz tests

* change file name gitdir/dir.go to gitdir/gitdir.go

* fs.OS tests

* metalinter utils/fs

* add NewRepositoryFromFS documentation to README

* Readability fixes to README

* move tgz to an external dependency

* move filesystem impl. example to example dir

* rename proxy/store.go to proxy/storage.go for coherence with memory/storage.go

* rename proxy package to seekable

Author: alcortesm

.travis.yml

@@ -2,7 +2,6 @@ language: go
 
 go:
   - 1.4
-  - 1.5
   - 1.6
   - tip
 

README.md

@@ -49,7 +49,10 @@ if err := r.PullDefault(); err != nil {
 	panic(err)
 }
 
-iter := r.Commits()
+iter, err := r.Commits()
+if err != nil {
+	panic(err)
+}
 defer iter.Close()
 
 for {
@@ -112,6 +115,61 @@ if err != nil {
 fmt.Println(commit)
 ```
 
+Creating a repository from an ordinary local git directory (that has been
+previously prepared by running `git gc` on it).
+
+```go
+// Download any git repository and prepare it as as follows:
+//
+//   $ git clone https://github.com/src-d/go-git /tmp/go-git
+//   $ pushd /tmp/go-git ; git gc ; popd
+//
+// Then, create a go-git repository from the local content
+// and print its commits as follows:
+
+package main
+
+import (
+	"fmt"
+	"io"
+
+	"gopkg.in/src-d/go-git.v3"
+	"gopkg.in/src-d/go-git.v3/utils/fs"
+)
+
+func main() {
+	fs := fs.NewOS() // a simple proxy for the local host filesystem
+	path := "/tmp/go-git/.git"
+
+	repo, err := git.NewRepositoryFromFS(fs, path)
+	if err != nil {
+		panic(err)
+	}
+
+	iter, err := repo.Commits()
+	if err != nil {
+		panic(err)
+	}
+	defer iter.Close()
+
+	for {
+		commit, err := iter.Next()
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			panic(err)
+		}
+
+		fmt.Println(commit)
+	}
+}
+```
+
+Implementing your own filesystem will let you access repositories stored on
+remote services (e.g. amazon S3), see the
+[examples](https://github.com/src-d/go-git/tree/master/examples/fs_implementation/)
+directory for a simple filesystem implementation and usage.
 
 Wrapping
 --------

blame_test.go

@@ -1,6 +1,8 @@
 package git
 
 import (
+	"bytes"
+	"io/ioutil"
 	"os"
 
 	"gopkg.in/src-d/go-git.v3/core"
@@ -19,24 +21,23 @@ var _ = Suite(&BlameCommon{})
 func (s *BlameCommon) SetUpSuite(c *C) {
 	s.repos = make(map[string]*Repository, 0)
 	for _, fixRepo := range fixtureRepos {
-		repo := NewPlainRepository()
-		repo.URL = fixRepo.url
+		r := NewPlainRepository()
 
-		d, err := os.Open(fixRepo.packfile)
+		f, err := os.Open(fixRepo.packfile)
 		c.Assert(err, IsNil)
 
-		r := packfile.NewReader(d)
-		// TODO: how to know the format of a pack file ahead of time?
-		// Some info at:
-		// https://codewords.recurse.com/issues/three/unpacking-git-packfiles
-		r.Format = packfile.OFSDeltaFormat
+		data, err := ioutil.ReadAll(f)
+		c.Assert(err, IsNil)
+
+		stream := packfile.NewStream(bytes.NewReader(data))
 
-		_, err = r.Read(repo.Storage)
+		d := packfile.NewDecoder(stream)
+		err = d.Decode(r.Storage)
 		c.Assert(err, IsNil)
 
-		c.Assert(d.Close(), IsNil)
+		c.Assert(f.Close(), IsNil)
 
-		s.repos[fixRepo.url] = repo
+		s.repos[fixRepo.url] = r
 	}
 }
 
@@ -48,22 +49,22 @@ type blameTest struct {
 }
 
 func (s *BlameCommon) mockBlame(t blameTest, c *C) (blame *Blame) {
-	repo, ok := s.repos[t.repo]
+	r, ok := s.repos[t.repo]
 	c.Assert(ok, Equals, true)
 
-	commit, err := repo.Commit(core.NewHash(t.rev))
-	c.Assert(err, IsNil, Commentf("%v: repo=%s, rev=%s", err, repo, t.rev))
+	commit, err := r.Commit(core.NewHash(t.rev))
+	c.Assert(err, IsNil, Commentf("%v: repo=%s, rev=%s", err, r, t.rev))
 
-	file, err := commit.File(t.path)
+	f, err := commit.File(t.path)
 	c.Assert(err, IsNil)
-	lines, err := file.Lines()
+	lines, err := f.Lines()
 	c.Assert(err, IsNil)
 	c.Assert(len(t.blames), Equals, len(lines), Commentf(
 		"repo=%s, path=%s, rev=%s: the number of lines in the file and the number of expected blames differ (len(blames)=%d, len(lines)=%d)\nblames=%#q\nlines=%#q", t.repo, t.path, t.rev, len(t.blames), len(lines), t.blames, lines))
 
 	blamedLines := make([]*line, 0, len(t.blames))
 	for i := range t.blames {
-		commit, err := repo.Commit(core.NewHash(t.blames[i]))
+		commit, err := r.Commit(core.NewHash(t.blames[i]))
 		c.Assert(err, IsNil)
 		l := &line{
 			author: commit.Author.Email,
@@ -82,17 +83,17 @@ func (s *BlameCommon) mockBlame(t blameTest, c *C) (blame *Blame) {
 // run a blame on all the suite's tests
 func (s *BlameCommon) TestBlame(c *C) {
 	for _, t := range blameTests {
-		expected := s.mockBlame(t, c)
+		exp := s.mockBlame(t, c)
 
-		repo, ok := s.repos[t.repo]
+		r, ok := s.repos[t.repo]
 		c.Assert(ok, Equals, true)
 
-		commit, err := repo.Commit(core.NewHash(t.rev))
+		commit, err := r.Commit(core.NewHash(t.rev))
 		c.Assert(err, IsNil)
 
-		obtained, err := commit.Blame(t.path)
+		obt, err := commit.Blame(t.path)
 		c.Assert(err, IsNil)
-		c.Assert(obtained, DeepEquals, expected)
+		c.Assert(obt, DeepEquals, exp)
 	}
 }
 
@@ -105,16 +106,18 @@ func repeat(s string, n int) []string {
 	for i := 0; i < n; i++ {
 		r = append(r, s)
 	}
+
 	return r
 }
 
 // utility function to concat slices
 func concat(vargs ...[]string) []string {
-	var result []string
+	var r []string
 	for _, ss := range vargs {
-		result = append(result, ss...)
+		r = append(r, ss...)
 	}
-	return result
+
+	return r
 }
 
 var blameTests = [...]blameTest{

clients/common.go

@@ -55,10 +55,10 @@ func NewGitUploadPackService(repoURL string) (common.GitUploadPackService, error
 	if err != nil {
 		return nil, fmt.Errorf("invalid url %q", repoURL)
 	}
-	service, ok := KnownProtocols[u.Scheme]
+	s, ok := KnownProtocols[u.Scheme]
 	if !ok {
 		return nil, fmt.Errorf("unsupported scheme %q", u.Scheme)
 	}
 
-	return service, nil
+	return s, nil
 }

clients/common/common.go

@@ -236,11 +236,7 @@ func (r *GitUploadPackInfo) decodeHeaderLine(line string) {
 }
 
 func (r *GitUploadPackInfo) isValidLine(line string) bool {
-	if line[0] == '#' {
-		return false
-	}
-
-	return true
+	return line[0] != '#'
 }
 
 func (r *GitUploadPackInfo) readLine(line string) {

clients/common_test.go

@@ -3,23 +3,41 @@ package clients
 import (
 	"fmt"
 	"io"
+	"os"
 	"testing"
 
-	. "gopkg.in/check.v1"
 	"gopkg.in/src-d/go-git.v3/clients/common"
+
+	"github.com/alcortesm/tgz"
+	. "gopkg.in/check.v1"
 )
 
 func Test(t *testing.T) { TestingT(t) }
 
-type SuiteCommon struct{}
+type SuiteCommon struct {
+	dirFixturePath string
+}
 
 var _ = Suite(&SuiteCommon{})
 
+const fixtureTGZ = "../storage/seekable/internal/gitdir/fixtures/spinnaker-gc.tgz"
+
+func (s *SuiteCommon) SetUpSuite(c *C) {
+	var err error
+	s.dirFixturePath, err = tgz.Extract(fixtureTGZ)
+	c.Assert(err, IsNil)
+}
+
+func (s *SuiteCommon) TearDownSuite(c *C) {
+	err := os.RemoveAll(s.dirFixturePath)
+	c.Assert(err, IsNil)
+}
+
 func (s *SuiteCommon) TestNewGitUploadPackService(c *C) {
 	var tests = [...]struct {
-		input    string
-		err      bool
-		expected string
+		input string
+		err   bool
+		exp   string
 	}{
 		{"://example.com", true, "<nil>"},
 		{"badscheme://github.com/src-d/go-git", true, "<nil>"},
@@ -30,8 +48,10 @@ func (s *SuiteCommon) TestNewGitUploadPackService(c *C) {
 
 	for i, t := range tests {
 		output, err := NewGitUploadPackService(t.input)
-		c.Assert(err != nil, Equals, t.err, Commentf("%d) %q: wrong error value", i, t.input))
-		c.Assert(typeAsString(output), Equals, t.expected, Commentf("%d) %q: wrong type", i, t.input))
+		c.Assert(err != nil, Equals, t.err,
+			Commentf("%d) %q: wrong error value (was: %s)", i, t.input, err))
+		c.Assert(typeAsString(output), Equals, t.exp,
+			Commentf("%d) %q: wrong type", i, t.input))
 	}
 }
 
@@ -70,7 +90,6 @@ func (s *SuiteCommon) TestInstallProtocol(c *C) {
 
 	for i, t := range tests {
 		if t.panic {
-			fmt.Println(t.service == nil)
 			c.Assert(func() { InstallProtocol(t.scheme, t.service) }, PanicMatches, `nil service`)
 			continue
 		}

clients/ssh/git_upload_pack_test.go

@@ -1,3 +1,5 @@
+// +build ssh
+
 package ssh
 
 import (

common_test.go

@@ -1,7 +1,9 @@
 package git
 
 import (
+	"bytes"
 	"io"
+	"io/ioutil"
 	"os"
 	"testing"
 
@@ -29,21 +31,22 @@ func (s *MockGitUploadPackService) ConnectWithAuth(url common.Endpoint, auth com
 }
 
 func (s *MockGitUploadPackService) Info() (*common.GitUploadPackInfo, error) {
-	hash := core.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5")
+	h := core.NewHash("6ecf0ef2c2dffb796033e5a02219af86ec6584e5")
 
-	cap := common.NewCapabilities()
-	cap.Decode("6ecf0ef2c2dffb796033e5a02219af86ec6584e5 HEADmulti_ack thin-pack side-band side-band-64k ofs-delta shallow no-progress include-tag multi_ack_detailed no-done symref=HEAD:refs/heads/master agent=git/2:2.4.8~dbussink-fix-enterprise-tokens-compilation-1167-gc7006cf")
+	c := common.NewCapabilities()
+	c.Decode("6ecf0ef2c2dffb796033e5a02219af86ec6584e5 HEADmulti_ack thin-pack side-band side-band-64k ofs-delta shallow no-progress include-tag multi_ack_detailed no-done symref=HEAD:refs/heads/master agent=git/2:2.4.8~dbussink-fix-enterprise-tokens-compilation-1167-gc7006cf")
 
 	return &common.GitUploadPackInfo{
-		Capabilities: cap,
-		Head:         hash,
-		Refs:         map[string]core.Hash{"refs/heads/master": hash},
+		Capabilities: c,
+		Head:         h,
+		Refs:         map[string]core.Hash{"refs/heads/master": h},
 	}, nil
 }
 
 func (s *MockGitUploadPackService) Fetch(*common.GitUploadPackRequest) (io.ReadCloser, error) {
 	var err error
 	s.RC, err = os.Open("formats/packfile/fixtures/git-fixture.ref-delta")
+
 	return s.RC, err
 }
 
@@ -65,20 +68,28 @@ func unpackFixtures(c *C, fixtures ...[]packedFixture) map[string]*Repository {
 			if _, existing := repos[fixture.url]; existing {
 				continue
 			}
-			repos[fixture.url] = NewPlainRepository()
 
-			d, err := os.Open(fixture.packfile)
-			c.Assert(err, IsNil)
+			comment := Commentf("fixture packfile: %q", fixture.packfile)
 
-			r := packfile.NewReader(d)
-			r.Format = packfile.OFSDeltaFormat // This is hardcoded because we don't have a good way to sniff the format
+			repos[fixture.url] = NewPlainRepository()
 
-			_, err = r.Read(repos[fixture.url].Storage)
+			f, err := os.Open(fixture.packfile)
+			c.Assert(err, IsNil, comment)
+
+			// increase memory consumption to speed up tests
+			data, err := ioutil.ReadAll(f)
 			c.Assert(err, IsNil)
+			memStream := bytes.NewReader(data)
+			r := packfile.NewStream(memStream)
+
+			d := packfile.NewDecoder(r)
+			err = d.Decode(repos[fixture.url].Storage)
+			c.Assert(err, IsNil, comment)
 
-			c.Assert(d.Close(), IsNil)
+			c.Assert(f.Close(), IsNil, comment)
 		}
 	}
+
 	return repos
 }
 

core/object.go

@@ -34,16 +34,16 @@ type Object interface {
 	SetType(ObjectType)
 	Size() int64
 	SetSize(int64)
+	Content() []byte
 	Reader() (ObjectReader, error)
 	Writer() (ObjectWriter, error)
 }
 
 // ObjectStorage generic storage of objects
 type ObjectStorage interface {
-	New() (Object, error)
 	Set(Object) (Hash, error)
 	Get(Hash) (Object, error)
-	Iter(ObjectType) ObjectIter
+	Iter(ObjectType) (ObjectIter, error)
 }
 
 // ObjectType internal object type's