Support non packed objects (#68)

* Support non packed git objects

* Support non packed git objects for Iterator

* Fix error handling from Writer() in FillObject()

* Fix format in func (r *Reader) FillObject(obj core.Object) error

* Fix to return d.addRefsFromPackedRefs() error

And if packed-refs dosen't exist not to return error in d.addRefsFromPackedRefs

* Remove debug code

* Add GoDoc for func (d *DotGit) Objectfile(h core.Hash) (fs.FS, string, error)

* Add GoDoc for func (r *Reader) FillObject(obj core.Object) error

* Add GoDoc for func (d *DotGit) Objectfiles() (fs.FS, []core.Hash, error)

* Fix format in func (d *DotGit) Objectfile(h core.Hash) (fs.FS, string, error)

* Rename value dotGitobjcts -> objsDir

* Change regexp.Compile -> regexp.MustCompile

* Move regexp to variable initialization

* Rename regexp value to be more coherent

* Fix object directory name and object file name to correct character

* Faster Objectfiles func

* Add test for FillObject

* Add GoDoc for func (s *ObjectStorage) Get(h core.Hash) (core.Object, error)

* defer Close()

* Return name values for defer function overwrite the error value.

* Fix error handling in func (s *ObjectStorage) Get()

Return error that gets error except for ErrObjfileNotFound from getFromUnpacked()

* Rename getFromObject -> getFromUnpacked

* Add test for func (d *DotGit) Objectfile(h core.Hash) (fs.FS, string, error)

* Add test for func (d *DotGit) Objectfiles() (fs.FS, []core.Hash, error)

* Faster check git object name

* Faster dotgit_test.go

* Fix Godoc for Objectfiles func

* Refactor variable name in Objectfiles func

* Fix GoDoc for objectfile func

* Fix TestObjectfile func and TestObjectfiles func

* Rename fixobj -> fixObj in Test Objectfile func

* Fix test compare method

* Refactor Get func in object.go

* Refactor getFromUnpacked func in object.go

* Fix GoDoc for ErrObjfileNotFound

* Fix TestObjectfiles for not guarantee the slice order

* Change error no such file or directory to target file not found

* Change spec func (s *ObjectStorage) Get(h core.Hash) (core.Object, error)

return core.ErrObjectNotFound, if index pointer is nil.

* Add space

* storage: Add object type hint parameter to ObjectStorage.getFromUnpacked

Author: sona-tar

formats/objfile/reader.go

@@ -121,3 +121,16 @@ func (r *Reader) Close() (err error) {
 
 	return
 }
+
+// FillObject fills the given object from an object entry
+func (r *Reader) FillObject(obj core.Object) error {
+	obj.SetType(r.header.t)
+	obj.SetSize(r.header.size)
+	w, err := obj.Writer()
+	if err != nil {
+		return err
+	}
+	_, err = io.Copy(w, r.r)
+
+	return err
+}

formats/objfile/reader_test.go

@@ -69,3 +69,35 @@ func (s *SuiteReader) TestReadCorruptZLib(c *C) {
 	_, err := NewReader(source)
 	c.Assert(err, NotNil)
 }
+
+func (s *SuiteReader) TestFillObject(c *C) {
+	for k, fixture := range objfileFixtures {
+		com := fmt.Sprintf("test %d: ", k)
+		hash := core.NewHash(fixture.hash)
+		content, _ := base64.StdEncoding.DecodeString(fixture.content)
+		data, _ := base64.StdEncoding.DecodeString(fixture.data)
+
+		testFillObject(c, bytes.NewReader(data), hash, fixture.t, content, com)
+	}
+}
+
+func testFillObject(c *C, source io.Reader, hash core.Hash, typ core.ObjectType, content []byte, com string) {
+	var o core.Object = &core.MemoryObject{}
+	r, err := NewReader(source)
+	c.Assert(err, IsNil)
+	err = r.FillObject(o)
+	c.Assert(err, IsNil)
+	c.Assert(o.Type(), Equals, typ)
+	c.Assert(o.Size(), Equals, int64(len(content)))
+	c.Assert(o.Hash(), Equals, hash)
+	or, err := o.Reader()
+	c.Assert(err, IsNil)
+	rc, err := ioutil.ReadAll(or)
+	c.Assert(err, IsNil)
+	c.Assert(rc, DeepEquals, content, Commentf("%scontent=%s, expected=%s", base64.StdEncoding.EncodeToString(rc), base64.StdEncoding.EncodeToString(content)))
+	c.Assert(or.Close(), IsNil)
+	_, err = or.Read(make([]byte, 0, 1))
+	c.Assert(err, Equals, nil)
+	ow, err := o.Writer()
+	c.Assert(ow, Equals, o)
+}

storage/filesystem/internal/dotgit/dotgit.go

@@ -22,6 +22,8 @@ var (
 	ErrIdxNotFound = errors.New("idx file not found")
 	// ErrPackfileNotFound is returned by Packfile when the packfile is not found
 	ErrPackfileNotFound = errors.New("packfile not found")
+	// ErrObjfileNotFound is returned by Objectfile when the objectffile is not found
+	ErrObjfileNotFound = errors.New("object file not found")
 	// ErrConfigNotFound is returned by Config when the config is not found
 	ErrConfigNotFound = errors.New("config file not found")
 )
@@ -74,6 +76,10 @@ func (d *DotGit) Packfile() (fs.FS, string, error) {
 	packDir := d.fs.Join(d.path, "objects", "pack")
 	files, err := d.fs.ReadDir(packDir)
 	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, "", ErrPackfileNotFound
+		}
+
 		return nil, "", err
 	}
 
@@ -93,6 +99,10 @@ func (d *DotGit) Idxfile() (fs.FS, string, error) {
 	packDir := d.fs.Join(d.path, "objects", "pack")
 	files, err := d.fs.ReadDir(packDir)
 	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, "", ErrIdxNotFound
+		}
+
 		return nil, "", err
 	}
 
@@ -118,3 +128,75 @@ func (d *DotGit) Config() (fs.FS, string, error) {
 
 	return d.fs, configFile, nil
 }
+
+// Objectfiles returns a slice with the hashes of objects found under the
+// .git/objects/ directory.
+func (dg *DotGit) Objectfiles() (fs.FS, []core.Hash, error) {
+	objsDir := dg.fs.Join(dg.path, "objects")
+
+	files, err := dg.fs.ReadDir(objsDir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil, ErrObjfileNotFound
+		}
+
+		return nil, nil, err
+	}
+
+	var objects []core.Hash
+	for _, f := range files {
+		if f.IsDir() && len(f.Name()) == 2 && isHex(f.Name()) {
+			objDir := f.Name()
+			d, err := dg.fs.ReadDir(dg.fs.Join(objsDir, objDir))
+			if err != nil {
+				return nil, nil, err
+			}
+
+			for _, o := range d {
+				objects = append(objects, core.NewHash(objDir+o.Name()))
+			}
+		}
+	}
+
+	return dg.fs, objects, nil
+}
+
+func isHex(s string) bool {
+	for _, b := range []byte(s) {
+		if isNum(b) {
+			continue
+		}
+		if isHexAlpha(b) {
+			continue
+		}
+
+		return false
+	}
+
+	return true
+}
+
+func isNum(b byte) bool {
+	return b >= '0' && b <= '9'
+}
+
+func isHexAlpha(b byte) bool {
+	return b >= 'a' && b <= 'f' || b >= 'A' && b <= 'F'
+}
+
+// Objectfile returns the path of the object file for a given hash
+// *if the file exists*, otherwise returns an ErrObjfileNotFound error.
+func (d *DotGit) Objectfile(h core.Hash) (fs.FS, string, error) {
+	hash := h.String()
+	objFile := d.fs.Join(d.path, "objects", hash[0:2], hash[2:40])
+
+	if _, err := d.fs.Stat(objFile); err != nil {
+		if os.IsNotExist(err) {
+			return nil, "", ErrObjfileNotFound
+		}
+
+		return nil, "", err
+	}
+
+	return d.fs, objFile, nil
+}

storage/filesystem/internal/dotgit/dotgit_test.go

@@ -22,6 +22,7 @@ var initFixtures = [...]struct {
 	capabilities [][2]string
 	packfile     string
 	idxfile      string
+	objectfiles  []fixtureObject
 }{
 	{
 		name: "spinnaker",
@@ -37,7 +38,47 @@ var initFixtures = [...]struct {
 	}, {
 		name: "empty",
 		tgz:  "fixtures/empty-gitdir.tgz",
+	}, {
+		name: "unpacked",
+		tgz:  "fixtures/unpacked-objects-no-packfile-no-idx.tgz",
+		objectfiles: []fixtureObject{
+			fixtureObject{
+				path: "objects/1e/0304e3cb54d0ad612ad70f1f15a285a65a4b8e",
+				hash: "1e0304e3cb54d0ad612ad70f1f15a285a65a4b8e",
+			},
+			fixtureObject{
+				path: "objects/5e/fb9bc29c482e023e40e0a2b3b7e49cec842034",
+				hash: "5efb9bc29c482e023e40e0a2b3b7e49cec842034",
+			},
+			fixtureObject{
+				path: "objects/e6/9de29bb2d1d6434b8b29ae775ad8c2e48c5391",
+				hash: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391",
+			},
+		},
 	},
+	{
+		name: "unpacked-dummy",
+		tgz:  "fixtures/unpacked-objects-exist-one-dummy-object-no-packfile-no-idx.tgz",
+		objectfiles: []fixtureObject{
+			fixtureObject{
+				path: "objects/1e/0304e3cb54d0ad612ad70f1f15a285a65a4b8e",
+				hash: "1e0304e3cb54d0ad612ad70f1f15a285a65a4b8e",
+			},
+			fixtureObject{
+				path: "objects/5e/fb9bc29c482e023e40e0a2b3b7e49cec842034",
+				hash: "5efb9bc29c482e023e40e0a2b3b7e49cec842034",
+			},
+			fixtureObject{
+				path: "objects/e6/9de29bb2d1d6434b8b29ae775ad8c2e48c5391",
+				hash: "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391",
+			},
+		},
+	},
+}
+
+type fixtureObject struct {
+	path string
+	hash string
 }
 
 type fixture struct {
@@ -47,6 +88,7 @@ type fixture struct {
 	capabilities *common.Capabilities // expected capabilities
 	packfile     string               // path of the packfile
 	idxfile      string               // path of the idxfile
+	objectfiles  []fixtureObject      // path and hash of the object files
 }
 
 type SuiteDotGit struct {
@@ -77,6 +119,7 @@ func (s *SuiteDotGit) SetUpSuite(c *C) {
 
 		f.packfile = init.packfile
 		f.idxfile = init.idxfile
+		f.objectfiles = init.objectfiles
 
 		s.fixtures[init.name] = f
 	}
@@ -193,11 +236,11 @@ func (s *SuiteDotGit) TestPackfile(c *C) {
 		}, {
 			fixture: "empty",
 			fn:      packfile,
-			err:     ".* no such file or directory",
+			err:     "packfile not found",
 		}, {
 			fixture: "empty",
 			fn:      idxfile,
-			err:     ".* no such file or directory",
+			err:     "idx file not found",
 		}, {
 			fixture: "no-packfile-no-idx",
 			fn:      packfile,
@@ -224,9 +267,87 @@ func (s *SuiteDotGit) TestPackfile(c *C) {
 	}
 }
 
+func (s *SuiteDotGit) TestObjectfiles(c *C) {
+	for _, test := range [...]struct {
+		fixture string
+		err     error
+	}{
+		{
+			fixture: "unpacked",
+		},
+		{
+			fixture: "unpacked-dummy",
+		}, {
+			fixture: "empty",
+			err:     ErrObjfileNotFound,
+		}, {
+			fixture: "no-packfile-no-idx",
+		},
+	} {
+		com := Commentf("fixture = %s", test.fixture)
+
+		fix, dir := s.newFixtureDir(c, test.fixture)
+
+		_, hashes, err := dir.Objectfiles()
+
+		if test.err != nil {
+			c.Assert(err, Equals, test.err, com)
+		} else {
+			c.Assert(err, IsNil, com)
+			c.Assert(len(hashes), Equals, len(fix.objectfiles), com)
+
+			for _, hash := range hashes {
+				c.Assert(containsObject(fix.objectfiles, hash), Equals, true, com)
+			}
+		}
+	}
+}
+
+func (s *SuiteDotGit) TestObjectfile(c *C) {
+	for _, test := range [...]struct {
+		fixture string
+		err     error
+	}{
+		{
+			fixture: "unpacked",
+		}, {
+			fixture: "empty",
+			err:     ErrObjfileNotFound,
+		}, {
+			fixture: "no-packfile-no-idx",
+			err:     ErrObjfileNotFound,
+		},
+	} {
+		com := Commentf("fixture = %s", test.fixture)
+
+		fix, dir := s.newFixtureDir(c, test.fixture)
+
+		for _, fixObj := range fix.objectfiles {
+			_, path, err := dir.Objectfile(core.NewHash(fixObj.hash))
+
+			if test.err != nil {
+				c.Assert(err, Equals, test.err, com)
+			} else {
+				c.Assert(err, IsNil, com)
+				c.Assert(strings.HasSuffix(path, fixObj.path),
+					Equals, true, com)
+			}
+		}
+	}
+}
+
 type getPathFn func(*DotGit) (fs.FS, string, error)
 
 func noExt(path string) string {
 	ext := filepath.Ext(path)
 	return path[0 : len(path)-len(ext)]
 }
+
+func containsObject(objs []fixtureObject, hash core.Hash) bool {
+	for _, o := range objs {
+		if strings.ToLower(o.hash) == strings.ToLower(hash.String()) {
+			return true
+		}
+	}
+	return false
+}

storage/filesystem/internal/dotgit/fixtures/unpacked-objects-exist-one-dummy-object-no-packfile-no-idx.tgz

@@ -32,10 +32,9 @@ func (d *DotGit) addRefsFromPackedRefs(refs *[]*core.Reference) (err error) {
 	path := d.fs.Join(d.path, packedRefsPath)
 	f, err := d.fs.Open(path)
 	if err != nil {
-		if err == os.ErrNotExist {
+		if os.IsNotExist(err) {
 			return nil
 		}
-
 		return err
 	}
 
@@ -44,7 +43,6 @@ func (d *DotGit) addRefsFromPackedRefs(refs *[]*core.Reference) (err error) {
 			err = errClose
 		}
 	}()
-
 	s := bufio.NewScanner(f)
 	for s.Scan() {
 		ref, err := d.processLine(s.Text())