Skip to content

Commit 4e6d433

Browse files
zeripathzeripath
committed
Add LFS support
Signed-off-by: Andrew Thornton <[email protected]>
1 parent eabbdca commit 4e6d433

File tree

6 files changed

+222
-13
lines changed

6 files changed

+222
-13
lines changed

‎models/lfs.go

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,11 @@
11
package models
22

33
import (
4+
"crypto/sha256"
5+
"encoding/hex"
46
"errors"
7+
"fmt"
8+
"io"
59

610
"code.gitea.io/gitea/modules/util"
711
)
@@ -16,6 +20,11 @@ type LFSMetaObject struct {
1620
CreatedUnix util.TimeStamp `xorm:"created"`
1721
}
1822

23+
// Pointer returns the string representation of an LFS pointer file
24+
func (m *LFSMetaObject) Pointer() string {
25+
return fmt.Sprintf("%s\n%s%s\nsize %d\n", LFSMetaFileIdentifier, LFSMetaFileOidPrefix, m.Oid, m.Size)
26+
}
27+
1928
// LFSTokenResponse defines the JSON structure in which the JWT token is stored.
2029
// This structure is fetched via SSH and passed by the Git LFS client to the server
2130
// endpoint for authorization.
@@ -67,6 +76,16 @@ func NewLFSMetaObject(m *LFSMetaObject) (*LFSMetaObject, error) {
6776
return m, sess.Commit()
6877
}
6978

79+
// GenerateLFSOid generates a Sha256Sum to represent an oid for arbitrary content
80+
func GenerateLFSOid(content io.Reader) (string, error) {
81+
h := sha256.New()
82+
if _, err := io.Copy(h, content); err != nil {
83+
return "", err
84+
}
85+
sum := h.Sum(nil)
86+
return hex.EncodeToString(sum), nil
87+
}
88+
7089
// GetLFSMetaObjectByOid selects a LFSMetaObject entry from database by its OID.
7190
// It may return ErrLFSObjectNotExist or a database error. If the error is nil,
7291
// the returned pointer is a valid LFSMetaObject.

‎modules/uploader/delete.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ type DeleteRepoFileOptions struct {
2121
}
2222

2323
// DeleteRepoFile deletes a file in the given repository
24-
func DeleteRepoFile(repo *models.Repository, doer *models.User, opts DeleteRepoFileOptions) error {
24+
func DeleteRepoFile(repo *models.Repository, doer *models.User, opts *DeleteRepoFileOptions) error {
2525
t, err := NewTemporaryUploadRepository(repo)
2626
defer t.Close()
2727
if err != nil {

‎modules/uploader/repo.go

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -295,3 +295,61 @@ func (t *TemporaryUploadRepository) DiffIndex() (diff *models.Diff, err error) {
295295

296296
return diff, nil
297297
}
298+
299+
// CheckAttribute checks the given attribute of the provided files
300+
func (t *TemporaryUploadRepository) CheckAttribute(attribute string, args ...string) (map[string]map[string]string, error) {
301+
stdOut := new(bytes.Buffer)
302+
stdErr := new(bytes.Buffer)
303+
304+
timeout := 5 * time.Minute
305+
ctx, cancel := context.WithTimeout(context.Background(), timeout)
306+
defer cancel()
307+
308+
cmdArgs := []string{"check-attr", "-z", attribute, "--cached", "--"}
309+
for _, arg := range args {
310+
if arg != "" {
311+
cmdArgs = append(cmdArgs, arg)
312+
}
313+
}
314+
315+
cmd := exec.CommandContext(ctx, "git", cmdArgs...)
316+
desc := fmt.Sprintf("checkAttr: (git check-attr) %s %v", attribute, cmdArgs)
317+
cmd.Dir = t.basePath
318+
cmd.Stdout = stdOut
319+
cmd.Stderr = stdErr
320+
321+
if err := cmd.Start(); err != nil {
322+
return nil, fmt.Errorf("exec(%s) failed: %v(%v)", desc, err, ctx.Err())
323+
}
324+
325+
pid := process.GetManager().Add(desc, cmd)
326+
err := cmd.Wait()
327+
process.GetManager().Remove(pid)
328+
329+
if err != nil {
330+
err = fmt.Errorf("exec(%d:%s) failed: %v(%v) stdout: %v stderr: %v", pid, desc, err, ctx.Err(), stdOut, stdErr)
331+
return nil, err
332+
}
333+
334+
fields := bytes.Split(stdOut.Bytes(), []byte{'\000'})
335+
336+
if len(fields)%3 != 1 {
337+
return nil, fmt.Errorf("Wrong number of fields in return from check-attr")
338+
}
339+
340+
var name2attribute2info = make(map[string]map[string]string)
341+
342+
for i := 0; i < (len(fields) / 3); i++ {
343+
filename := string(fields[3*i])
344+
attribute := string(fields[3*i+1])
345+
info := string(fields[3*i+2])
346+
attribute2info := name2attribute2info[filename]
347+
if attribute2info == nil {
348+
attribute2info = make(map[string]string)
349+
}
350+
attribute2info[attribute] = info
351+
name2attribute2info[filename] = attribute2info
352+
}
353+
354+
return name2attribute2info, err
355+
}

‎modules/uploader/update.go

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ import (
1010

1111
"code.gitea.io/git"
1212
"code.gitea.io/gitea/models"
13+
"code.gitea.io/gitea/modules/lfs"
14+
"code.gitea.io/gitea/modules/setting"
1315
)
1416

1517
// UpdateRepoFileOptions holds the repository file update options
@@ -25,7 +27,7 @@ type UpdateRepoFileOptions struct {
2527
}
2628

2729
// UpdateRepoFile adds or updates a file in the given repository
28-
func UpdateRepoFile(repo *models.Repository, doer *models.User, opts UpdateRepoFileOptions) error {
30+
func UpdateRepoFile(repo *models.Repository, doer *models.User, opts *UpdateRepoFileOptions) error {
2931
t, err := NewTemporaryUploadRepository(repo)
3032
defer t.Close()
3133
if err != nil {
@@ -63,8 +65,27 @@ func UpdateRepoFile(repo *models.Repository, doer *models.User, opts UpdateRepoF
6365

6466
}
6567

68+
// Check there is no way this can return multiple infos
69+
filename2attribute2info, err := t.CheckAttribute("filter", opts.NewTreeName)
70+
if err != nil {
71+
return err
72+
}
73+
74+
content := opts.Content
75+
var lfsMetaObject *models.LFSMetaObject
76+
77+
if filename2attribute2info[opts.NewTreeName] != nil && filename2attribute2info[opts.NewTreeName]["filter"] == "lfs" {
78+
// OK so we are supposed to LFS this data!
79+
oid, err := models.GenerateLFSOid(strings.NewReader(opts.Content))
80+
if err != nil {
81+
return err
82+
}
83+
lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: int64(len(opts.Content)), RepositoryID: repo.ID}
84+
content = lfsMetaObject.Pointer()
85+
}
86+
6687
// Add the object to the database
67-
objectHash, err := t.HashObject(strings.NewReader(opts.Content))
88+
objectHash, err := t.HashObject(strings.NewReader(content))
6889
if err != nil {
6990
return err
7091
}
@@ -86,6 +107,23 @@ func UpdateRepoFile(repo *models.Repository, doer *models.User, opts UpdateRepoF
86107
return err
87108
}
88109

110+
if lfsMetaObject != nil {
111+
// We have an LFS object - create it
112+
lfsMetaObject, err = models.NewLFSMetaObject(lfsMetaObject)
113+
if err != nil {
114+
return err
115+
}
116+
contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath}
117+
if !contentStore.Exists(lfsMetaObject) {
118+
if err := contentStore.Put(lfsMetaObject, strings.NewReader(opts.Content)); err != nil {
119+
if err2 := repo.RemoveLFSMetaObjectByOid(lfsMetaObject.Oid); err2 != nil {
120+
return fmt.Errorf("Error whilst removing failed inserted LFS object %s: %v (Prev Error: %v)", lfsMetaObject.Oid, err2, err)
121+
}
122+
return err
123+
}
124+
}
125+
}
126+
89127
// Then push this tree to NewBranch
90128
if err := t.Push(doer, commitHash, opts.NewBranch); err != nil {
91129
return err

‎modules/uploader/upload.go

Lines changed: 101 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,10 @@ import (
88
"fmt"
99
"os"
1010
"path"
11+
"strings"
12+
13+
"code.gitea.io/gitea/modules/lfs"
14+
"code.gitea.io/gitea/modules/setting"
1115

1216
"code.gitea.io/git"
1317
"code.gitea.io/gitea/models"
@@ -23,8 +27,27 @@ type UploadRepoFileOptions struct {
2327
Files []string // In UUID format.
2428
}
2529

30+
type uploadInfo struct {
31+
upload *models.Upload
32+
lfsMetaObject *models.LFSMetaObject
33+
}
34+
35+
func cleanUpAfterFailure(infos *[]uploadInfo, t *TemporaryUploadRepository, original error) error {
36+
for _, info := range *infos {
37+
if info.lfsMetaObject == nil {
38+
continue
39+
}
40+
if !info.lfsMetaObject.Existing {
41+
if err := t.repo.RemoveLFSMetaObjectByOid(info.lfsMetaObject.Oid); err != nil {
42+
original = fmt.Errorf("%v, %v", original, err)
43+
}
44+
}
45+
}
46+
return original
47+
}
48+
2649
// UploadRepoFiles uploads files to the given repository
27-
func UploadRepoFiles(repo *models.Repository, doer *models.User, opts UploadRepoFileOptions) error {
50+
func UploadRepoFiles(repo *models.Repository, doer *models.User, opts *UploadRepoFileOptions) error {
2851
if len(opts.Files) == 0 {
2952
return nil
3053
}
@@ -46,22 +69,56 @@ func UploadRepoFiles(repo *models.Repository, doer *models.User, opts UploadRepo
4669
return err
4770
}
4871

72+
names := make([]string, len(uploads))
73+
infos := make([]uploadInfo, len(uploads))
74+
for i, upload := range uploads {
75+
names[i] = upload.Name
76+
infos[i] = uploadInfo{upload: upload}
77+
}
78+
79+
filename2attribute2info, err := t.CheckAttribute("filter", names...)
80+
if err != nil {
81+
return err
82+
}
83+
4984
// Copy uploaded files into repository.
50-
for _, upload := range uploads {
51-
file, err := os.Open(upload.LocalPath())
85+
for i, uploadInfo := range infos {
86+
file, err := os.Open(uploadInfo.upload.LocalPath())
5287
if err != nil {
5388
return err
5489
}
5590
defer file.Close()
5691

57-
objectHash, err := t.HashObject(file)
58-
if err != nil {
59-
return err
92+
var objectHash string
93+
if filename2attribute2info[uploadInfo.upload.Name] != nil && filename2attribute2info[uploadInfo.upload.Name]["filter"] == "lfs" {
94+
// Handle LFS
95+
// FIXME: Inefficient! this should probably happen in models.Upload
96+
oid, err := models.GenerateLFSOid(file)
97+
if err != nil {
98+
return err
99+
}
100+
fileInfo, err := file.Stat()
101+
if err != nil {
102+
return err
103+
}
104+
105+
uploadInfo.lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: fileInfo.Size(), RepositoryID: t.repo.ID}
106+
107+
if objectHash, err = t.HashObject(strings.NewReader(uploadInfo.lfsMetaObject.Pointer())); err != nil {
108+
return err
109+
}
110+
infos[i] = uploadInfo
111+
112+
} else {
113+
if objectHash, err = t.HashObject(file); err != nil {
114+
return err
115+
}
60116
}
61117

62118
// Add the object to the index
63-
if err := t.AddObjectToIndex("100644", objectHash, path.Join(opts.TreePath, upload.Name)); err != nil {
119+
if err := t.AddObjectToIndex("100644", objectHash, path.Join(opts.TreePath, uploadInfo.upload.Name)); err != nil {
64120
return err
121+
65122
}
66123
}
67124

@@ -77,6 +134,43 @@ func UploadRepoFiles(repo *models.Repository, doer *models.User, opts UploadRepo
77134
return err
78135
}
79136

137+
// Now deal with LFS objects
138+
for _, uploadInfo := range infos {
139+
if uploadInfo.lfsMetaObject == nil {
140+
continue
141+
}
142+
uploadInfo.lfsMetaObject, err = models.NewLFSMetaObject(uploadInfo.lfsMetaObject)
143+
if err != nil {
144+
// OK Now we need to cleanup
145+
return cleanUpAfterFailure(&infos, t, err)
146+
}
147+
// Don't move the files yet - we need to ensure that
148+
// everything can be inserted first
149+
}
150+
151+
// OK now we can insert the data into the store - there's no way to clean up the store
152+
// once it's in there, it's in there.
153+
contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath}
154+
for _, uploadInfo := range infos {
155+
if uploadInfo.lfsMetaObject == nil {
156+
continue
157+
}
158+
if !contentStore.Exists(uploadInfo.lfsMetaObject) {
159+
file, err := os.Open(uploadInfo.upload.LocalPath())
160+
if err != nil {
161+
return cleanUpAfterFailure(&infos, t, err)
162+
}
163+
defer file.Close()
164+
// FIXME: Put regenerates the hash and copies the file over.
165+
// I guess this strictly ensures the soundness of the store but this is inefficient.
166+
if err := contentStore.Put(uploadInfo.lfsMetaObject, file); err != nil {
167+
// OK Now we need to cleanup
168+
// Can't clean up the store, once uploaded there they're there.
169+
return cleanUpAfterFailure(&infos, t, err)
170+
}
171+
}
172+
}
173+
80174
// Then push this tree to NewBranch
81175
if err := t.Push(doer, commitHash, opts.NewBranch); err != nil {
82176
return err

‎routers/repo/editor.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -309,7 +309,7 @@ func editFilePost(ctx *context.Context, form auth.EditRepoFileForm, isNewFile bo
309309
message += "\n\n" + form.CommitMessage
310310
}
311311

312-
if err := uploader.UpdateRepoFile(ctx.Repo.Repository, ctx.User, uploader.UpdateRepoFileOptions{
312+
if err := uploader.UpdateRepoFile(ctx.Repo.Repository, ctx.User, &uploader.UpdateRepoFileOptions{
313313
LastCommitID: lastCommit,
314314
OldBranch: oldBranchName,
315315
NewBranch: branchName,
@@ -448,7 +448,7 @@ func DeleteFilePost(ctx *context.Context, form auth.DeleteRepoFileForm) {
448448
message += "\n\n" + form.CommitMessage
449449
}
450450

451-
if err := uploader.DeleteRepoFile(ctx.Repo.Repository, ctx.User, uploader.DeleteRepoFileOptions{
451+
if err := uploader.DeleteRepoFile(ctx.Repo.Repository, ctx.User, &uploader.DeleteRepoFileOptions{
452452
LastCommitID: ctx.Repo.CommitID,
453453
OldBranch: oldBranchName,
454454
NewBranch: branchName,
@@ -583,7 +583,7 @@ func UploadFilePost(ctx *context.Context, form auth.UploadRepoFileForm) {
583583
message += "\n\n" + form.CommitMessage
584584
}
585585

586-
if err := uploader.UploadRepoFiles(ctx.Repo.Repository, ctx.User, uploader.UploadRepoFileOptions{
586+
if err := uploader.UploadRepoFiles(ctx.Repo.Repository, ctx.User, &uploader.UploadRepoFileOptions{
587587
LastCommitID: ctx.Repo.CommitID,
588588
OldBranch: oldBranchName,
589589
NewBranch: branchName,

0 commit comments

Comments
 (0)