do not fetch existing files with the same size & lint files

Author: bl00mber

integrations/lfs_getobject_test.go

@@ -16,7 +16,6 @@ import (
 	"testing"
 
 	"code.gitea.io/gitea/models"
-	"code.gitea.io/gitea/modules/lfs"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/storage"
 	"code.gitea.io/gitea/routers/routes"
@@ -50,7 +49,7 @@ func storeObjectInRepo(t *testing.T, repositoryID int64, content *[]byte) string
 	lfsID++
 	lfsMetaObject, err = models.NewLFSMetaObject(lfsMetaObject)
 	assert.NoError(t, err)
-	contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS}
+	contentStore := &models.ContentStore{ObjectStorage: storage.LFS}
 	exist, err := contentStore.Exists(lfsMetaObject)
 	assert.NoError(t, err)
 	if !exist {

models/lfs.go

@@ -11,13 +11,13 @@ import (
 	"fmt"
 	"io"
 	"path"
-	"time"
-	"strings"
 	"strconv"
+	"strings"
+	"time"
 
-	"code.gitea.io/gitea/modules/timeutil"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/storage"
+	"code.gitea.io/gitea/modules/timeutil"
 
 	"xorm.io/builder"
 )

models/repo.go

@@ -215,12 +215,12 @@ type Repository struct {
 	NumClosedProjects   int `xorm:"NOT NULL DEFAULT 0"`
 	NumOpenProjects     int `xorm:"-"`
 
-	IsPrivate  bool `xorm:"INDEX"`
-	IsEmpty    bool `xorm:"INDEX"`
-	IsArchived bool `xorm:"INDEX"`
-	IsMirror   bool `xorm:"INDEX"`
-	LFS        bool `yaml:"lfs"`
-	LFSServer  string `yaml:"lfs_server"`
+	IsPrivate  bool   `xorm:"INDEX"`
+	IsEmpty    bool   `xorm:"INDEX"`
+	IsArchived bool   `xorm:"INDEX"`
+	IsMirror   bool   `xorm:"INDEX"`
+	LFS        bool   `xorm:"INDEX"`
+	LFSServer  string `xorm:"TEXT"`
 	*Mirror    `xorm:"-"`
 	Status     RepositoryStatus `xorm:"NOT NULL DEFAULT 0"`
 

modules/lfs/server.go

@@ -7,14 +7,14 @@ package lfs
 import (
 	"encoding/base64"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
 	"path"
 	"regexp"
 	"strconv"
 	"strings"
-	"errors"
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/context"

modules/lfsclient/client.go

@@ -5,149 +5,138 @@
 package lfsclient
 
 import (
+	"bytes"
+	"context"
 	"encoding/json"
-  "bytes"
 	"fmt"
 	"io"
 	"net/http"
-	"context"
 	"strconv"
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/storage"
 )
 
 const (
 	metaMediaType = "application/vnd.git-lfs+json"
 )
 
+// BatchRequest encodes json object using in a lfs batch api request
 type BatchRequest struct {
 	Operation string                       `json:"operation"`
 	Transfers []string                     `json:"transfers,omitempty"`
-  Ref       *Reference                   `json:"ref,omitempty"`
-  Objects   []*models.LFSMetaObjectBasic `json:"objects"`
+	Ref       *Reference                   `json:"ref,omitempty"`
+	Objects   []*models.LFSMetaObjectBasic `json:"objects"`
 }
 
+// Reference is a reference field of BatchRequest
 type Reference struct {
-  Name string `json:"name"`
+	Name string `json:"name"`
 }
 
+// packbatch packs lfs batch request to json encoded as bytes
 func packbatch(operation string, transfers []string, ref *Reference, metaObjects []*models.LFSMetaObject) (*bytes.Buffer, error) {
-  metaObjectsBasic := []*models.LFSMetaObjectBasic{}
+	metaObjectsBasic := []*models.LFSMetaObjectBasic{}
 	for _, meta := range metaObjects {
-		metaBasic := &models.LFSMetaObjectBasic{meta.Oid, meta.Size}
+		metaBasic := &models.LFSMetaObjectBasic{Oid: meta.Oid, Size: meta.Size}
 		metaObjectsBasic = append(metaObjectsBasic, metaBasic)
 	}
 
-  reqobj := &BatchRequest{operation, transfers, ref, metaObjectsBasic}
+	reqobj := &BatchRequest{operation, transfers, ref, metaObjectsBasic}
 
-  buf := &bytes.Buffer{}
-  if err := json.NewEncoder(buf).Encode(reqobj); err != nil {
-    return buf, fmt.Errorf("Failed to encode BatchRequest as json. Error: %v", err)
+	buf := &bytes.Buffer{}
+	if err := json.NewEncoder(buf).Encode(reqobj); err != nil {
+		return buf, fmt.Errorf("Failed to encode BatchRequest as json. Error: %v", err)
 	}
-  return buf, nil
+	return buf, nil
 }
 
+// BasicTransferAdapter makes request to lfs server and returns io.ReadCLoser
 func BasicTransferAdapter(ctx context.Context, client *http.Client, href string, size int64) (io.ReadCloser, error) {
-  req, err := http.NewRequestWithContext(ctx, http.MethodGet, href, nil)
-  if err != nil {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, href, nil)
+	if err != nil {
 		return nil, err
 	}
 	req.Header.Set("Content-type", "application/octet-stream")
 	req.Header.Set("Content-Length", strconv.Itoa(int(size)))
 
-  resp, err := client.Do(req)
-  if err != nil {
-    select {
-  	case <-ctx.Done():
-      return nil, ctx.Err()
-  	default:
-  	}
-    return nil, err
-  }
-  defer resp.Body.Close()
-
-  if resp.StatusCode != http.StatusOK {
-    return nil, fmt.Errorf("Failed to query BasicTransferAdapter with response: %s", resp.Status)
+	resp, err := client.Do(req)
+	if err != nil {
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		default:
+		}
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("Failed to query BasicTransferAdapter with response: %s", resp.Status)
 	}
-  return resp.Body, nil
+	return resp.Body, nil
 }
 
-func FetchLFSFilesToContentStore(ctx context.Context, metaObjects []*models.LFSMetaObject, userName string, repo *models.Repository, LFSServer string) error {
-  client := http.DefaultClient
+// FetchLFSFilesToContentStore downloads []LFSMetaObject from lfsServer to ContentStore
+func FetchLFSFilesToContentStore(ctx context.Context, metaObjects []*models.LFSMetaObject, userName string, repo *models.Repository, lfsServer string, contentStore *models.ContentStore) error {
+	client := http.DefaultClient
 
-  rv, err := packbatch("download", nil, nil, metaObjects)
-  if err != nil {
-    return err
-  }
-  batchAPIURL := LFSServer + "/objects/batch"
-  req, err := http.NewRequestWithContext(ctx, http.MethodGet, batchAPIURL, rv)
-  if err != nil {
+	rv, err := packbatch("download", nil, nil, metaObjects)
+	if err != nil {
+		return err
+	}
+	batchAPIURL := lfsServer + "/objects/batch"
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, batchAPIURL, rv)
+	if err != nil {
 		return err
 	}
 	req.Header.Set("Content-type", metaMediaType)
 	req.Header.Set("Accept", metaMediaType)
 
-  resp, err := client.Do(req)
-  if err != nil {
-    select {
-  	case <-ctx.Done():
-      return ctx.Err()
-  	default:
-  	}
-    return err
-  }
-  defer resp.Body.Close()
-
-  if resp.StatusCode != http.StatusOK {
-    return fmt.Errorf("Failed to query Batch with response: %s", resp.Status)
+	resp, err := client.Do(req)
+	if err != nil {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+		}
+		return err
 	}
+	defer resp.Body.Close()
 
-  var respBatch models.BatchResponse
-  err = json.NewDecoder(resp.Body).Decode(&respBatch)
-  if err != nil {
-    return err
-  }
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("Failed to query Batch with response: %s", resp.Status)
+	}
 
-  if len(respBatch.Transfer) == 0 {
-		respBatch.Transfer = "basic"
+	var respBatch models.BatchResponse
+	err = json.NewDecoder(resp.Body).Decode(&respBatch)
+	if err != nil {
+		return err
 	}
 
-  contentStore := &models.ContentStore{ObjectStorage: storage.LFS}
+	if len(respBatch.Transfer) == 0 {
+		respBatch.Transfer = "basic"
+	}
 
-  for _, rep := range respBatch.Objects {
-    rc, err := BasicTransferAdapter(ctx, client, rep.Actions["download"].Href, rep.Size)
+	for _, rep := range respBatch.Objects {
+		rc, err := BasicTransferAdapter(ctx, client, rep.Actions["download"].Href, rep.Size)
+		if err != nil {
+			log.Error("Unable to use BasicTransferAdapter. Error: %v", err)
+			return err
+		}
+		meta, err := repo.GetLFSMetaObjectByOid(rep.Oid)
 		if err != nil {
-  		log.Error("Unable to use BasicTransferAdapter. Error: %v", err)
-  		return err
-  	}
-    meta, err := repo.GetLFSMetaObjectByOid(rep.Oid)
-    if err != nil {
-  		log.Error("Unable to get LFS OID[%s] Error: %v", rep.Oid, err)
-  		return err
-  	}
-
-    // put LFS file to contentStore
-    exist, err := contentStore.Exists(meta)
-    if err != nil {
-      log.Error("Unable to check if LFS OID[%s] exist on %s/%s. Error: %v", meta.Oid, userName, repo.Name, err)
-      return err
-    }
-
-    if exist {
-      // remove collision
-      if _, err := repo.RemoveLFSMetaObjectByOid(meta.Oid); err != nil {
-        return fmt.Errorf("Error whilst removing matched LFS object %s: %v", meta.Oid, err)
-      }
-    }
-
-    if err := contentStore.Put(meta, rc); err != nil {
-      if _, err2 := repo.RemoveLFSMetaObjectByOid(meta.Oid); err2 != nil {
-        return fmt.Errorf("Error whilst removing failed inserted LFS object %s: %v (Prev Error: %v)", meta.Oid, err2, err)
-      }
-      return err
-    }
-  }
+			log.Error("Unable to get LFS OID[%s] Error: %v", rep.Oid, err)
+			return err
+		}
+
+		// put LFS file to contentStore
+		if err := contentStore.Put(meta, rc); err != nil {
+			if _, err2 := repo.RemoveLFSMetaObjectByOid(meta.Oid); err2 != nil {
+				return fmt.Errorf("Error whilst removing failed inserted LFS object %s: %v (Prev Error: %v)", meta.Oid, err2, err)
+			}
+			return err
+		}
+	}
 	return nil
 }