Add bucket storage for attachments

Author: masudur-rahman

models/attachment.go

@@ -5,9 +5,9 @@
 package models
 
 import (
+	"context"
 	"fmt"
 	"io"
-	"os"
 	"path"
 
 	"code.gitea.io/gitea/modules/setting"
@@ -16,6 +16,7 @@ import (
 
 	"github.com/go-xorm/xorm"
 	gouuid "github.com/satori/go.uuid"
+	"gocloud.dev/blob"
 )
 
 // Attachment represent a attachment of issue/comment/release.
@@ -71,33 +72,41 @@ func (a *Attachment) DownloadURL() string {
 	return fmt.Sprintf("%sattachments/%s", setting.AppURL, a.UUID)
 }
 
-// NewAttachment creates a new attachment object.
-func NewAttachment(attach *Attachment, buf []byte, file io.Reader) (_ *Attachment, err error) {
-	attach.UUID = gouuid.NewV4().String()
-
-	localPath := attach.LocalPath()
-	if err = os.MkdirAll(path.Dir(localPath), os.ModePerm); err != nil {
-		return nil, fmt.Errorf("MkdirAll: %v", err)
+// uploadAttachmentToBucket uploads attachments to bucket
+func (attach *Attachment) UploadAttachmentToBucket(buf []byte, file io.Reader) (*Attachment, error) {
+	ctx := context.Background()
+	bucket, err := blob.OpenBucket(ctx, setting.FileStorage.Bucket)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to setup bucket: %v", err)
 	}
+	bucket = blob.PrefixedBucket(bucket, "data/attachments/")
 
-	fw, err := os.Create(localPath)
+	bucketWriter, err := bucket.NewWriter(ctx, attach.LocalPath(), nil)
 	if err != nil {
-		return nil, fmt.Errorf("Create: %v", err)
+		return nil, fmt.Errorf("Failed to obtain writer: %v", err)
 	}
-	defer fw.Close()
+	var fileSize int64
+	if _, err = bucketWriter.Write(buf); err != nil {
+		return nil, fmt.Errorf("error occurred while writing: %v", err)
+	} else if fileSize, err = io.Copy(bucketWriter, file); err != nil {
+		return nil, fmt.Errorf("error occurred while copying: %v", err)
+	}
+	attach.Size = fileSize
 
-	if _, err = fw.Write(buf); err != nil {
-		return nil, fmt.Errorf("Write: %v", err)
-	} else if _, err = io.Copy(fw, file); err != nil {
-		return nil, fmt.Errorf("Copy: %v", err)
+	if err = bucketWriter.Close(); err != nil {
+		return nil, fmt.Errorf("Failed to close: %v", err)
 	}
+	return attach, nil
+}
+
+// NewAttachment creates a new attachment object.
+func NewAttachment(attach *Attachment, buf []byte, file io.Reader) (_ *Attachment, err error) {
+	attach.UUID = gouuid.NewV4().String()
 
-	// Update file size
-	var fi os.FileInfo
-	if fi, err = fw.Stat(); err != nil {
-		return nil, fmt.Errorf("file size: %v", err)
+	attach, err = attach.UploadAttachmentToBucket(buf, file)
+	if err != nil {
+		return nil, err
 	}
-	attach.Size = fi.Size()
 
 	if _, err := x.Insert(attach); err != nil {
 		return nil, err
@@ -185,6 +194,49 @@ func getAttachmentByReleaseIDFileName(e Engine, releaseID int64, fileName string
 	return attach, nil
 }
 
+// GetAttachmentReader provides attachment reader from bucket
+func (attach *Attachment) GetAttachmentReader() (io.ReadCloser, error) {
+	ctx := context.Background()
+	bucket, err := blob.OpenBucket(ctx, setting.FileStorage.Bucket)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to setup bucket %v", err)
+	}
+	bucket = blob.PrefixedBucket(bucket, "data/attachments/")
+
+	exist, err := bucket.Exists(ctx, attach.LocalPath())
+	if err != nil {
+		return nil, err
+	} else if !exist {
+		return nil, fmt.Errorf("Attachment not found")
+	}
+
+	reader, err := bucket.NewReader(ctx, attach.LocalPath(), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	return reader, nil
+}
+
+// deleteAttachmentFromBucket deletes attachments from bucket
+func (attach *Attachment) deleteAttachmentFromBucket() error {
+	ctx := context.Background()
+	bucket, err := blob.OpenBucket(ctx, setting.FileStorage.Bucket)
+	if err != nil {
+		return fmt.Errorf("Failed to setup bucket: %v", err)
+	}
+	bucket = blob.PrefixedBucket(bucket, "data/attachments/")
+
+	exist, err := bucket.Exists(ctx, attach.LocalPath())
+	if err != nil {
+		return err
+	} else if !exist {
+		return fmt.Errorf("repo avatar not found")
+	}
+
+	return bucket.Delete(ctx, attach.LocalPath())
+}
+
 // DeleteAttachment deletes the given attachment and optionally the associated file.
 func DeleteAttachment(a *Attachment, remove bool) error {
 	_, err := DeleteAttachments([]*Attachment{a}, remove)
@@ -209,7 +261,7 @@ func DeleteAttachments(attachments []*Attachment, remove bool) (int, error) {
 
 	if remove {
 		for i, a := range attachments {
-			if err := os.Remove(a.LocalPath()); err != nil {
+			if err := a.deleteAttachmentFromBucket(); err != nil {
 				return i, err
 			}
 		}

modules/migrations/gitea.go

@@ -10,7 +10,6 @@ import (
 	"io"
 	"net/http"
 	"os"
-	"path"
 	"path/filepath"
 	"strings"
 	"sync"
@@ -204,22 +203,17 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error {
 			}
 			defer resp.Body.Close()
 
-			localPath := attach.LocalPath()
-			if err = os.MkdirAll(path.Dir(localPath), os.ModePerm); err != nil {
-				return fmt.Errorf("MkdirAll: %v", err)
+			buf := make([]byte, 1024)
+			n, _ := resp.Body.Read(buf)
+			if n > 0 {
+				buf = buf[:n]
 			}
-
-			fw, err := os.Create(localPath)
+			attachment, err := attach.UploadAttachmentToBucket(buf, resp.Body)
 			if err != nil {
-				return fmt.Errorf("Create: %v", err)
-			}
-			defer fw.Close()
-
-			if _, err := io.Copy(fw, resp.Body); err != nil {
 				return err
 			}
 
-			rel.Attachments = append(rel.Attachments, &attach)
+			rel.Attachments = append(rel.Attachments, attachment)
 		}
 
 		rels = append(rels, &rel)

modules/setting/setting.go

@@ -782,9 +782,6 @@ func NewContext() {
 
 	sec = Cfg.Section("attachment")
 	AttachmentPath = sec.Key("PATH").MustString(path.Join(AppDataPath, "attachments"))
-	if !filepath.IsAbs(AttachmentPath) {
-		AttachmentPath = path.Join(AppWorkPath, AttachmentPath)
-	}
 	AttachmentAllowedTypes = strings.Replace(sec.Key("ALLOWED_TYPES").MustString("image/jpeg,image/png,application/zip,application/gzip"), "|", ",", -1)
 	AttachmentMaxSize = sec.Key("MAX_SIZE").MustInt64(4)
 	AttachmentMaxFiles = sec.Key("MAX_FILES").MustInt(5)

routers/routes/routes.go

@@ -8,7 +8,6 @@ import (
 	"bytes"
 	"encoding/gob"
 	"net/http"
-	"os"
 	"path"
 	"text/template"
 	"time"
@@ -489,19 +488,19 @@ func RegisterRoutes(m *macaron.Macaron) {
 				return
 			}
 
-			fr, err := os.Open(attach.LocalPath())
+			reader, err := attach.GetAttachmentReader()
 			if err != nil {
 				ctx.ServerError("Open", err)
 				return
 			}
-			defer fr.Close()
+			defer reader.Close()
 
 			if err := attach.IncreaseDownloadCount(); err != nil {
 				ctx.ServerError("Update", err)
 				return
 			}
 
-			if err = repo.ServeData(ctx, attach.Name, fr); err != nil {
+			if err = repo.ServeData(ctx, attach.Name, reader); err != nil {
 				ctx.ServerError("ServeData", err)
 				return
 			}