Add Bucket Storage Support for files

Signed-off-by: Masudur Rahman <[email protected]>

Author: masudur-rahman

custom/conf/app.ini.sample

@@ -869,3 +869,9 @@ QUEUE_LENGTH = 1000
 ; Task queue connction string, available only when `QUEUE_TYPE` is `redis`. 
 ; If there is a password of redis, use `addrs=127.0.0.1:6379 password=123 db=0`.
 QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0"
+
+[storage]
+; URL of bucket where files are stored (such as attachments, avatars, etc.).
+; If unset, `BUCKET_URL` defaults to `file:// <AppWorkPath>`.
+; Consult https://gocloud.dev/howto/blob for URL format for various cloud providers and their credential paths and env variables.
+BUCKET_URL =

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -560,6 +560,10 @@ Two special environment variables are passed to the render command:
 - `QUEUE_LENGTH`: **1000**: Task queue length, available only when `QUEUE_TYPE` is `channel`.
 - `QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: Task queue connection string, available only when `QUEUE_TYPE` is `redis`. If there redis needs a password, use `addrs=127.0.0.1:6379 password=123 db=0`.
 
+## Storage (`storage`)
+
+- `BUCKET_URL`: URL of bucket where files are stored (such as attachments, avatars, etc.). If unset, `BUCKET_URL` defaults to `file:// <AppWorkPath>`. Consult https://gocloud.dev/howto/blob for URL format for various cloud providers and their credential paths and env variables.
+
 ## Other (`other`)
 
 - `SHOW_FOOTER_BRANDING`: **false**: Show Gitea branding in the footer.

docs/content/doc/advanced/config-cheat-sheet.zh-cn.md

@@ -256,6 +256,10 @@ IS_INPUT_FILE = false
 - `QUEUE_LENGTH`: **1000**: 任务队列长度，当 `QUEUE_TYPE` 为 `channel` 时有效。
 - `QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: 任务队列连接字符串，当 `QUEUE_TYPE` 为 `redis` 时有效。如果redis有密码，则可以 `addrs=127.0.0.1:6379 password=123 db=0`。
 
+## Storage (`storage`)
+
+- `BUCKET_URL`: 存储文件的存储桶的URL（例如附件，头像等）. 如果未设置，`BUCKET_URL`默认为`file：// <AppWorkPath>`. 有关各种云提供商及其凭证路径和env变量的URL格式，请参阅https://gocloud.dev/howto/blob .
+
 ## Other (`other`)
 
 - `SHOW_FOOTER_BRANDING`: 为真则在页面底部显示Gitea的字样。

go.mod

@@ -53,7 +53,6 @@ require (
 	github.com/issue9/identicon v0.0.0-20160320065130-d36b54562f4c
 	github.com/jaytaylor/html2text v0.0.0-20160923191438-8fb95d837f7d
 	github.com/jmhodges/levigo v1.0.0 // indirect
-	github.com/joho/godotenv v1.3.0 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20170619183022-cd60e84ee657
 	github.com/keybase/go-crypto v0.0.0-20170605145657-00ac4db533f6
 	github.com/klauspost/compress v0.0.0-20161025140425-8df558b6cb6f
@@ -65,7 +64,7 @@ require (
 	github.com/lunny/levelqueue v0.0.0-20190217115915-02b525a4418e
 	github.com/mailru/easyjson v0.7.0 // indirect
 	github.com/markbates/goth v1.56.0
-	github.com/mattn/go-isatty v0.0.7
+	github.com/mattn/go-isatty v0.0.8
 	github.com/mattn/go-oci8 v0.0.0-20190320171441-14ba190cf52d // indirect
 	github.com/mattn/go-sqlite3 v1.11.0
 	github.com/mcuadros/go-version v0.0.0-20190308113854-92cdf37c5b75
@@ -99,6 +98,7 @@ require (
 	github.com/urfave/cli v1.20.0
 	github.com/willf/bitset v0.0.0-20180426185212-8ce1146b8621 // indirect
 	github.com/yohcop/openid-go v0.0.0-20160914080427-2c050d2dae53
+	gocloud.dev v0.16.0
 	golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad
 	golang.org/x/net v0.0.0-20190909003024-a7b16738d86b
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45

go.sum

@@ -5,12 +5,13 @@
 package models
 
 import (
+	"context"
 	"fmt"
 	"io"
-	"os"
 	"path"
 
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/storage"
 	api "code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/timeutil"
 
@@ -66,6 +67,11 @@ func (a *Attachment) LocalPath() string {
 	return AttachmentLocalPath(a.UUID)
 }
 
+// AttachmentBasePath returns the file name of attachment
+func (a *Attachment) AttachmentBasePath() string {
+	return path.Join(a.UUID[0:1], a.UUID[1:2], a.UUID)
+}
+
 // DownloadURL returns the download url of the attached file
 func (a *Attachment) DownloadURL() string {
 	return fmt.Sprintf("%sattachments/%s", setting.AppURL, a.UUID)
@@ -75,29 +81,32 @@ func (a *Attachment) DownloadURL() string {
 func NewAttachment(attach *Attachment, buf []byte, file io.Reader) (_ *Attachment, err error) {
 	attach.UUID = gouuid.NewV4().String()
 
-	localPath := attach.LocalPath()
-	if err = os.MkdirAll(path.Dir(localPath), os.ModePerm); err != nil {
-		return nil, fmt.Errorf("MkdirAll: %v", err)
+	fs := storage.FileStorage{
+		Ctx:      context.Background(),
+		Path:     setting.AttachmentPath,
+		FileName: attach.AttachmentBasePath(),
 	}
 
-	fw, err := os.Create(localPath)
+	fw, err := fs.NewWriter()
 	if err != nil {
 		return nil, fmt.Errorf("Create: %v", err)
 	}
-	defer fw.Close()
 
 	if _, err = fw.Write(buf); err != nil {
+		fw.Close()
 		return nil, fmt.Errorf("Write: %v", err)
 	} else if _, err = io.Copy(fw, file); err != nil {
+		fw.Close()
 		return nil, fmt.Errorf("Copy: %v", err)
 	}
+	fw.Close()
 
 	// Update file size
-	var fi os.FileInfo
-	if fi, err = fw.Stat(); err != nil {
+	fi, err := fs.Attributes()
+	if err != nil {
 		return nil, fmt.Errorf("file size: %v", err)
 	}
-	attach.Size = fi.Size()
+	attach.Size = fi.Size
 
 	if _, err := x.Insert(attach); err != nil {
 		return nil, err
@@ -209,7 +218,12 @@ func DeleteAttachments(attachments []*Attachment, remove bool) (int, error) {
 
 	if remove {
 		for i, a := range attachments {
-			if err := os.Remove(a.LocalPath()); err != nil {
+			fs := storage.FileStorage{
+				Ctx:      context.Background(),
+				Path:     setting.AttachmentPath,
+				FileName: a.AttachmentBasePath(),
+			}
+			if err := fs.Delete(); err != nil {
 				return i, err
 			}
 		}

models/attachment.go

@@ -5,12 +5,13 @@
 package migrations
 
 import (
+	"context"
 	"fmt"
-	"os"
 	"path"
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/storage"
 
 	"xorm.io/xorm"
 )
@@ -29,14 +30,20 @@ func addSizeToAttachment(x *xorm.Engine) error {
 	if err := x.Find(&attachments); err != nil {
 		return fmt.Errorf("query attachments: %v", err)
 	}
+
+	fs := storage.FileStorage{
+		Ctx:  context.Background(),
+		Path: setting.AttachmentPath,
+	}
+
 	for _, attach := range attachments {
-		localPath := path.Join(setting.AttachmentPath, attach.UUID[0:1], attach.UUID[1:2], attach.UUID)
-		fi, err := os.Stat(localPath)
+		fs.FileName = path.Join(attach.UUID[0:1], attach.UUID[1:2], attach.UUID)
+		attrs, err := fs.Attributes()
 		if err != nil {
 			log.Error("calculate file size of attachment[UUID: %s]: %v", attach.UUID, err)
 			continue
 		}
-		attach.Size = fi.Size()
+		attach.Size = attrs.Size
 		if _, err := x.ID(attach.ID).Cols("size").Update(attach); err != nil {
 			return fmt.Errorf("update size column: %v", err)
 		}

models/migrations/v61.go

@@ -6,15 +6,16 @@
 package models
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"strings"
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/storage"
 	"code.gitea.io/gitea/modules/structs"
 
-	"github.com/unknwon/com"
 	"xorm.io/builder"
 	"xorm.io/xorm"
 )
@@ -270,10 +271,13 @@ func deleteOrg(e *xorm.Session, u *User) error {
 
 	if len(u.Avatar) > 0 {
 		avatarPath := u.CustomAvatarPath()
-		if com.IsExist(avatarPath) {
-			if err := os.Remove(avatarPath); err != nil {
-				return fmt.Errorf("Failed to remove %s: %v", avatarPath, err)
-			}
+		fs := storage.FileStorage{
+			Ctx:      context.Background(),
+			Path:     setting.AvatarUploadPath,
+			FileName: u.Avatar,
+		}
+		if err := fs.Delete(); err != nil {
+			return fmt.Errorf("Failed to remove %s: %v", avatarPath, err)
 		}
 	}