Add bucket storage for avatars

Author: masudur-rahman

go.mod

@@ -25,6 +25,7 @@ require (
 	github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d // indirect
 	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
 	github.com/cznic/strutil v0.0.0-20181122101858-275e90344537 // indirect
+	github.com/davecgh/go-spew v1.1.1
 	github.com/denisenkom/go-mssqldb v0.0.0-20190724012636-11b2859924c1
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712 // indirect
@@ -62,7 +63,6 @@ require (
 	github.com/issue9/identicon v0.0.0-20160320065130-d36b54562f4c
 	github.com/jaytaylor/html2text v0.0.0-20160923191438-8fb95d837f7d
 	github.com/jmhodges/levigo v1.0.0 // indirect
-	github.com/joho/godotenv v1.3.0 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20170619183022-cd60e84ee657
 	github.com/keybase/go-crypto v0.0.0-20170605145657-00ac4db533f6
 	github.com/klauspost/compress v0.0.0-20161025140425-8df558b6cb6f
@@ -106,9 +106,10 @@ require (
 	github.com/willf/bitset v0.0.0-20180426185212-8ce1146b8621 // indirect
 	github.com/yohcop/openid-go v0.0.0-20160914080427-2c050d2dae53
 	go.etcd.io/bbolt v1.3.2 // indirect
+	gocloud.dev v0.15.0
 	golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4
 	golang.org/x/net v0.0.0-20190724013045-ca1201d0de80
-	golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421
+	golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a
 	golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3
 	golang.org/x/text v0.3.2
 	golang.org/x/tools v0.0.0-20190731214159-1e85ed8060aa // indirect

go.sum

@@ -24,6 +24,12 @@ import (
 	// for embed
 	_ "github.com/shurcooL/vfsgen"
 
+	// Google, Azure and S3 packages for bucket storage
+	_ "gocloud.dev/blob/azureblob"
+	_ "gocloud.dev/blob/fileblob"
+	_ "gocloud.dev/blob/gcsblob"
+	_ "gocloud.dev/blob/s3blob"
+
 	"github.com/urfave/cli"
 )
 

main.go

@@ -6,13 +6,16 @@
 package models
 
 import (
+	"bytes"
 	"container/list"
+	"context"
 	"crypto/md5"
 	"crypto/sha256"
 	"crypto/subtle"
 	"encoding/hex"
 	"errors"
 	"fmt"
+	"image"
 	_ "image/jpeg" // Needed for jpeg support
 	"image/png"
 	"os"
@@ -34,6 +37,7 @@ import (
 
 	"github.com/Unknwon/com"
 	"github.com/go-xorm/xorm"
+	"gocloud.dev/blob"
 	"golang.org/x/crypto/argon2"
 	"golang.org/x/crypto/bcrypt"
 	"golang.org/x/crypto/pbkdf2"
@@ -345,27 +349,44 @@ func (u *User) generateRandomAvatar(e Engine) error {
 	if u.Avatar == "" {
 		u.Avatar = fmt.Sprintf("%d", u.ID)
 	}
-	if err = os.MkdirAll(filepath.Dir(u.CustomAvatarPath()), os.ModePerm); err != nil {
-		return fmt.Errorf("MkdirAll: %v", err)
-	}
-	fw, err := os.Create(u.CustomAvatarPath())
-	if err != nil {
-		return fmt.Errorf("Create: %v", err)
-	}
-	defer fw.Close()
 
-	if _, err := e.ID(u.ID).Cols("avatar").Update(u); err != nil {
+	if err := u.uploadAvatarToBucket(img); err != nil {
 		return err
 	}
 
-	if err = png.Encode(fw, img); err != nil {
-		return fmt.Errorf("Encode: %v", err)
+	if _, err := e.ID(u.ID).Cols("avatar").Update(u); err != nil {
+		return err
 	}
 
 	log.Info("New random avatar created: %d", u.ID)
 	return nil
 }
 
+func (u *User) getAvatarLink() (string, error) {
+	ctx := context.Background()
+
+	bucket, err := blob.OpenBucket(ctx, setting.FileStorage.Bucket)
+	if err != nil {
+		return "", fmt.Errorf("Failed to setup bucket: %v", err)
+	}
+	bucket = blob.PrefixedBucket(bucket, setting.AvatarUploadPath+"/")
+
+	exist, err := bucket.Exists(ctx, u.Avatar)
+	if exist {
+		opts := &blob.SignedURLOptions{
+			Expiry: blob.DefaultSignedURLExpiry,
+		}
+		signedUrl, err := bucket.SignedURL(ctx, u.Avatar, opts)
+		if err != nil {
+			return "", err
+		}
+
+		return signedUrl, nil
+	}
+
+	return "", fmt.Errorf("file doesn't exist, error: %v", err)
+}
+
 // SizedRelAvatarLink returns a relative link to the user's avatar. When
 // applicable, the link is for an avatar of the indicated size (in pixels).
 func (u *User) SizedRelAvatarLink(size int) string {
@@ -375,10 +396,10 @@ func (u *User) SizedRelAvatarLink(size int) string {
 
 	switch {
 	case u.UseCustomAvatar:
-		if !com.IsFile(u.CustomAvatarPath()) {
-			return base.DefaultAvatarLink()
+		if link, err := u.getAvatarLink(); err == nil {
+			return link
 		}
-		return setting.AppSubURL + "/avatars/" + u.Avatar
+		return base.DefaultAvatarLink()
 	case setting.DisableGravatar, setting.OfflineMode:
 		if !com.IsFile(u.CustomAvatarPath()) {
 			if err := u.GenerateRandomAvatar(); err != nil {
@@ -485,6 +506,35 @@ func (u *User) IsPasswordSet() bool {
 	return len(u.Passwd) > 0
 }
 
+// uploadAvatarToBucket uploads avatar to bucket
+func (u *User) uploadAvatarToBucket(img image.Image) error {
+	ctx := context.Background()
+	bucket, err := blob.OpenBucket(ctx, setting.FileStorage.Bucket)
+	if err != nil {
+		return fmt.Errorf("Failed to setup bucket: %v", err)
+	}
+	bucket = blob.PrefixedBucket(bucket, setting.AvatarUploadPath+"/")
+
+	buf := new(bytes.Buffer)
+	if err = png.Encode(buf, img); err != nil {
+		return fmt.Errorf("Failed to encode: %v", err)
+	}
+	imgData := buf.Bytes()
+
+	bucketWriter, err := bucket.NewWriter(ctx, u.Avatar, nil)
+	if err != nil {
+		return fmt.Errorf("Failed to obtain writer: %v", err)
+	}
+
+	if _, err = bucketWriter.Write(imgData); err != nil {
+		return fmt.Errorf("error occurred: %v", err)
+	}
+	if err = bucketWriter.Close(); err != nil {
+		return fmt.Errorf("Failed to close: %v", err)
+	}
+	return nil
+}
+
 // UploadAvatar saves custom avatar for user.
 // FIXME: split uploads to different subdirs in case we have massive users.
 func (u *User) UploadAvatar(data []byte) error {
@@ -500,34 +550,45 @@ func (u *User) UploadAvatar(data []byte) error {
 	}
 
 	u.UseCustomAvatar = true
-	u.Avatar = fmt.Sprintf("%x", md5.Sum(data))
+	u.Avatar = fmt.Sprintf("%v-%x", u.ID, md5.Sum(data))
 	if err = updateUser(sess, u); err != nil {
 		return fmt.Errorf("updateUser: %v", err)
 	}
 
-	if err := os.MkdirAll(setting.AvatarUploadPath, os.ModePerm); err != nil {
-		return fmt.Errorf("Failed to create dir %s: %v", setting.AvatarUploadPath, err)
+	if err := u.uploadAvatarToBucket(*m); err != nil {
+		return err
 	}
+	return sess.Commit()
+}
 
-	fw, err := os.Create(u.CustomAvatarPath())
+// deleteAvatarFromBucket deletes user avatar from bucket
+func (u *User) deleteAvatarFromBucket() error {
+	ctx := context.Background()
+	bucket, err := blob.OpenBucket(ctx, setting.FileStorage.Bucket)
 	if err != nil {
-		return fmt.Errorf("Create: %v", err)
+		return fmt.Errorf("Failed to setup bucket: %v", err)
 	}
-	defer fw.Close()
+	bucket = blob.PrefixedBucket(bucket, setting.AvatarUploadPath+"/")
 
-	if err = png.Encode(fw, *m); err != nil {
-		return fmt.Errorf("Encode: %v", err)
+	exist, err := bucket.Exists(ctx, u.Avatar)
+	if err != nil {
+		return err
+	} else if !exist {
+		return fmt.Errorf("Avatar %s not found", u.Avatar)
 	}
 
-	return sess.Commit()
+	if err := bucket.Delete(ctx, u.Avatar); err != nil {
+		return fmt.Errorf("Failed to remove %s: %v", u.Avatar, err)
+	}
+	return nil
 }
 
 // DeleteAvatar deletes the user's custom avatar.
 func (u *User) DeleteAvatar() error {
 	log.Trace("DeleteAvatar[%d]: %s", u.ID, u.CustomAvatarPath())
 	if len(u.Avatar) > 0 {
-		if err := os.Remove(u.CustomAvatarPath()); err != nil {
-			return fmt.Errorf("Failed to remove %s: %v", u.CustomAvatarPath(), err)
+		if err := u.deleteAvatarFromBucket(); err != nil {
+			return err
 		}
 	}
 
@@ -1146,11 +1207,8 @@ func deleteUser(e *xorm.Session, u *User) error {
 	}
 
 	if len(u.Avatar) > 0 {
-		avatarPath := u.CustomAvatarPath()
-		if com.IsExist(avatarPath) {
-			if err := os.Remove(avatarPath); err != nil {
-				return fmt.Errorf("Failed to remove %s: %v", avatarPath, err)
-			}
+		if err := u.deleteAvatarFromBucket(); err != nil {
+			return err
 		}
 	}
 

models/user.go

@@ -0,0 +1,20 @@
+package setting
+
+import (
+	"os"
+
+	"github.com/davecgh/go-spew/spew"
+)
+
+// FileStorage represents where to save avatars
+var FileStorage struct {
+	Bucket string
+}
+
+func newFileStorage() {
+	sec := Cfg.Section("storage")
+	cwd, _ := os.Getwd()
+	FileStorage.Bucket = sec.Key("BUCKET").MustString("file://" + cwd) // Preferred: "gs://<bucket-name>?required_key1=required_value1&rq_k2=rq_v2"
+	// Default Credential path for GoogleStorage => $HOME/.config/gcloud/application_default_credentials.json
+	spew.Dump(FileStorage.Bucket)
+}

modules/setting/file_storage.go

@@ -611,7 +611,7 @@ func NewContext() {
 	OfflineMode = sec.Key("OFFLINE_MODE").MustBool()
 	DisableRouterLog = sec.Key("DISABLE_ROUTER_LOG").MustBool()
 	StaticRootPath = sec.Key("STATIC_ROOT_PATH").MustString(AppWorkPath)
-	AppDataPath = sec.Key("APP_DATA_PATH").MustString(path.Join(AppWorkPath, "data"))
+	AppDataPath = sec.Key("APP_DATA_PATH").MustString("data")
 	EnableGzip = sec.Key("ENABLE_GZIP").MustBool()
 	EnablePprof = sec.Key("ENABLE_PPROF").MustBool(false)
 	PprofDataPath = sec.Key("PPROF_DATA_PATH").MustString(path.Join(AppWorkPath, "data/tmp/pprof"))
@@ -849,9 +849,6 @@ func NewContext() {
 	sec = Cfg.Section("picture")
 	AvatarUploadPath = sec.Key("AVATAR_UPLOAD_PATH").MustString(path.Join(AppDataPath, "avatars"))
 	forcePathSeparator(AvatarUploadPath)
-	if !filepath.IsAbs(AvatarUploadPath) {
-		AvatarUploadPath = path.Join(AppWorkPath, AvatarUploadPath)
-	}
 	RepositoryAvatarUploadPath = sec.Key("REPOSITORY_AVATAR_UPLOAD_PATH").MustString(path.Join(AppDataPath, "repo-avatars"))
 	forcePathSeparator(RepositoryAvatarUploadPath)
 	if !filepath.IsAbs(RepositoryAvatarUploadPath) {
@@ -1047,4 +1044,5 @@ func NewServices() {
 	newNotifyMailService()
 	newWebhookService()
 	newIndexerService()
+	newFileStorage()
 }