Skip to content

Commit ef4b47a

Browse files
masudur-rahmanmasudur-rahman
committed
Make local storage working without changing app.ini
1 parent 4d0a95c commit ef4b47a

File tree

12 files changed

+327
-95
lines changed

12 files changed

+327
-95
lines changed

‎models/admin.go

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,13 @@
55
package models
66

77
import (
8+
"context"
89
"fmt"
910
"os"
11+
"path/filepath"
12+
13+
"code.gitea.io/gitea/modules/setting"
14+
"gocloud.dev/blob"
1015

1116
"code.gitea.io/gitea/modules/log"
1217
"code.gitea.io/gitea/modules/timeutil"
@@ -60,6 +65,36 @@ func RemoveAllWithNotice(title, path string) {
6065
removeAllWithNotice(x, title, path)
6166
}
6267

68+
func removeAllFromBucket(bucketPath, objKey string) error {
69+
var bucket *blob.Bucket
70+
var err error
71+
ctx := context.Background()
72+
if filepath.IsAbs(bucketPath) {
73+
if err := os.MkdirAll(bucketPath, 0700); err != nil {
74+
log.Fatal("Failed to create '%s': %v", bucketPath, err)
75+
}
76+
bucket, err = blob.OpenBucket(ctx, "file://"+bucketPath)
77+
if err != nil {
78+
return err
79+
}
80+
} else {
81+
bucket, err = blob.OpenBucket(ctx, setting.FileStorage.BucketURL)
82+
if err != nil {
83+
return err
84+
}
85+
bucket = blob.PrefixedBucket(bucket, setting.AttachmentPath)
86+
}
87+
defer bucket.Close()
88+
89+
exist, err := bucket.Exists(ctx, objKey)
90+
if err != nil {
91+
return err
92+
} else if exist {
93+
return bucket.Delete(ctx, objKey)
94+
}
95+
return nil
96+
}
97+
6398
func removeAllWithNotice(e Engine, title, path string) {
6499
if err := os.RemoveAll(path); err != nil {
65100
desc := fmt.Sprintf("%s [%s]: %v", title, path, err)

‎models/attachment.go

Lines changed: 61 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,9 @@ import (
1010
"io"
1111
"os"
1212
"path"
13+
"path/filepath"
1314

15+
"code.gitea.io/gitea/modules/log"
1416
"code.gitea.io/gitea/modules/setting"
1517
api "code.gitea.io/gitea/modules/structs"
1618
"code.gitea.io/gitea/modules/timeutil"
@@ -68,22 +70,39 @@ func (a *Attachment) LocalPath() string {
6870
return AttachmentLocalPath(a.UUID)
6971
}
7072

73+
// AttachmentBasePath returns the file name of attachment
74+
func (a *Attachment) AttachmentBasePath() string {
75+
return path.Join(a.UUID[0:1], a.UUID[1:2], a.UUID)
76+
}
77+
7178
// DownloadURL returns the download url of the attached file
7279
func (a *Attachment) DownloadURL() string {
7380
return fmt.Sprintf("%sattachments/%s", setting.AppURL, a.UUID)
7481
}
7582

7683
// UploadToBucket uploads attachments to bucket
7784
func (a *Attachment) UploadToBucket(buf []byte, file io.Reader) (*Attachment, error) {
85+
var bucket *blob.Bucket
86+
var err error
7887
ctx := context.Background()
79-
bucket, err := blob.OpenBucket(ctx, setting.FileStorage.BucketURL)
80-
if err != nil {
81-
return nil, fmt.Errorf("could not open bucket: %v", err)
88+
if filepath.IsAbs(setting.AttachmentPath) {
89+
if err := os.MkdirAll(setting.AttachmentPath, 0700); err != nil {
90+
log.Fatal("Failed to create '%s': %v", setting.AttachmentPath, err)
91+
}
92+
bucket, err = blob.OpenBucket(ctx, "file://"+setting.AttachmentPath)
93+
if err != nil {
94+
return nil, fmt.Errorf("could not open bucket: %v", err)
95+
}
96+
} else {
97+
bucket, err = blob.OpenBucket(ctx, setting.FileStorage.BucketURL)
98+
if err != nil {
99+
return nil, fmt.Errorf("could not open bucket: %v", err)
100+
}
101+
bucket = blob.PrefixedBucket(bucket, setting.AttachmentPath)
82102
}
83-
bucket = blob.PrefixedBucket(bucket, setting.AttachmentPath)
84103
defer bucket.Close()
85104

86-
bw, err := bucket.NewWriter(ctx, a.LocalPath(), nil)
105+
bw, err := bucket.NewWriter(ctx, a.AttachmentBasePath(), nil)
87106
if err != nil {
88107
return nil, fmt.Errorf("failed to obtain writer: %v", err)
89108
}
@@ -97,7 +116,7 @@ func (a *Attachment) UploadToBucket(buf []byte, file io.Reader) (*Attachment, er
97116
return nil, fmt.Errorf("failed to close: %v", err)
98117
}
99118

100-
attrs, err := bucket.Attributes(ctx, a.LocalPath())
119+
attrs, err := bucket.Attributes(ctx, a.AttachmentBasePath())
101120
if err != nil {
102121
return nil, fmt.Errorf("failed to read attributes: %v", err)
103122
}
@@ -203,42 +222,66 @@ func getAttachmentByReleaseIDFileName(e Engine, releaseID int64, fileName string
203222

204223
// Open provides attachment reader from bucket
205224
func (a *Attachment) Open() (io.ReadCloser, error) {
225+
var bucket *blob.Bucket
226+
var err error
206227
ctx := context.Background()
207-
bucket, err := blob.OpenBucket(ctx, setting.FileStorage.BucketURL)
208-
if err != nil {
209-
return nil, fmt.Errorf("could not open bucket %v", err)
228+
if filepath.IsAbs(setting.AttachmentPath) {
229+
if err := os.MkdirAll(setting.AttachmentPath, 0700); err != nil {
230+
log.Fatal("Failed to create '%s': %v", setting.AttachmentPath, err)
231+
}
232+
bucket, err = blob.OpenBucket(ctx, "file://"+setting.AttachmentPath)
233+
if err != nil {
234+
return nil, fmt.Errorf("could not open bucket: %v", err)
235+
}
236+
} else {
237+
bucket, err = blob.OpenBucket(ctx, setting.FileStorage.BucketURL)
238+
if err != nil {
239+
return nil, fmt.Errorf("could not open bucket: %v", err)
240+
}
241+
bucket = blob.PrefixedBucket(bucket, setting.AttachmentPath)
210242
}
211-
bucket = blob.PrefixedBucket(bucket, setting.AttachmentPath)
212243
defer bucket.Close()
213244

214-
exist, err := bucket.Exists(ctx, a.LocalPath())
245+
exist, err := bucket.Exists(ctx, a.AttachmentBasePath())
215246
if err != nil {
216247
return nil, err
217248
} else if !exist {
218249
return nil, os.ErrNotExist
219250
}
220251

221-
return bucket.NewReader(ctx, a.LocalPath(), nil)
252+
return bucket.NewReader(ctx, a.AttachmentBasePath(), nil)
222253
}
223254

224255
// deleteFromBucket deletes attachments from bucket
225256
func (a *Attachment) deleteFromBucket() error {
257+
var bucket *blob.Bucket
258+
var err error
226259
ctx := context.Background()
227-
bucket, err := blob.OpenBucket(ctx, setting.FileStorage.BucketURL)
228-
if err != nil {
229-
return fmt.Errorf("could not open bucket: %v", err)
260+
if filepath.IsAbs(setting.AttachmentPath) {
261+
if err := os.MkdirAll(setting.AttachmentPath, 0700); err != nil {
262+
log.Fatal("Failed to create '%s': %v", setting.AttachmentPath, err)
263+
}
264+
bucket, err = blob.OpenBucket(ctx, "file://"+setting.AttachmentPath)
265+
if err != nil {
266+
return fmt.Errorf("could not open bucket: %v", err)
267+
}
268+
} else {
269+
bucket, err = blob.OpenBucket(ctx, setting.FileStorage.BucketURL)
270+
if err != nil {
271+
return fmt.Errorf("could not open bucket: %v", err)
272+
}
273+
bucket = blob.PrefixedBucket(bucket, setting.AttachmentPath)
230274
}
231-
bucket = blob.PrefixedBucket(bucket, setting.AttachmentPath)
232275
defer bucket.Close()
233276

234-
exist, err := bucket.Exists(ctx, a.LocalPath())
277+
exist, err := bucket.Exists(ctx, a.AttachmentBasePath())
235278
if err != nil {
236279
return err
237280
} else if !exist {
238281
return os.ErrNotExist
239282
}
240283

241-
return bucket.Delete(ctx, a.LocalPath())
284+
return bucket.Delete(ctx, a.AttachmentBasePath())
242285
}
243286

244287
// DeleteAttachment deletes the given attachment and optionally the associated file.

‎models/migrations/migrations.go

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,12 @@ import (
1717
"strings"
1818
"time"
1919

20+
"code.gitea.io/gitea/models"
21+
2022
"github.com/Unknwon/com"
2123
"github.com/go-xorm/xorm"
2224
gouuid "github.com/satori/go.uuid"
23-
ini "gopkg.in/ini.v1"
25+
"gopkg.in/ini.v1"
2426

2527
"code.gitea.io/gitea/modules/generate"
2628
"code.gitea.io/gitea/modules/log"
@@ -587,7 +589,7 @@ func attachmentRefactor(x *xorm.Engine) error {
587589

588590
attachments := make([]*Attachment, 0, len(results))
589591
for _, attach := range results {
590-
if !com.IsExist(string(attach["path"])) {
592+
if models.IsAvatarValid(setting.AttachmentPath, string(attach["path"])) {
591593
// If the attachment is already missing, there is no point to update it.
592594
continue
593595
}
@@ -638,14 +640,16 @@ func attachmentRefactor(x *xorm.Engine) error {
638640
log.Info("Failed to rename some attachments, old and new paths are saved into: %s", dumpPath)
639641
}()
640642
for _, attach := range attachments {
641-
if err = os.MkdirAll(path.Dir(attach.NewPath), os.ModePerm); err != nil {
642-
isSucceed = false
643-
return err
644-
}
643+
if filepath.IsAbs(setting.AttachmentPath) {
644+
if err = os.MkdirAll(path.Dir(attach.NewPath), os.ModePerm); err != nil {
645+
isSucceed = false
646+
return err
647+
}
645648

646-
if err = os.Rename(attach.Path, attach.NewPath); err != nil {
647-
isSucceed = false
648-
return err
649+
if err = os.Rename(attach.Path, attach.NewPath); err != nil {
650+
isSucceed = false
651+
return err
652+
}
649653
}
650654
}
651655

‎models/migrations/v61.go

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,17 @@
55
package migrations
66

77
import (
8+
"context"
89
"fmt"
910
"os"
1011
"path"
12+
"path/filepath"
1113

1214
"code.gitea.io/gitea/modules/log"
1315
"code.gitea.io/gitea/modules/setting"
1416

1517
"github.com/go-xorm/xorm"
18+
"gocloud.dev/blob"
1619
)
1720

1821
func addSizeToAttachment(x *xorm.Engine) error {
@@ -29,14 +32,35 @@ func addSizeToAttachment(x *xorm.Engine) error {
2932
if err := x.Find(&attachments); err != nil {
3033
return fmt.Errorf("query attachments: %v", err)
3134
}
35+
36+
var bucket *blob.Bucket
37+
var err error
38+
ctx := context.Background()
39+
if filepath.IsAbs(setting.AttachmentPath) {
40+
if err := os.MkdirAll(setting.AttachmentPath, 0700); err != nil {
41+
log.Fatal("Failed to create '%s': %v", setting.AttachmentPath, err)
42+
}
43+
bucket, err = blob.OpenBucket(ctx, "file://"+setting.AttachmentPath)
44+
if err != nil {
45+
return fmt.Errorf("could not open bucket: %v", err)
46+
}
47+
} else {
48+
bucket, err = blob.OpenBucket(ctx, setting.FileStorage.BucketURL)
49+
if err != nil {
50+
return fmt.Errorf("could not open bucket: %v", err)
51+
}
52+
bucket = blob.PrefixedBucket(bucket, setting.AttachmentPath)
53+
}
54+
defer bucket.Close()
55+
3256
for _, attach := range attachments {
33-
localPath := path.Join(setting.AttachmentPath, attach.UUID[0:1], attach.UUID[1:2], attach.UUID)
34-
fi, err := os.Stat(localPath)
57+
basePath := path.Join(attach.UUID[0:1], attach.UUID[1:2], attach.UUID)
58+
attrs, err := bucket.Attributes(ctx, basePath)
3559
if err != nil {
3660
log.Error("calculate file size of attachment[UUID: %s]: %v", attach.UUID, err)
3761
continue
3862
}
39-
attach.Size = fi.Size()
63+
attach.Size = attrs.Size
4064
if _, err := x.ID(attach.ID).Cols("size").Update(attach); err != nil {
4165
return fmt.Errorf("update size column: %v", err)
4266
}

‎models/org.go

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ import (
1414
"code.gitea.io/gitea/modules/log"
1515
"code.gitea.io/gitea/modules/structs"
1616

17-
"github.com/Unknwon/com"
1817
"github.com/go-xorm/xorm"
1918
"xorm.io/builder"
2019
)
@@ -275,10 +274,8 @@ func deleteOrg(e *xorm.Session, u *User) error {
275274

276275
if len(u.Avatar) > 0 {
277276
avatarPath := u.CustomAvatarPath()
278-
if com.IsExist(avatarPath) {
279-
if err := os.Remove(avatarPath); err != nil {
280-
return fmt.Errorf("Failed to remove %s: %v", avatarPath, err)
281-
}
277+
if err := u.deleteAvatarFromBucket(); err != nil {
278+
return fmt.Errorf("Failed to remove %s: %v", avatarPath, err)
282279
}
283280
}
284281

‎models/repo.go

Lines changed: 34 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1842,7 +1842,7 @@ func DeleteRepository(doer *User, uid, repoID int64) error {
18421842
return err
18431843
}
18441844
for j := range attachments {
1845-
attachmentPaths = append(attachmentPaths, attachments[j].LocalPath())
1845+
attachmentPaths = append(attachmentPaths, attachments[j].AttachmentBasePath())
18461846
}
18471847

18481848
if _, err = sess.In("issue_id", deleteCond).
@@ -1879,7 +1879,14 @@ func DeleteRepository(doer *User, uid, repoID int64) error {
18791879

18801880
// Remove attachment files.
18811881
for i := range attachmentPaths {
1882-
removeAllWithNotice(sess, "Delete attachment", attachmentPaths[i])
1882+
if err := removeAllFromBucket(setting.AttachmentPath, attachmentPaths[i]); err != nil {
1883+
title, attachPath := "Delete attachment", setting.AttachmentPath+attachmentPaths[i]
1884+
desc := fmt.Sprintf("%s [%s]: %v", title, attachPath, err)
1885+
log.Warn(title+" [%s]: %v", attachPath, err)
1886+
if err = createNotice(sess, NoticeRepository, desc); err != nil {
1887+
log.Error("CreateRepositoryNotice: %v", err)
1888+
}
1889+
}
18831890
}
18841891

18851892
// Remove LFS objects
@@ -1900,6 +1907,14 @@ func DeleteRepository(doer *User, uid, repoID int64) error {
19001907

19011908
oidPath := filepath.Join(v.Oid[0:2], v.Oid[2:4], v.Oid[4:len(v.Oid)])
19021909
removeAllWithNotice(sess, "Delete orphaned LFS file", oidPath)
1910+
if err := removeAllFromBucket(setting.LFS.ContentPath, oidPath); err != nil {
1911+
title := "Delete attachment"
1912+
desc := fmt.Sprintf("%s [%s]: %v", title, oidPath, err)
1913+
log.Warn(title+" [%s]: %v", oidPath, err)
1914+
if err = createNotice(sess, NoticeRepository, desc); err != nil {
1915+
log.Error("CreateRepositoryNotice: %v", err)
1916+
}
1917+
}
19031918
}
19041919

19051920
if _, err := sess.Delete(&LFSMetaObject{RepositoryID: repoID}); err != nil {
@@ -2578,7 +2593,7 @@ func (repo *Repository) RelAvatarLink() string {
25782593
func (repo *Repository) relAvatarLink(e Engine) string {
25792594
// If no avatar - path is empty
25802595
avatarPath := repo.CustomAvatarPath()
2581-
if len(avatarPath) == 0 || !isAvatarValid(repo.CustomAvatarPath()) {
2596+
if len(avatarPath) == 0 || !IsAvatarValid(setting.RepositoryAvatarUploadPath, repo.Avatar) {
25822597
switch mode := setting.RepositoryAvatarFallback; mode {
25832598
case "image":
25842599
return setting.RepositoryAvatarFallbackImage
@@ -2648,12 +2663,24 @@ func (repo *Repository) UploadAvatar(data []byte) error {
26482663

26492664
// deleteAvatarFromBucket deletes repo avatar from bucket
26502665
func (repo *Repository) deleteAvatarFromBucket() error {
2666+
var bucket *blob.Bucket
2667+
var err error
26512668
ctx := context.Background()
2652-
bucket, err := blob.OpenBucket(ctx, setting.FileStorage.BucketURL)
2653-
if err != nil {
2654-
return fmt.Errorf("could not open bucket: %v", err)
2669+
if filepath.IsAbs(setting.RepositoryAvatarUploadPath) {
2670+
if err := os.MkdirAll(setting.RepositoryAvatarUploadPath, 0700); err != nil {
2671+
log.Fatal("Failed to create '%s': %v", setting.RepositoryAvatarUploadPath, err)
2672+
}
2673+
bucket, err = blob.OpenBucket(ctx, "file://"+setting.RepositoryAvatarUploadPath)
2674+
if err != nil {
2675+
return fmt.Errorf("could not open bucket: %v", err)
2676+
}
2677+
} else {
2678+
bucket, err = blob.OpenBucket(ctx, setting.FileStorage.BucketURL)
2679+
if err != nil {
2680+
return fmt.Errorf("could not open bucket: %v", err)
2681+
}
2682+
bucket = blob.PrefixedBucket(bucket, setting.RepositoryAvatarUploadPath)
26552683
}
2656-
bucket = blob.PrefixedBucket(bucket, setting.RepositoryAvatarUploadPath)
26572684
defer bucket.Close()
26582685

26592686
exist, err := bucket.Exists(ctx, repo.Avatar)

0 commit comments

Comments
 (0)