Skip to content

Commit 3878e98

Browse files
lunnyzeripath
lunny
and
zeripath
authored
Add default storage configurations (#12813)
Signed-off-by: Andrew Thornton <[email protected]> Co-authored-by: zeripath <[email protected]>
1 parent 4c6ac08 commit 3878e98

File tree

19 files changed

+457
-183
lines changed

19 files changed

+457
-183
lines changed

‎.golangci.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,3 +98,6 @@ issues:
9898
- path: models/update.go
9999
linters:
100100
- unused
101+
- path: cmd/dump.go
102+
linters:
103+
- dupl

‎cmd/dump.go

Lines changed: 40 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"code.gitea.io/gitea/models"
1919
"code.gitea.io/gitea/modules/log"
2020
"code.gitea.io/gitea/modules/setting"
21+
"code.gitea.io/gitea/modules/storage"
2122
"code.gitea.io/gitea/modules/util"
2223

2324
"gitea.com/macaron/session"
@@ -57,6 +58,8 @@ func addRecursive(w archiver.Writer, dirPath string, absPath string, verbose boo
5758
if err != nil {
5859
return fmt.Errorf("Could not open directory %s: %s", absPath, err)
5960
}
61+
defer dir.Close()
62+
6063
files, err := dir.Readdir(0)
6164
if err != nil {
6265
return fmt.Errorf("Unable to list files in %s: %s", absPath, err)
@@ -197,6 +200,10 @@ func runDump(ctx *cli.Context) error {
197200
return err
198201
}
199202

203+
if err := storage.Init(); err != nil {
204+
return err
205+
}
206+
200207
if file == nil {
201208
file, err = os.Create(fileName)
202209
if err != nil {
@@ -231,11 +238,21 @@ func runDump(ctx *cli.Context) error {
231238
fatal("Failed to include repositories: %v", err)
232239
}
233240

234-
if _, err := os.Stat(setting.LFS.ContentPath); !os.IsNotExist(err) {
235-
log.Info("Dumping lfs... %s", setting.LFS.ContentPath)
236-
if err := addRecursive(w, "lfs", setting.LFS.ContentPath, verbose); err != nil {
237-
fatal("Failed to include lfs: %v", err)
241+
if err := storage.LFS.IterateObjects(func(objPath string, object storage.Object) error {
242+
info, err := object.Stat()
243+
if err != nil {
244+
return err
238245
}
246+
247+
return w.Write(archiver.File{
248+
FileInfo: archiver.FileInfo{
249+
FileInfo: info,
250+
CustomName: path.Join("data", "lfs", objPath),
251+
},
252+
ReadCloser: object,
253+
})
254+
}); err != nil {
255+
fatal("Failed to dump LFS objects: %v", err)
239256
}
240257
}
241258

@@ -302,13 +319,31 @@ func runDump(ctx *cli.Context) error {
302319
}
303320

304321
excludes = append(excludes, setting.RepoRootPath)
305-
excludes = append(excludes, setting.LFS.ContentPath)
322+
excludes = append(excludes, setting.LFS.Path)
323+
excludes = append(excludes, setting.Attachment.Path)
306324
excludes = append(excludes, setting.LogRootPath)
307325
if err := addRecursiveExclude(w, "data", setting.AppDataPath, excludes, verbose); err != nil {
308326
fatal("Failed to include data directory: %v", err)
309327
}
310328
}
311329

330+
if err := storage.Attachments.IterateObjects(func(objPath string, object storage.Object) error {
331+
info, err := object.Stat()
332+
if err != nil {
333+
return err
334+
}
335+
336+
return w.Write(archiver.File{
337+
FileInfo: archiver.FileInfo{
338+
FileInfo: info,
339+
CustomName: path.Join("data", "attachments", objPath),
340+
},
341+
ReadCloser: object,
342+
})
343+
}); err != nil {
344+
fatal("Failed to dump attachments: %v", err)
345+
}
346+
312347
// Doesn't check if LogRootPath exists before processing --skip-log intentionally,
313348
// ensuring that it's clear the dump is skipped whether the directory's initialized
314349
// yet or not.

‎cmd/migrate_storage.go

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ package cmd
77
import (
88
"context"
99
"fmt"
10+
"strings"
1011

1112
"code.gitea.io/gitea/models"
1213
"code.gitea.io/gitea/models/migrations"
@@ -30,8 +31,8 @@ var CmdMigrateStorage = cli.Command{
3031
Usage: "Kinds of files to migrate, currently only 'attachments' is supported",
3132
},
3233
cli.StringFlag{
33-
Name: "store, s",
34-
Value: "local",
34+
Name: "storage, s",
35+
Value: setting.LocalStorageType,
3536
Usage: "New storage type, local or minio",
3637
},
3738
cli.StringFlag{
@@ -112,15 +113,15 @@ func runMigrateStorage(ctx *cli.Context) error {
112113

113114
var dstStorage storage.ObjectStorage
114115
var err error
115-
switch ctx.String("store") {
116-
case "local":
116+
switch strings.ToLower(ctx.String("storage")) {
117+
case setting.LocalStorageType:
117118
p := ctx.String("path")
118119
if p == "" {
119-
log.Fatal("Path must be given when store is loal")
120+
log.Fatal("Path must be given when storage is loal")
120121
return nil
121122
}
122123
dstStorage, err = storage.NewLocalStorage(p)
123-
case "minio":
124+
case setting.MinioStorageType:
124125
dstStorage, err = storage.NewMinioStorage(
125126
context.Background(),
126127
ctx.String("minio-endpoint"),
@@ -132,14 +133,14 @@ func runMigrateStorage(ctx *cli.Context) error {
132133
ctx.Bool("minio-use-ssl"),
133134
)
134135
default:
135-
return fmt.Errorf("Unsupported attachments store type: %s", ctx.String("store"))
136+
return fmt.Errorf("Unsupported attachments storage type: %s", ctx.String("storage"))
136137
}
137138

138139
if err != nil {
139140
return err
140141
}
141142

142-
tp := ctx.String("type")
143+
tp := strings.ToLower(ctx.String("type"))
143144
switch tp {
144145
case "attachments":
145146
if err := migrateAttachments(dstStorage); err != nil {

‎custom/conf/app.example.ini

Lines changed: 34 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -778,25 +778,25 @@ MAX_SIZE = 4
778778
MAX_FILES = 5
779779
; Storage type for attachments, `local` for local disk or `minio` for s3 compatible
780780
; object storage service, default is `local`.
781-
STORE_TYPE = local
781+
STORAGE_TYPE = local
782782
; Allows the storage driver to redirect to authenticated URLs to serve files directly
783783
; Currently, only `minio` is supported.
784784
SERVE_DIRECT = false
785-
; Path for attachments. Defaults to `data/attachments` only available when STORE_TYPE is `local`
785+
; Path for attachments. Defaults to `data/attachments` only available when STORAGE_TYPE is `local`
786786
PATH = data/attachments
787-
; Minio endpoint to connect only available when STORE_TYPE is `minio`
787+
; Minio endpoint to connect only available when STORAGE_TYPE is `minio`
788788
MINIO_ENDPOINT = localhost:9000
789-
; Minio accessKeyID to connect only available when STORE_TYPE is `minio`
789+
; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`
790790
MINIO_ACCESS_KEY_ID =
791-
; Minio secretAccessKey to connect only available when STORE_TYPE is `minio`
791+
; Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio`
792792
MINIO_SECRET_ACCESS_KEY =
793-
; Minio bucket to store the attachments only available when STORE_TYPE is `minio`
793+
; Minio bucket to store the attachments only available when STORAGE_TYPE is `minio`
794794
MINIO_BUCKET = gitea
795-
; Minio location to create bucket only available when STORE_TYPE is `minio`
795+
; Minio location to create bucket only available when STORAGE_TYPE is `minio`
796796
MINIO_LOCATION = us-east-1
797-
; Minio base path on the bucket only available when STORE_TYPE is `minio`
797+
; Minio base path on the bucket only available when STORAGE_TYPE is `minio`
798798
MINIO_BASE_PATH = attachments/
799-
; Minio enabled ssl only available when STORE_TYPE is `minio`
799+
; Minio enabled ssl only available when STORAGE_TYPE is `minio`
800800
MINIO_USE_SSL = false
801801

802802
[time]
@@ -1161,3 +1161,28 @@ QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0"
11611161
MAX_ATTEMPTS = 3
11621162
; Backoff time per http/https request retry (seconds)
11631163
RETRY_BACKOFF = 3
1164+
1165+
; default storage for attachments, lfs and avatars
1166+
[storage]
1167+
; storage type
1168+
STORAGE_TYPE = local
1169+
1170+
; lfs storage will override storage
1171+
[lfs]
1172+
STORAGE_TYPE = local
1173+
1174+
; customize storage
1175+
;[storage.my_minio]
1176+
;STORAGE_TYPE = minio
1177+
; Minio endpoint to connect only available when STORAGE_TYPE is `minio`
1178+
;MINIO_ENDPOINT = localhost:9000
1179+
; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`
1180+
;MINIO_ACCESS_KEY_ID =
1181+
; Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio`
1182+
;MINIO_SECRET_ACCESS_KEY =
1183+
; Minio bucket to store the attachments only available when STORAGE_TYPE is `minio`
1184+
;MINIO_BUCKET = gitea
1185+
; Minio location to create bucket only available when STORAGE_TYPE is `minio`
1186+
;MINIO_LOCATION = us-east-1
1187+
; Minio enabled ssl only available when STORAGE_TYPE is `minio`
1188+
;MINIO_USE_SSL = false

‎docs/content/doc/advanced/config-cheat-sheet.en-us.md

Lines changed: 59 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -220,16 +220,6 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
220220
- `LANDING_PAGE`: **home**: Landing page for unauthenticated users \[home, explore, organizations, login\].
221221

222222
- `LFS_START_SERVER`: **false**: Enables git-lfs support.
223-
- `LFS_STORE_TYPE`: **local**: Storage type for lfs, `local` for local disk or `minio` for s3 compatible object storage service.
224-
- `LFS_SERVE_DIRECT`: **false**: Allows the storage driver to redirect to authenticated URLs to serve files directly. Currently, only Minio/S3 is supported via signed URLs, local does nothing.
225-
- `LFS_CONTENT_PATH`: **./data/lfs**: Where to store LFS files, only available when `LFS_STORE_TYPE` is `local`.
226-
- `LFS_MINIO_ENDPOINT`: **localhost:9000**: Minio endpoint to connect only available when `LFS_STORE_TYPE` is `minio`
227-
- `LFS_MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `LFS_STORE_TYPE` is `minio`
228-
- `LFS_MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when `LFS_STORE_TYPE is` `minio`
229-
- `LFS_MINIO_BUCKET`: **gitea**: Minio bucket to store the lfs only available when `LFS_STORE_TYPE` is `minio`
230-
- `LFS_MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when `LFS_STORE_TYPE` is `minio`
231-
- `LFS_MINIO_BASE_PATH`: **lfs/**: Minio base path on the bucket only available when `LFS_STORE_TYPE` is `minio`
232-
- `LFS_MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `LFS_STORE_TYPE` is `minio`
233223
- `LFS_JWT_SECRET`: **\<empty\>**: LFS authentication secret, change this a unique string.
234224
- `LFS_HTTP_AUTH_EXPIRY`: **20m**: LFS authentication validity period in time.Duration, pushes taking longer than this may fail.
235225
- `LFS_MAX_FILE_SIZE`: **0**: Maximum allowed LFS file size in bytes (Set to 0 for no limit).
@@ -501,16 +491,16 @@ relation to port exhaustion.
501491
Use `*/*` for all types.
502492
- `MAX_SIZE`: **4**: Maximum size (MB).
503493
- `MAX_FILES`: **5**: Maximum number of attachments that can be uploaded at once.
504-
- `STORE_TYPE`: **local**: Storage type for attachments, `local` for local disk or `minio` for s3 compatible object storage service, default is `local`.
494+
- `STORAGE_TYPE`: **local**: Storage type for attachments, `local` for local disk or `minio` for s3 compatible object storage service, default is `local` or other name defined with `[storage.xxx]`
505495
- `SERVE_DIRECT`: **false**: Allows the storage driver to redirect to authenticated URLs to serve files directly. Currently, only Minio/S3 is supported via signed URLs, local does nothing.
506-
- `PATH`: **data/attachments**: Path to store attachments only available when STORE_TYPE is `local`
507-
- `MINIO_ENDPOINT`: **localhost:9000**: Minio endpoint to connect only available when STORE_TYPE is `minio`
508-
- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when STORE_TYPE is `minio`
509-
- `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when STORE_TYPE is `minio`
510-
- `MINIO_BUCKET`: **gitea**: Minio bucket to store the attachments only available when STORE_TYPE is `minio`
511-
- `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when STORE_TYPE is `minio`
512-
- `MINIO_BASE_PATH`: **attachments/**: Minio base path on the bucket only available when STORE_TYPE is `minio`
513-
- `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when STORE_TYPE is `minio`
496+
- `PATH`: **data/attachments**: Path to store attachments only available when STORAGE_TYPE is `local`
497+
- `MINIO_ENDPOINT`: **localhost:9000**: Minio endpoint to connect only available when STORAGE_TYPE is `minio`
498+
- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`
499+
- `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio`
500+
- `MINIO_BUCKET`: **gitea**: Minio bucket to store the attachments only available when STORAGE_TYPE is `minio`
501+
- `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when STORAGE_TYPE is `minio`
502+
- `MINIO_BASE_PATH`: **attachments/**: Minio base path on the bucket only available when STORAGE_TYPE is `minio`
503+
- `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when STORAGE_TYPE is `minio`
514504

515505
## Log (`log`)
516506

@@ -714,6 +704,56 @@ Task queue configuration has been moved to `queue.task`. However, the below conf
714704
- `MAX_ATTEMPTS`: **3**: Max attempts per http/https request on migrations.
715705
- `RETRY_BACKOFF`: **3**: Backoff time per http/https request retry (seconds)
716706

707+
## LFS (`lfs`)
708+
709+
Storage configuration for lfs data. It will be derived from default `[storage]` or
710+
`[storage.xxx]` when set `STORAGE_TYPE` to `xxx`. When derived, the default of `PATH`
711+
is `data/lfs` and the default of `MINIO_BASE_PATH` is `lfs/`.
712+
713+
- `STORAGE_TYPE`: **local**: Storage type for lfs, `local` for local disk or `minio` for s3 compatible object storage service or other name defined with `[storage.xxx]`
714+
- `SERVE_DIRECT`: **false**: Allows the storage driver to redirect to authenticated URLs to serve files directly. Currently, only Minio/S3 is supported via signed URLs, local does nothing.
715+
- `CONTENT_PATH`: **./data/lfs**: Where to store LFS files, only available when `STORAGE_TYPE` is `local`.
716+
- `MINIO_ENDPOINT`: **localhost:9000**: Minio endpoint to connect only available when `STORAGE_TYPE` is `minio`
717+
- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `STORAGE_TYPE` is `minio`
718+
- `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when `STORAGE_TYPE is` `minio`
719+
- `MINIO_BUCKET`: **gitea**: Minio bucket to store the lfs only available when `STORAGE_TYPE` is `minio`
720+
- `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when `STORAGE_TYPE` is `minio`
721+
- `MINIO_BASE_PATH`: **lfs/**: Minio base path on the bucket only available when `STORAGE_TYPE` is `minio`
722+
- `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio`
723+
724+
## Storage (`storage`)
725+
726+
Default storage configuration for attachments, lfs, avatars and etc.
727+
728+
- `SERVE_DIRECT`: **false**: Allows the storage driver to redirect to authenticated URLs to serve files directly. Currently, only Minio/S3 is supported via signed URLs, local does nothing.
729+
- `MINIO_ENDPOINT`: **localhost:9000**: Minio endpoint to connect only available when `STORAGE_TYPE` is `minio`
730+
- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `STORAGE_TYPE` is `minio`
731+
- `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when `STORAGE_TYPE is` `minio`
732+
- `MINIO_BUCKET`: **gitea**: Minio bucket to store the data only available when `STORAGE_TYPE` is `minio`
733+
- `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when `STORAGE_TYPE` is `minio`
734+
- `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio`
735+
736+
And you can also define a customize storage like below:
737+
738+
```ini
739+
[storage.my_minio]
740+
STORAGE_TYPE = minio
741+
; Minio endpoint to connect only available when STORAGE_TYPE is `minio`
742+
MINIO_ENDPOINT = localhost:9000
743+
; Minio accessKeyID to connect only available when STORAGE_TYPE is `minio`
744+
MINIO_ACCESS_KEY_ID =
745+
; Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio`
746+
MINIO_SECRET_ACCESS_KEY =
747+
; Minio bucket to store the attachments only available when STORAGE_TYPE is `minio`
748+
MINIO_BUCKET = gitea
749+
; Minio location to create bucket only available when STORAGE_TYPE is `minio`
750+
MINIO_LOCATION = us-east-1
751+
; Minio enabled ssl only available when STORAGE_TYPE is `minio`
752+
MINIO_USE_SSL = false
753+
```
754+
755+
And used by `[attachment]`, `[lfs]` and etc. as `STORAGE_TYPE`.
756+
717757
## Other (`other`)
718758

719759
- `SHOW_FOOTER_BRANDING`: **false**: Show Gitea branding in the footer.

0 commit comments

Comments
 (0)