Merge master into master-IB#1105071

Author: pboguslawski

.changelog.yml

@@ -22,6 +22,10 @@ groups:
     name: SECURITY
     labels:
       - kind/security
+  -
+    name: API
+    labels:
+      - kind/api
   -
     name: BUGFIXES
     labels:

.drone.yml

@@ -113,18 +113,6 @@ services:
     environment:
       MYSQL_ALLOW_EMPTY_PASSWORD: yes
       MYSQL_DATABASE: test
-      GOPROXY: off
-      TAGS: bindata sqlite sqlite_unlock_notify
-      GITLAB_READ_TOKEN:
-        from_secret: gitlab_read_token
-    depends_on:
-      - build
-    when:
-      branch:
-        - master
-      event:
-        - push
-        - pull_request
 
   - name: mysql8
     pull: default
@@ -152,7 +140,7 @@ services:
     image: elasticsearch:7.5.0
 
   - name: minio
-    image: minio/minio:RELEASE.2020-05-16T01-33-21Z
+    image: minio/minio:RELEASE.2020-10-09T22-55-05Z
     commands:
     - minio server /data
     environment:

.golangci.yml

@@ -98,3 +98,12 @@ issues:
     - path: models/update.go
       linters:
         - unused
+    - path: cmd/dump.go
+      linters:
+        - dupl
+    - text: "commentFormatting: put a space between `//` and comment text"
+      linters:
+        - gocritic
+    - text: "exitAfterDefer:"
+      linters:
+        - gocritic

CHANGELOG.md

@@ -8,7 +8,7 @@ ENV GOPROXY ${GOPROXY:-direct}
 
 ARG GITEA_VERSION
 ARG TAGS="sqlite sqlite_unlock_notify"
-ENV TAGS "bindata $TAGS"
+ENV TAGS "bindata timetzdata $TAGS"
 ARG CGO_EXTRA_CFLAGS
 
 #Build deps
@@ -38,7 +38,6 @@ RUN apk --no-cache add \
     s6 \
     sqlite \
     su-exec \
-    tzdata \
     gnupg
 
 RUN addgroup \

Dockerfile

@@ -120,11 +120,12 @@ endif
 
 GO_SOURCES_OWN := $(filter-out vendor/% %/bindata.go, $(GO_SOURCES))
 
-#To update swagger use: GO111MODULE=on go get -u github.com/go-swagger/go-swagger/cmd/swagger@v0.20.1
+#To update swagger use: GO111MODULE=on go get -u github.com/go-swagger/go-swagger/cmd/swagger
 SWAGGER := $(GO) run -mod=vendor github.com/go-swagger/go-swagger/cmd/swagger
 SWAGGER_SPEC := templates/swagger/v1_json.tmpl
 SWAGGER_SPEC_S_TMPL := s|"basePath": *"/api/v1"|"basePath": "{{AppSubUrl}}/api/v1"|g
 SWAGGER_SPEC_S_JSON := s|"basePath": *"{{AppSubUrl}}/api/v1"|"basePath": "/api/v1"|g
+SWAGGER_EXCLUDE := code.gitea.io/sdk
 SWAGGER_NEWLINE_COMMAND := -e '$$a\'
 
 TEST_MYSQL_HOST ?= mysql:3306
@@ -185,7 +186,7 @@ help:
 
 .PHONY: go-check
 go-check:
-	$(eval GO_VERSION := $(shell printf "%03d%03d%03d" $(shell go version | grep -Eo '[0-9]+\.[0-9.]+' | tr '.' ' ');))
+	$(eval GO_VERSION := $(shell printf "%03d%03d%03d" $(shell $(GO) version | grep -Eo '[0-9]+\.[0-9.]+' | tr '.' ' ');))
 	@if [ "$(GO_VERSION)" -lt "$(MIN_GO_VERSION)" ]; then \
 		echo "Gitea requires Go 1.13 or greater to build. You can get it at https://golang.org/dl/"; \
 		exit 1; \
@@ -243,7 +244,7 @@ endif
 
 .PHONY: generate-swagger
 generate-swagger:
-	$(SWAGGER) generate spec -o './$(SWAGGER_SPEC)'
+	$(SWAGGER) generate spec -x "$(SWAGGER_EXCLUDE)" -o './$(SWAGGER_SPEC)'
 	$(SED_INPLACE) '$(SWAGGER_SPEC_S_TMPL)' './$(SWAGGER_SPEC)'
 	$(SED_INPLACE) $(SWAGGER_NEWLINE_COMMAND) './$(SWAGGER_SPEC)'
 
@@ -697,7 +698,7 @@ pr\#%: clean-all
 golangci-lint:
 	@hash golangci-lint > /dev/null 2>&1; if [ $$? -ne 0 ]; then \
 		export BINARY="golangci-lint"; \
-		curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(GOPATH)/bin v1.30.0; \
+		curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(GOPATH)/bin v1.31.0; \
 	fi
 	golangci-lint run --timeout 5m
 

Makefile

@@ -18,6 +18,7 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/storage"
 	"code.gitea.io/gitea/modules/util"
 
 	"gitea.com/macaron/session"
@@ -57,6 +58,8 @@ func addRecursive(w archiver.Writer, dirPath string, absPath string, verbose boo
 	if err != nil {
 		return fmt.Errorf("Could not open directory %s: %s", absPath, err)
 	}
+	defer dir.Close()
+
 	files, err := dir.Readdir(0)
 	if err != nil {
 		return fmt.Errorf("Unable to list files in %s: %s", absPath, err)
@@ -197,6 +200,10 @@ func runDump(ctx *cli.Context) error {
 		return err
 	}
 
+	if err := storage.Init(); err != nil {
+		return err
+	}
+
 	if file == nil {
 		file, err = os.Create(fileName)
 		if err != nil {
@@ -231,11 +238,21 @@ func runDump(ctx *cli.Context) error {
 			fatal("Failed to include repositories: %v", err)
 		}
 
-		if _, err := os.Stat(setting.LFS.ContentPath); !os.IsNotExist(err) {
-			log.Info("Dumping lfs... %s", setting.LFS.ContentPath)
-			if err := addRecursive(w, "lfs", setting.LFS.ContentPath, verbose); err != nil {
-				fatal("Failed to include lfs: %v", err)
+		if err := storage.LFS.IterateObjects(func(objPath string, object storage.Object) error {
+			info, err := object.Stat()
+			if err != nil {
+				return err
 			}
+
+			return w.Write(archiver.File{
+				FileInfo: archiver.FileInfo{
+					FileInfo:   info,
+					CustomName: path.Join("data", "lfs", objPath),
+				},
+				ReadCloser: object,
+			})
+		}); err != nil {
+			fatal("Failed to dump LFS objects: %v", err)
 		}
 	}
 
@@ -302,13 +319,31 @@ func runDump(ctx *cli.Context) error {
 		}
 
 		excludes = append(excludes, setting.RepoRootPath)
-		excludes = append(excludes, setting.LFS.ContentPath)
+		excludes = append(excludes, setting.LFS.Path)
+		excludes = append(excludes, setting.Attachment.Path)
 		excludes = append(excludes, setting.LogRootPath)
 		if err := addRecursiveExclude(w, "data", setting.AppDataPath, excludes, verbose); err != nil {
 			fatal("Failed to include data directory: %v", err)
 		}
 	}
 
+	if err := storage.Attachments.IterateObjects(func(objPath string, object storage.Object) error {
+		info, err := object.Stat()
+		if err != nil {
+			return err
+		}
+
+		return w.Write(archiver.File{
+			FileInfo: archiver.FileInfo{
+				FileInfo:   info,
+				CustomName: path.Join("data", "attachments", objPath),
+			},
+			ReadCloser: object,
+		})
+	}); err != nil {
+		fatal("Failed to dump attachments: %v", err)
+	}
+
 	// Doesn't check if LogRootPath exists before processing --skip-log intentionally,
 	// ensuring that it's clear the dump is skipped whether the directory's initialized
 	// yet or not.

cmd/dump.go

@@ -285,6 +285,12 @@ func runHookUpdate(c *cli.Context) error {
 }
 
 func runHookPostReceive(c *cli.Context) error {
+	// First of all run update-server-info no matter what
+	if _, err := git.NewCommand("update-server-info").Run(); err != nil {
+		return fmt.Errorf("Failed to call 'git update-server-info': %v", err)
+	}
+
+	// Now if we're an internal don't do anything else
 	if os.Getenv(models.EnvIsInternal) == "true" {
 		return nil
 	}

cmd/hook.go

@@ -7,6 +7,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/models/migrations"
@@ -30,9 +31,9 @@ var CmdMigrateStorage = cli.Command{
 			Usage: "Kinds of files to migrate, currently only 'attachments' is supported",
 		},
 		cli.StringFlag{
-			Name:  "store, s",
-			Value: "local",
-			Usage: "New storage type, local or minio",
+			Name:  "storage, s",
+			Value: "",
+			Usage: "New storage type: local (default) or minio",
 		},
 		cli.StringFlag{
 			Name:  "path, p",
@@ -90,6 +91,20 @@ func migrateLFS(dstStorage storage.ObjectStorage) error {
 	})
 }
 
+func migrateAvatars(dstStorage storage.ObjectStorage) error {
+	return models.IterateUser(func(user *models.User) error {
+		_, err := storage.Copy(dstStorage, user.CustomAvatarRelativePath(), storage.Avatars, user.CustomAvatarRelativePath())
+		return err
+	})
+}
+
+func migrateRepoAvatars(dstStorage storage.ObjectStorage) error {
+	return models.IterateRepository(func(repo *models.Repository) error {
+		_, err := storage.Copy(dstStorage, repo.CustomAvatarRelativePath(), storage.RepoAvatars, repo.CustomAvatarRelativePath())
+		return err
+	})
+}
+
 func runMigrateStorage(ctx *cli.Context) error {
 	if err := initDB(); err != nil {
 		return err
@@ -106,40 +121,48 @@ func runMigrateStorage(ctx *cli.Context) error {
 		return err
 	}
 
+	goCtx := context.Background()
+
 	if err := storage.Init(); err != nil {
 		return err
 	}
 
 	var dstStorage storage.ObjectStorage
 	var err error
-	switch ctx.String("store") {
-	case "local":
+	switch strings.ToLower(ctx.String("storage")) {
+	case "":
+		fallthrough
+	case string(storage.LocalStorageType):
 		p := ctx.String("path")
 		if p == "" {
-			log.Fatal("Path must be given when store is loal")
+			log.Fatal("Path must be given when storage is loal")
 			return nil
 		}
-		dstStorage, err = storage.NewLocalStorage(p)
-	case "minio":
+		dstStorage, err = storage.NewLocalStorage(
+			goCtx,
+			storage.LocalStorageConfig{
+				Path: p,
+			})
+	case string(storage.MinioStorageType):
 		dstStorage, err = storage.NewMinioStorage(
-			context.Background(),
-			ctx.String("minio-endpoint"),
-			ctx.String("minio-access-key-id"),
-			ctx.String("minio-secret-access-key"),
-			ctx.String("minio-bucket"),
-			ctx.String("minio-location"),
-			ctx.String("minio-base-path"),
-			ctx.Bool("minio-use-ssl"),
-		)
+			goCtx,
+			storage.MinioStorageConfig{
+				Endpoint:        ctx.String("minio-endpoint"),
+				AccessKeyID:     ctx.String("minio-access-key-id"),
+				SecretAccessKey: ctx.String("minio-secret-access-key"),
+				Bucket:          ctx.String("minio-bucket"),
+				Location:        ctx.String("minio-location"),
+				BasePath:        ctx.String("minio-base-path"),
+				UseSSL:          ctx.Bool("minio-use-ssl"),
+			})
 	default:
-		return fmt.Errorf("Unsupported attachments store type: %s", ctx.String("store"))
+		return fmt.Errorf("Unsupported storage type: %s", ctx.String("storage"))
 	}
-
 	if err != nil {
 		return err
 	}
 
-	tp := ctx.String("type")
+	tp := strings.ToLower(ctx.String("type"))
 	switch tp {
 	case "attachments":
 		if err := migrateAttachments(dstStorage); err != nil {
@@ -149,6 +172,14 @@ func runMigrateStorage(ctx *cli.Context) error {
 		if err := migrateLFS(dstStorage); err != nil {
 			return err
 		}
+	case "avatars":
+		if err := migrateAvatars(dstStorage); err != nil {
+			return err
+		}
+	case "repo-avatars":
+		if err := migrateRepoAvatars(dstStorage); err != nil {
+			return err
+		}
 	default:
 		return fmt.Errorf("Unsupported storage: %s", ctx.String("type"))
 	}

cmd/migrate_storage.go

@@ -54,6 +54,8 @@ func setup(logPath string, debug bool) {
 	_ = log.DelLogger("console")
 	if debug {
 		_ = log.NewLogger(1000, "console", "console", `{"level":"trace","stacktracelevel":"NONE","stderr":true}`)
+	} else {
+		_ = log.NewLogger(1000, "console", "console", `{"level":"fatal","stacktracelevel":"NONE","stderr":true}`)
 	}
 	setting.NewContext()
 	if debug {
@@ -111,9 +113,12 @@ func runServ(c *cli.Context) error {
 		if err != nil {
 			fail("Internal error", "Failed to check provided key: %v", err)
 		}
-		if key.Type == models.KeyTypeDeploy {
+		switch key.Type {
+		case models.KeyTypeDeploy:
 			println("Hi there! You've successfully authenticated with the deploy key named " + key.Name + ", but Gitea does not provide shell access.")
-		} else {
+		case models.KeyTypePrincipal:
+			println("Hi there! You've successfully authenticated with the principal " + key.Content + ", but Gitea does not provide shell access.")
+		default:
 			println("Hi there, " + user.Name + "! You've successfully authenticated with the key named " + key.Name + ", but Gitea does not provide shell access.")
 		}
 		println("If this is unexpected, please log in with password and setup Gitea under another user.")

cmd/serv.go

@@ -93,7 +93,7 @@ func runLetsEncryptFallbackHandler(w http.ResponseWriter, r *http.Request) {
 	// Remove the trailing slash at the end of setting.AppURL, the request
 	// URI always contains a leading slash, which would result in a double
 	// slash
-	target := strings.TrimRight(setting.AppURL, "/") + r.URL.RequestURI()
+	target := strings.TrimSuffix(setting.AppURL, "/") + r.URL.RequestURI()
 	http.Redirect(w, r, target, http.StatusFound)
 }