Dump: add output format tar and output to stdout (#10376)

* Dump: Use mholt/archive/v3 to support tar including many compressions

Signed-off-by: Philipp Homann <[email protected]>

* Dump: Allow dump output to stdout

Signed-off-by: Philipp Homann <[email protected]>

* Dump: Fixed bug present since #6677 where SessionConfig.Provider is never "file"

Signed-off-by: Philipp Homann <[email protected]>

* Dump: never pack RepoRootPath, LFS.ContentPath and LogRootPath when they are below AppDataPath

Signed-off-by: Philipp Homann <[email protected]>

* Dump: also dump LFS (fixes #10058)

Signed-off-by: Philipp Homann <[email protected]>

* Dump: never dump CustomPath if CustomPath is a subdir of or equal to AppDataPath (fixes #10365)

Signed-off-by: Philipp Homann <[email protected]>

* Use log.Info instead of fmt.Fprintf

Signed-off-by: Philipp Homann <[email protected]>

* import ordering

* make fmt

Co-authored-by: zeripath <[email protected]>
Co-authored-by: techknowlogick <[email protected]>
Co-authored-by: Matti R <[email protected]>

Author: 3 people

cmd/dump.go

@@ -6,22 +6,120 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 
-	"github.com/unknwon/cae/zip"
+	"gitea.com/macaron/session"
+	archiver "github.com/mholt/archiver/v3"
 	"github.com/unknwon/com"
 	"github.com/urfave/cli"
 )
 
+func addFile(w archiver.Writer, filePath string, absPath string, verbose bool) error {
+	if verbose {
+		log.Info("Adding file %s\n", filePath)
+	}
+	file, err := os.Open(absPath)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	fileInfo, err := file.Stat()
+	if err != nil {
+		return err
+	}
+
+	return w.Write(archiver.File{
+		FileInfo: archiver.FileInfo{
+			FileInfo:   fileInfo,
+			CustomName: filePath,
+		},
+		ReadCloser: file,
+	})
+}
+
+func addRecursive(w archiver.Writer, dirPath string, absPath string, verbose bool) error {
+	if verbose {
+		log.Info("Adding dir  %s\n", dirPath)
+	}
+	dir, err := os.Open(absPath)
+	if err != nil {
+		return fmt.Errorf("Could not open directory %s: %s", absPath, err)
+	}
+	files, err := dir.Readdir(0)
+	if err != nil {
+		return fmt.Errorf("Unable to list files in %s: %s", absPath, err)
+	}
+
+	if err := addFile(w, dirPath, absPath, false); err != nil {
+		return err
+	}
+
+	for _, fileInfo := range files {
+		if fileInfo.IsDir() {
+			err = addRecursive(w, filepath.Join(dirPath, fileInfo.Name()), filepath.Join(absPath, fileInfo.Name()), verbose)
+		} else {
+			err = addFile(w, filepath.Join(dirPath, fileInfo.Name()), filepath.Join(absPath, fileInfo.Name()), verbose)
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func isSubdir(upper string, lower string) (bool, error) {
+	if relPath, err := filepath.Rel(upper, lower); err != nil {
+		return false, err
+	} else if relPath == "." || !strings.HasPrefix(relPath, ".") {
+		return true, nil
+	}
+	return false, nil
+}
+
+type outputType struct {
+	Enum     []string
+	Default  string
+	selected string
+}
+
+func (o outputType) Join() string {
+	return strings.Join(o.Enum, ", ")
+}
+
+func (o *outputType) Set(value string) error {
+	for _, enum := range o.Enum {
+		if enum == value {
+			o.selected = value
+			return nil
+		}
+	}
+
+	return fmt.Errorf("allowed values are %s", o.Join())
+}
+
+func (o outputType) String() string {
+	if o.selected == "" {
+		return o.Default
+	}
+	return o.selected
+}
+
+var outputTypeEnum = &outputType{
+	Enum:    []string{"zip", "tar", "tar.gz", "tar.xz", "tar.bz2"},
+	Default: "zip",
+}
+
 // CmdDump represents the available dump sub-command.
 var CmdDump = cli.Command{
 	Name:  "dump",
@@ -33,7 +131,7 @@ It can be used for backup and capture Gitea server image to send to maintainer`,
 		cli.StringFlag{
 			Name:  "file, f",
 			Value: fmt.Sprintf("gitea-dump-%d.zip", time.Now().Unix()),
-			Usage: "Name of the dump file which will be created.",
+			Usage: "Name of the dump file which will be created. Supply '-' for stdout. See type for available types.",
 		},
 		cli.BoolFlag{
 			Name:  "verbose, V",
@@ -56,6 +154,11 @@ It can be used for backup and capture Gitea server image to send to maintainer`,
 			Name:  "skip-log, L",
 			Usage: "Skip the log dumping",
 		},
+		cli.GenericFlag{
+			Name:  "type",
+			Value: outputTypeEnum,
+			Usage: fmt.Sprintf("Dump output format: %s", outputTypeEnum.Join()),
+		},
 	},
 }
 
@@ -65,79 +168,113 @@ func fatal(format string, args ...interface{}) {
 }
 
 func runDump(ctx *cli.Context) error {
+	var file *os.File
+	fileName := ctx.String("file")
+	if fileName == "-" {
+		file = os.Stdout
+		err := log.DelLogger("console")
+		if err != nil {
+			fatal("Deleting default logger failed. Can not write to stdout: %v", err)
+		}
+	}
 	setting.NewContext()
+	// make sure we are logging to the console no matter what the configuration tells us do to
+	if _, err := setting.Cfg.Section("log").NewKey("MODE", "console"); err != nil {
+		fatal("Setting logging mode to console failed: %v", err)
+	}
+	if _, err := setting.Cfg.Section("log.console").NewKey("STDERR", "true"); err != nil {
+		fatal("Setting console logger to stderr failed: %v", err)
+	}
 	setting.NewServices() // cannot access session settings otherwise
 
 	err := models.SetEngine()
 	if err != nil {
 		return err
 	}
 
-	tmpDir := ctx.String("tempdir")
-	if _, err := os.Stat(tmpDir); os.IsNotExist(err) {
-		fatal("Path does not exist: %s", tmpDir)
-	}
-	tmpWorkDir, err := ioutil.TempDir(tmpDir, "gitea-dump-")
-	if err != nil {
-		fatal("Failed to create tmp work directory: %v", err)
+	if file == nil {
+		file, err = os.Create(fileName)
+		if err != nil {
+			fatal("Unable to open %s: %v", fileName, err)
+		}
 	}
-	log.Info("Creating tmp work dir: %s", tmpWorkDir)
+	defer file.Close()
 
-	// work-around #1103
-	if os.Getenv("TMPDIR") == "" {
-		os.Setenv("TMPDIR", tmpWorkDir)
+	verbose := ctx.Bool("verbose")
+	outType := ctx.String("type")
+	var iface interface{}
+	if fileName == "-" {
+		iface, err = archiver.ByExtension(fmt.Sprintf(".%s", outType))
+	} else {
+		iface, err = archiver.ByExtension(fileName)
 	}
-
-	dbDump := path.Join(tmpWorkDir, "gitea-db.sql")
-
-	fileName := ctx.String("file")
-	log.Info("Packing dump files...")
-	z, err := zip.Create(fileName)
 	if err != nil {
-		fatal("Failed to create %s: %v", fileName, err)
+		fatal("Unable to get archiver for extension: %v", err)
 	}
 
-	zip.Verbose = ctx.Bool("verbose")
+	w, _ := iface.(archiver.Writer)
+	if err := w.Create(file); err != nil {
+		fatal("Creating archiver.Writer failed: %v", err)
+	}
+	defer w.Close()
 
 	if ctx.IsSet("skip-repository") && ctx.Bool("skip-repository") {
 		log.Info("Skip dumping local repositories")
 	} else {
-		log.Info("Dumping local repositories...%s", setting.RepoRootPath)
-		reposDump := path.Join(tmpWorkDir, "gitea-repo.zip")
-		if err := zip.PackTo(setting.RepoRootPath, reposDump, true); err != nil {
-			fatal("Failed to dump local repositories: %v", err)
+		log.Info("Dumping local repositories... %s", setting.RepoRootPath)
+		if err := addRecursive(w, "repos", setting.RepoRootPath, verbose); err != nil {
+			fatal("Failed to include repositories: %v", err)
 		}
-		if err := z.AddFile("gitea-repo.zip", reposDump); err != nil {
-			fatal("Failed to include gitea-repo.zip: %v", err)
+
+		if _, err := os.Stat(setting.LFS.ContentPath); !os.IsNotExist(err) {
+			log.Info("Dumping lfs... %s", setting.LFS.ContentPath)
+			if err := addRecursive(w, "lfs", setting.LFS.ContentPath, verbose); err != nil {
+				fatal("Failed to include lfs: %v", err)
+			}
 		}
 	}
 
+	tmpDir := ctx.String("tempdir")
+	if _, err := os.Stat(tmpDir); os.IsNotExist(err) {
+		fatal("Path does not exist: %s", tmpDir)
+	}
+
+	dbDump, err := ioutil.TempFile(tmpDir, "gitea-db.sql")
+	if err != nil {
+		fatal("Failed to create tmp file: %v", err)
+	}
+	defer os.Remove(dbDump.Name())
+
 	targetDBType := ctx.String("database")
 	if len(targetDBType) > 0 && targetDBType != setting.Database.Type {
 		log.Info("Dumping database %s => %s...", setting.Database.Type, targetDBType)
 	} else {
 		log.Info("Dumping database...")
 	}
 
-	if err := models.DumpDatabase(dbDump, targetDBType); err != nil {
+	if err := models.DumpDatabase(dbDump.Name(), targetDBType); err != nil {
 		fatal("Failed to dump database: %v", err)
 	}
 
-	if err := z.AddFile("gitea-db.sql", dbDump); err != nil {
+	if err := addFile(w, "gitea-db.sql", dbDump.Name(), verbose); err != nil {
 		fatal("Failed to include gitea-db.sql: %v", err)
 	}
 
 	if len(setting.CustomConf) > 0 {
 		log.Info("Adding custom configuration file from %s", setting.CustomConf)
-		if err := z.AddFile("app.ini", setting.CustomConf); err != nil {
+		if err := addFile(w, "app.ini", setting.CustomConf, verbose); err != nil {
 			fatal("Failed to include specified app.ini: %v", err)
 		}
 	}
 
 	customDir, err := os.Stat(setting.CustomPath)
 	if err == nil && customDir.IsDir() {
-		if err := z.AddDir("custom", setting.CustomPath); err != nil {
-			fatal("Failed to include custom: %v", err)
+		if is, _ := isSubdir(setting.AppDataPath, setting.CustomPath); !is {
+			if err := addRecursive(w, "custom", setting.CustomPath, verbose); err != nil {
+				fatal("Failed to include custom: %v", err)
+			}
+		} else {
+			log.Info("Custom dir %s is inside data dir %s, skipped", setting.CustomPath, setting.AppDataPath)
 		}
 	} else {
 		log.Info("Custom dir %s doesn't exist, skipped", setting.CustomPath)
@@ -146,11 +283,19 @@ func runDump(ctx *cli.Context) error {
 	if com.IsExist(setting.AppDataPath) {
 		log.Info("Packing data directory...%s", setting.AppDataPath)
 
-		var sessionAbsPath string
-		if setting.SessionConfig.Provider == "file" {
-			sessionAbsPath = setting.SessionConfig.ProviderConfig
+		var excludes []string
+		if setting.Cfg.Section("session").Key("PROVIDER").Value() == "file" {
+			var opts session.Options
+			if err = json.Unmarshal([]byte(setting.SessionConfig.ProviderConfig), &opts); err != nil {
+				return err
+			}
+			excludes = append(excludes, opts.ProviderConfig)
 		}
-		if err := zipAddDirectoryExclude(z, "data", setting.AppDataPath, sessionAbsPath); err != nil {
+
+		excludes = append(excludes, setting.RepoRootPath)
+		excludes = append(excludes, setting.LFS.ContentPath)
+		excludes = append(excludes, setting.LogRootPath)
+		if err := addRecursiveExclude(w, "data", setting.AppDataPath, excludes, verbose); err != nil {
 			fatal("Failed to include data directory: %v", err)
 		}
 	}
@@ -161,32 +306,42 @@ func runDump(ctx *cli.Context) error {
 	if ctx.IsSet("skip-log") && ctx.Bool("skip-log") {
 		log.Info("Skip dumping log files")
 	} else if com.IsExist(setting.LogRootPath) {
-		if err := z.AddDir("log", setting.LogRootPath); err != nil {
+		if err := addRecursive(w, "log", setting.LogRootPath, verbose); err != nil {
 			fatal("Failed to include log: %v", err)
 		}
 	}
 
-	if err = z.Close(); err != nil {
-		_ = os.Remove(fileName)
-		fatal("Failed to save %s: %v", fileName, err)
-	}
+	if fileName != "-" {
+		if err = w.Close(); err != nil {
+			_ = os.Remove(fileName)
+			fatal("Failed to save %s: %v", fileName, err)
+		}
 
-	if err := os.Chmod(fileName, 0600); err != nil {
-		log.Info("Can't change file access permissions mask to 0600: %v", err)
+		if err := os.Chmod(fileName, 0600); err != nil {
+			log.Info("Can't change file access permissions mask to 0600: %v", err)
+		}
 	}
 
-	log.Info("Removing tmp work dir: %s", tmpWorkDir)
-
-	if err := os.RemoveAll(tmpWorkDir); err != nil {
-		fatal("Failed to remove %s: %v", tmpWorkDir, err)
+	if fileName != "-" {
+		log.Info("Finish dumping in file %s", fileName)
+	} else {
+		log.Info("Finish dumping to stdout")
 	}
-	log.Info("Finish dumping in file %s", fileName)
 
 	return nil
 }
 
-// zipAddDirectoryExclude zips absPath to specified zipPath inside z excluding excludeAbsPath
-func zipAddDirectoryExclude(zip *zip.ZipArchive, zipPath, absPath string, excludeAbsPath string) error {
+func contains(slice []string, s string) bool {
+	for _, v := range slice {
+		if v == s {
+			return true
+		}
+	}
+	return false
+}
+
+// addRecursiveExclude zips absPath to specified insidePath inside writer excluding excludeAbsPath
+func addRecursiveExclude(w archiver.Writer, insidePath, absPath string, excludeAbsPath []string, verbose bool) error {
 	absPath, err := filepath.Abs(absPath)
 	if err != nil {
 		return err
@@ -197,24 +352,24 @@ func zipAddDirectoryExclude(zip *zip.ZipArchive, zipPath, absPath string, exclud
 	}
 	defer dir.Close()
 
-	zip.AddEmptyDir(zipPath)
-
 	files, err := dir.Readdir(0)
 	if err != nil {
 		return err
 	}
 	for _, file := range files {
 		currentAbsPath := path.Join(absPath, file.Name())
-		currentZipPath := path.Join(zipPath, file.Name())
+		currentInsidePath := path.Join(insidePath, file.Name())
 		if file.IsDir() {
-			if currentAbsPath != excludeAbsPath {
-				if err = zipAddDirectoryExclude(zip, currentZipPath, currentAbsPath, excludeAbsPath); err != nil {
+			if !contains(excludeAbsPath, currentAbsPath) {
+				if err := addFile(w, currentInsidePath, currentAbsPath, false); err != nil {
+					return err
+				}
+				if err = addRecursiveExclude(w, currentInsidePath, currentAbsPath, excludeAbsPath, verbose); err != nil {
 					return err
 				}
 			}
-
 		} else {
-			if err = zip.AddFile(currentZipPath, currentAbsPath); err != nil {
+			if err = addFile(w, currentInsidePath, currentAbsPath, verbose); err != nil {
 				return err
 			}
 		}