Merge branch 'main' into cache-last-commit-data-as-found

Author: zeripath

.drone.yml

@@ -217,6 +217,17 @@ steps:
       GITHUB_READ_TOKEN:
         from_secret: github_read_token
 
+  - name: unit-test-race
+    image: golang:1.16
+    commands:
+      - make test-backend
+    environment:
+      GOPROXY: off
+      TAGS: sqlite sqlite_unlock_notify
+      RACE_ENABLED: true
+      GITHUB_READ_TOKEN:
+        from_secret: github_read_token
+
   - name: unit-test-gogit
     pull: always
     image: golang:1.16

Makefile

@@ -359,7 +359,7 @@ test: test-frontend test-backend
 
 .PHONY: test-backend
 test-backend:
-	@echo "Running go test with -tags '$(TEST_TAGS)'..."
+	@echo "Running go test with $(GOTESTFLAGS) -tags '$(TEST_TAGS)'..."
 	@$(GO) test $(GOTESTFLAGS) -mod=vendor -tags='$(TEST_TAGS)' $(GO_PACKAGES)
 
 .PHONY: test-frontend
@@ -389,7 +389,7 @@ coverage:
 
 .PHONY: unit-test-coverage
 unit-test-coverage:
-	@echo "Running unit-test-coverage -tags '$(TEST_TAGS)'..."
+	@echo "Running unit-test-coverage $(GOTESTFLAGS) -tags '$(TEST_TAGS)'..."
 	@$(GO) test $(GOTESTFLAGS) -mod=vendor -tags='$(TEST_TAGS)' -cover -coverprofile coverage.out $(GO_PACKAGES) && echo "\n==>\033[32m Ok\033[m\n" || exit 1
 
 .PHONY: vendor

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -527,12 +527,11 @@ relation to port exhaustion.
 - `USER_DELETE_WITH_COMMENTS_MAX_TIME`: **0** Minimum amount of time a user must exist before comments are kept when the user is deleted.
 - `VALID_SITE_URL_SCHEMES`: **http, https**: Valid site url schemes for user profiles
 
-### Service - Expore (`service.explore`)
+### Service - Explore (`service.explore`)
 
 - `REQUIRE_SIGNIN_VIEW`: **false**: Only allow signed in users to view the explore pages.
 - `DISABLE_USERS_PAGE`: **false**: Disable the users explore page.
 
-
 ## SSH Minimum Key Sizes (`ssh.minimum_key_sizes`)
 
 Define allowed algorithms and their minimum key length (use -1 to disable a type):

docs/content/doc/advanced/customizing-gitea.en-us.md

@@ -200,7 +200,7 @@ to the file `templates/custom/footer.tmpl`
 
 You also need to download the content of the library [Madeleine.js](https://jinjunho.github.io/Madeleine.js/) and place it under `$GITEA_CUSTOM/public/` folder.
 
-You should end-up with a folder structucture similar to:
+You should end-up with a folder structure similar to:
 
 ```
 $GITEA_CUSTOM/templates

docs/content/doc/advanced/make.fr-fr.md

@@ -37,8 +37,8 @@ sudo yum install make
 
 Si vous utilisez Windows, vous pouvez télécharger une des versions suivantes de Make:
 
-- [Simple binaire](http://www.equation.com/servlet/equation.cmd?fa=make). Copiez le quelque part et mettez à jour `PATH`.
+- [Simple binaire](http://www.equation.com/servlet/equation.cmd?fa=make). Copiez-le quelque part et mettez à jour `PATH`.
   - [32-bits version](ftp://ftp.equation.com/make/32/make.exe)
   - [64-bits version](ftp://ftp.equation.com/make/64/make.exe)
-- [MinGW](http://www.mingw.org/) includes a build. The binary is called `mingw32-make.exe` instead of `make.exe`. Add the `bin` folder to your `PATH`.
-- [Chocolatey package](https://chocolatey.org/packages/make). Run `choco install make`
+- [MinGW](http://www.mingw.org/) inclut un _build_. Le fichier binaire est nommé `mingw32-make.exe` plutôt que `make.exe`. Ajoutez le dossier `bin` à votre `PATH`.
+- [Chocolatey package](https://chocolatey.org/packages/make). Exécutez `choco install make`.

docs/content/doc/installation/windows-service.fr-fr.md

@@ -21,9 +21,9 @@ Pour activer le service Windows Gitea, ouvrez une `cmd` en tant qu'Administrateu
 sc create gitea start= auto binPath= "\"C:\gitea\gitea.exe\" web --config \"C:\gitea\custom\conf\app.ini\""
 ```
 
-N'oubliez pas de remplacer `C:\gitea` par le chemin que vous avez utilisez pour votre installation.
+N'oubliez pas de remplacer `C:\gitea` par le chemin que vous avez utilisé pour votre installation.
 
-Ensuite, ouvrez "Services Windows", puis recherchez le service `gitea`, faites un clique droit et selectionnez "Run". Si tout fonctionne, vous devriez être capable d'accèder à Gitea à l'URL `http://localhost:3000` (ou sur le port configuré si différent de 3000).
+Ensuite, ouvrez "Services Windows", puis recherchez le service `gitea`, faites un clic droit et selectionnez "Run". Si tout fonctionne, vous devriez être capable d'accèder à Gitea à l'URL `http://localhost:3000` (ou sur le port configuré si différent de 3000).
 
 ## Désactiver un service Windows
 

integrations/auth_ldap_test.go

@@ -144,6 +144,60 @@ func TestLDAPUserSignin(t *testing.T) {
 	assert.Equal(t, u.Email, htmlDoc.Find(`label[for="email"]`).Siblings().First().Text())
 }
 
+func TestLDAPAuthChange(t *testing.T) {
+	defer prepareTestEnv(t)()
+	addAuthSourceLDAP(t, "")
+
+	session := loginUser(t, "user1")
+	req := NewRequest(t, "GET", "/admin/auths")
+	resp := session.MakeRequest(t, req, http.StatusOK)
+	doc := NewHTMLParser(t, resp.Body)
+	href, exists := doc.Find("table.table td a").Attr("href")
+	if !exists {
+		assert.True(t, exists, "No authentication source found")
+		return
+	}
+
+	req = NewRequest(t, "GET", href)
+	resp = session.MakeRequest(t, req, http.StatusOK)
+	doc = NewHTMLParser(t, resp.Body)
+	csrf := doc.GetCSRF()
+	host, _ := doc.Find(`input[name="host"]`).Attr("value")
+	assert.Equal(t, host, getLDAPServerHost())
+	binddn, _ := doc.Find(`input[name="bind_dn"]`).Attr("value")
+	assert.Equal(t, binddn, "uid=gitea,ou=service,dc=planetexpress,dc=com")
+
+	req = NewRequestWithValues(t, "POST", href, map[string]string{
+		"_csrf":                    csrf,
+		"type":                     "2",
+		"name":                     "ldap",
+		"host":                     getLDAPServerHost(),
+		"port":                     "389",
+		"bind_dn":                  "uid=gitea,ou=service,dc=planetexpress,dc=com",
+		"bind_password":            "password",
+		"user_base":                "ou=people,dc=planetexpress,dc=com",
+		"filter":                   "(&(objectClass=inetOrgPerson)(memberOf=cn=git,ou=people,dc=planetexpress,dc=com)(uid=%s))",
+		"admin_filter":             "(memberOf=cn=admin_staff,ou=people,dc=planetexpress,dc=com)",
+		"restricted_filter":        "(uid=leela)",
+		"attribute_username":       "uid",
+		"attribute_name":           "givenName",
+		"attribute_surname":        "sn",
+		"attribute_mail":           "mail",
+		"attribute_ssh_public_key": "",
+		"is_sync_enabled":          "on",
+		"is_active":                "on",
+	})
+	session.MakeRequest(t, req, http.StatusFound)
+
+	req = NewRequest(t, "GET", href)
+	resp = session.MakeRequest(t, req, http.StatusOK)
+	doc = NewHTMLParser(t, resp.Body)
+	host, _ = doc.Find(`input[name="host"]`).Attr("value")
+	assert.Equal(t, host, getLDAPServerHost())
+	binddn, _ = doc.Find(`input[name="bind_dn"]`).Attr("value")
+	assert.Equal(t, binddn, "uid=gitea,ou=service,dc=planetexpress,dc=com")
+}
+
 func TestLDAPUserSync(t *testing.T) {
 	if skipLDAPTests() {
 		t.Skip()

integrations/gitea-repositories-meta/user2/repo1.wiki.git/objects/0d/ca5bd9b5d7ef937710e056f575e86c0184ba85

@@ -1 +1 @@
-423313fbd38093bb10d0c8387db9105409c6f196
+0dca5bd9b5d7ef937710e056f575e86c0184ba85

integrations/gitea-repositories-meta/user2/repo1.wiki.git/objects/89/43a1d5f93c00439d5ffc0f8e36f5d60abae46c

@@ -7,6 +7,7 @@ package models
 
 import (
 	"crypto/tls"
+	"encoding/binary"
 	"errors"
 	"fmt"
 	"net/smtp"
@@ -70,11 +71,30 @@ var (
 	_ convert.Conversion = &SSPIConfig{}
 )
 
-// jsonUnmarshalIgnoreErroneousBOM - due to a bug in xorm (see https://gitea.com/xorm/xorm/pulls/1957) - it's
-// possible that a Blob may gain an unwanted prefix of 0xff 0xfe.
-func jsonUnmarshalIgnoreErroneousBOM(bs []byte, v interface{}) error {
+// jsonUnmarshalHandleDoubleEncode - due to a bug in xorm (see https://gitea.com/xorm/xorm/pulls/1957) - it's
+// possible that a Blob may be double encoded or gain an unwanted prefix of 0xff 0xfe.
+func jsonUnmarshalHandleDoubleEncode(bs []byte, v interface{}) error {
 	json := jsoniter.ConfigCompatibleWithStandardLibrary
 	err := json.Unmarshal(bs, v)
+	if err != nil {
+		ok := true
+		rs := []byte{}
+		temp := make([]byte, 2)
+		for _, rn := range string(bs) {
+			if rn > 0xffff {
+				ok = false
+				break
+			}
+			binary.LittleEndian.PutUint16(temp, uint16(rn))
+			rs = append(rs, temp...)
+		}
+		if ok {
+			if rs[0] == 0xff && rs[1] == 0xfe {
+				rs = rs[2:]
+			}
+			err = json.Unmarshal(rs, v)
+		}
+	}
 	if err != nil && len(bs) > 2 && bs[0] == 0xff && bs[1] == 0xfe {
 		err = json.Unmarshal(bs[2:], v)
 	}
@@ -88,7 +108,7 @@ type LDAPConfig struct {
 
 // FromDB fills up a LDAPConfig from serialized format.
 func (cfg *LDAPConfig) FromDB(bs []byte) error {
-	err := jsonUnmarshalIgnoreErroneousBOM(bs, &cfg)
+	err := jsonUnmarshalHandleDoubleEncode(bs, &cfg)
 	if err != nil {
 		return err
 	}
@@ -129,7 +149,7 @@ type SMTPConfig struct {
 
 // FromDB fills up an SMTPConfig from serialized format.
 func (cfg *SMTPConfig) FromDB(bs []byte) error {
-	return jsonUnmarshalIgnoreErroneousBOM(bs, cfg)
+	return jsonUnmarshalHandleDoubleEncode(bs, cfg)
 }
 
 // ToDB exports an SMTPConfig to a serialized format.
@@ -146,7 +166,7 @@ type PAMConfig struct {
 
 // FromDB fills up a PAMConfig from serialized format.
 func (cfg *PAMConfig) FromDB(bs []byte) error {
-	return jsonUnmarshalIgnoreErroneousBOM(bs, cfg)
+	return jsonUnmarshalHandleDoubleEncode(bs, cfg)
 }
 
 // ToDB exports a PAMConfig to a serialized format.
@@ -167,7 +187,7 @@ type OAuth2Config struct {
 
 // FromDB fills up an OAuth2Config from serialized format.
 func (cfg *OAuth2Config) FromDB(bs []byte) error {
-	return jsonUnmarshalIgnoreErroneousBOM(bs, cfg)
+	return jsonUnmarshalHandleDoubleEncode(bs, cfg)
 }
 
 // ToDB exports an SMTPConfig to a serialized format.
@@ -187,7 +207,7 @@ type SSPIConfig struct {
 
 // FromDB fills up an SSPIConfig from serialized format.
 func (cfg *SSPIConfig) FromDB(bs []byte) error {
-	return jsonUnmarshalIgnoreErroneousBOM(bs, cfg)
+	return jsonUnmarshalHandleDoubleEncode(bs, cfg)
 }
 
 // ToDB exports an SSPIConfig to a serialized format.

integrations/gitea-repositories-meta/user2/repo1.wiki.git/refs/heads/master

@@ -28,7 +28,7 @@ type UnitConfig struct{}
 
 // FromDB fills up a UnitConfig from serialized format.
 func (cfg *UnitConfig) FromDB(bs []byte) error {
-	return jsonUnmarshalIgnoreErroneousBOM(bs, &cfg)
+	return jsonUnmarshalHandleDoubleEncode(bs, &cfg)
 }
 
 // ToDB exports a UnitConfig to a serialized format.
@@ -44,7 +44,7 @@ type ExternalUncycloConfig struct {
 
 // FromDB fills up a ExternalUncycloConfig from serialized format.
 func (cfg *ExternalUncycloConfig) FromDB(bs []byte) error {
-	return jsonUnmarshalIgnoreErroneousBOM(bs, &cfg)
+	return jsonUnmarshalHandleDoubleEncode(bs, &cfg)
 }
 
 // ToDB exports a ExternalUncycloConfig to a serialized format.
@@ -62,7 +62,7 @@ type ExternalTrackerConfig struct {
 
 // FromDB fills up a ExternalTrackerConfig from serialized format.
 func (cfg *ExternalTrackerConfig) FromDB(bs []byte) error {
-	return jsonUnmarshalIgnoreErroneousBOM(bs, &cfg)
+	return jsonUnmarshalHandleDoubleEncode(bs, &cfg)
 }
 
 // ToDB exports a ExternalTrackerConfig to a serialized format.
@@ -80,7 +80,7 @@ type IssuesConfig struct {
 
 // FromDB fills up a IssuesConfig from serialized format.
 func (cfg *IssuesConfig) FromDB(bs []byte) error {
-	return jsonUnmarshalIgnoreErroneousBOM(bs, &cfg)
+	return jsonUnmarshalHandleDoubleEncode(bs, &cfg)
 }
 
 // ToDB exports a IssuesConfig to a serialized format.
@@ -104,7 +104,7 @@ type PullRequestsConfig struct {
 
 // FromDB fills up a PullRequestsConfig from serialized format.
 func (cfg *PullRequestsConfig) FromDB(bs []byte) error {
-	return jsonUnmarshalIgnoreErroneousBOM(bs, &cfg)
+	return jsonUnmarshalHandleDoubleEncode(bs, &cfg)
 }
 
 // ToDB exports a PullRequestsConfig to a serialized format.

models/login_source.go

@@ -7,6 +7,7 @@ package git
 import (
 	"bufio"
 	"bytes"
+	"context"
 	"io"
 	"math"
 	"strconv"
@@ -28,23 +29,28 @@ type WriteCloserError interface {
 func CatFileBatchCheck(repoPath string) (WriteCloserError, *bufio.Reader, func()) {
 	batchStdinReader, batchStdinWriter := io.Pipe()
 	batchStdoutReader, batchStdoutWriter := io.Pipe()
+	ctx, ctxCancel := context.WithCancel(DefaultContext)
+	closed := make(chan struct{})
 	cancel := func() {
 		_ = batchStdinReader.Close()
 		_ = batchStdinWriter.Close()
 		_ = batchStdoutReader.Close()
 		_ = batchStdoutWriter.Close()
+		ctxCancel()
+		<-closed
 	}
 
 	go func() {
 		stderr := strings.Builder{}
-		err := NewCommand("cat-file", "--batch-check").RunInDirFullPipeline(repoPath, batchStdoutWriter, &stderr, batchStdinReader)
+		err := NewCommandContext(ctx, "cat-file", "--batch-check").RunInDirFullPipeline(repoPath, batchStdoutWriter, &stderr, batchStdinReader)
 		if err != nil {
 			_ = batchStdoutWriter.CloseWithError(ConcatenateError(err, (&stderr).String()))
 			_ = batchStdinReader.CloseWithError(ConcatenateError(err, (&stderr).String()))
 		} else {
 			_ = batchStdoutWriter.Close()
 			_ = batchStdinReader.Close()
 		}
+		close(closed)
 	}()
 
 	// For simplicities sake we'll use a buffered reader to read from the cat-file --batch-check
@@ -59,23 +65,28 @@ func CatFileBatch(repoPath string) (WriteCloserError, *bufio.Reader, func()) {
 	// so let's create a batch stdin and stdout
 	batchStdinReader, batchStdinWriter := io.Pipe()
 	batchStdoutReader, batchStdoutWriter := nio.Pipe(buffer.New(32 * 1024))
+	ctx, ctxCancel := context.WithCancel(DefaultContext)
+	closed := make(chan struct{})
 	cancel := func() {
 		_ = batchStdinReader.Close()
 		_ = batchStdinWriter.Close()
 		_ = batchStdoutReader.Close()
 		_ = batchStdoutWriter.Close()
+		ctxCancel()
+		<-closed
 	}
 
 	go func() {
 		stderr := strings.Builder{}
-		err := NewCommand("cat-file", "--batch").RunInDirFullPipeline(repoPath, batchStdoutWriter, &stderr, batchStdinReader)
+		err := NewCommandContext(ctx, "cat-file", "--batch").RunInDirFullPipeline(repoPath, batchStdoutWriter, &stderr, batchStdinReader)
 		if err != nil {
 			_ = batchStdoutWriter.CloseWithError(ConcatenateError(err, (&stderr).String()))
 			_ = batchStdinReader.CloseWithError(ConcatenateError(err, (&stderr).String()))
 		} else {
 			_ = batchStdoutWriter.Close()
 			_ = batchStdinReader.Close()
 		}
+		close(closed)
 	}()
 
 	// For simplicities sake we'll us a buffered reader to read from the cat-file --batch

models/repo_unit.go

@@ -6,6 +6,7 @@
 package git
 
 import (
+	"bytes"
 	"strings"
 )
 
@@ -45,3 +46,23 @@ func (t *Tree) SubTree(rpath string) (*Tree, error) {
 	}
 	return g, nil
 }
+
+// LsTree checks if the given filenames are in the tree
+func (repo *Repository) LsTree(ref string, filenames ...string) ([]string, error) {
+	cmd := NewCommand("ls-tree", "-z", "--name-only", "--", ref)
+	for _, arg := range filenames {
+		if arg != "" {
+			cmd.AddArguments(arg)
+		}
+	}
+	res, err := cmd.RunInDirBytes(repo.Path)
+	if err != nil {
+		return nil, err
+	}
+	filelist := make([]string, 0, len(filenames))
+	for _, line := range bytes.Split(res, []byte{'\000'}) {
+		filelist = append(filelist, string(line))
+	}
+
+	return filelist, err
+}