Merge branch 'master' into custom-repo-buttons

Author: a1012112796

contrib/pr/checkout.go

@@ -114,7 +114,7 @@ func runPR() {
 
 	log.Printf("[PR] Setting up router\n")
 	//routers.GlobalInit()
-	external.RegisterParsers()
+	external.RegisterRenderers()
 	markup.Init()
 	c := routes.NormalRoutes()
 

docs/content/doc/advanced/customizing-gitea.en-us.md

@@ -42,7 +42,7 @@ environment variable; this can be used to override the default path to something
 `GITEA_CUSTOM` might, for example, be set by an init script. You can check whether the value
 is set under the "Configuration" tab on the site administration page.
 
-- [List of Environment Variables](https://docs.gitea.io/en-us/specific-variables/)
+- [List of Environment Variables](https://docs.gitea.io/en-us/environment-variables/)
 
 **Note:** Gitea must perform a full restart to see configuration changes.
 

docs/content/doc/help/seek-help.en-us.md

@@ -33,4 +33,4 @@ If you found a bug, please create an [issue on GitHub](https://github.com/go-git
 
 ## Chinese Support
 
-Support for the Chinese language is provided at [gocn.vip](https://gocn.vip/topic/gitea).
+Support for the Chinese language is provided at [Our discourse](https://discourse.gitea.io/c/5-category/5).

docs/content/doc/help/seek-help.zh-cn.md

@@ -18,6 +18,6 @@ menu:
 如果您在使用或者开发过程中遇到问题，请到以下渠道咨询：
 
 - 到[Github issue](https://github.com/go-gitea/gitea/issues)提问(因为项目维护人员来自世界各地，为保证沟通顺畅，请使用英文提问)
-- 中文问题到[gocn.vip](https://gocn.vip/topic/gitea)提问
+- 中文问题到 [Gitea 论坛](https://discourse.gitea.io/c/5-category/5)提问
 - 访问 [Discord server - 英文](https://discord.gg/Gitea)
 - 加入 QQ群 328432459 获得进一步的支持

models/issue_comment.go

@@ -16,6 +16,7 @@ import (
 
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/markup/markdown"
 	"code.gitea.io/gitea/modules/references"
 	"code.gitea.io/gitea/modules/structs"
@@ -1178,8 +1179,13 @@ func findCodeComments(e Engine, opts FindCommentsOptions, issue *Issue, currentU
 			return nil, err
 		}
 
-		comment.RenderedContent = string(markdown.Render([]byte(comment.Content), issue.Repo.Link(),
-			issue.Repo.ComposeMetas()))
+		var err error
+		if comment.RenderedContent, err = markdown.RenderString(&markup.RenderContext{
+			URLPrefix: issue.Repo.Link(),
+			Metas:     issue.Repo.ComposeMetas(),
+		}, comment.Content); err != nil {
+			return nil, err
+		}
 	}
 	return comments[:n], nil
 }

models/repo.go

@@ -867,7 +867,10 @@ func (repo *Repository) getUsersWithAccessMode(e Engine, mode AccessMode) (_ []*
 
 // DescriptionHTML does special handles to description and return HTML string.
 func (repo *Repository) DescriptionHTML() template.HTML {
-	desc, err := markup.RenderDescriptionHTML([]byte(repo.Description), repo.HTMLURL(), repo.ComposeMetas())
+	desc, err := markup.RenderDescriptionHTML(&markup.RenderContext{
+		URLPrefix: repo.HTMLURL(),
+		Metas:     repo.ComposeMetas(),
+	}, repo.Description)
 	if err != nil {
 		log.Error("Failed to render description for %s (ID: %d): %v", repo.Name, repo.ID, err)
 		return template.HTML(markup.Sanitize(repo.Description))

models/repo_generate.go

@@ -5,13 +5,14 @@
 package models
 
 import (
+	"bufio"
+	"bytes"
 	"strconv"
 	"strings"
 
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/storage"
-	"code.gitea.io/gitea/modules/util"
 
 	"github.com/gobwas/glob"
 )
@@ -49,9 +50,9 @@ func (gt GiteaTemplate) Globs() []glob.Glob {
 	}
 
 	gt.globs = make([]glob.Glob, 0)
-	lines := strings.Split(string(util.NormalizeEOL(gt.Content)), "\n")
-	for _, line := range lines {
-		line = strings.TrimSpace(line)
+	scanner := bufio.NewScanner(bytes.NewReader(gt.Content))
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
 		if line == "" || strings.HasPrefix(line, "#") {
 			continue
 		}

modules/charset/charset.go

@@ -7,6 +7,8 @@ package charset
 import (
 	"bytes"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"strings"
 	"unicode/utf8"
 
@@ -21,6 +23,33 @@ import (
 // UTF8BOM is the utf-8 byte-order marker
 var UTF8BOM = []byte{'\xef', '\xbb', '\xbf'}
 
+// ToUTF8WithFallbackReader detects the encoding of content and coverts to UTF-8 reader if possible
+func ToUTF8WithFallbackReader(rd io.Reader) io.Reader {
+	var buf = make([]byte, 2048)
+	n, err := rd.Read(buf)
+	if err != nil {
+		return rd
+	}
+
+	charsetLabel, err := DetectEncoding(buf[:n])
+	if err != nil || charsetLabel == "UTF-8" {
+		return io.MultiReader(bytes.NewReader(RemoveBOMIfPresent(buf[:n])), rd)
+	}
+
+	encoding, _ := charset.Lookup(charsetLabel)
+	if encoding == nil {
+		return io.MultiReader(bytes.NewReader(buf[:n]), rd)
+	}
+
+	return transform.NewReader(
+		io.MultiReader(
+			bytes.NewReader(RemoveBOMIfPresent(buf[:n])),
+			rd,
+		),
+		encoding.NewDecoder(),
+	)
+}
+
 // ToUTF8WithErr converts content to UTF8 encoding
 func ToUTF8WithErr(content []byte) (string, error) {
 	charsetLabel, err := DetectEncoding(content)
@@ -49,24 +78,8 @@ func ToUTF8WithErr(content []byte) (string, error) {
 
 // ToUTF8WithFallback detects the encoding of content and coverts to UTF-8 if possible
 func ToUTF8WithFallback(content []byte) []byte {
-	charsetLabel, err := DetectEncoding(content)
-	if err != nil || charsetLabel == "UTF-8" {
-		return RemoveBOMIfPresent(content)
-	}
-
-	encoding, _ := charset.Lookup(charsetLabel)
-	if encoding == nil {
-		return content
-	}
-
-	// If there is an error, we concatenate the nicely decoded part and the
-	// original left over. This way we won't lose data.
-	result, n, err := transform.Bytes(encoding.NewDecoder(), content)
-	if err != nil {
-		return append(result, content[n:]...)
-	}
-
-	return RemoveBOMIfPresent(result)
+	bs, _ := ioutil.ReadAll(ToUTF8WithFallbackReader(bytes.NewReader(content)))
+	return bs
 }
 
 // ToUTF8 converts content to UTF8 encoding and ignore error

modules/csv/csv.go

@@ -7,7 +7,9 @@ package csv
 import (
 	"bytes"
 	"encoding/csv"
+	stdcsv "encoding/csv"
 	"errors"
+	"io"
 	"regexp"
 	"strings"
 
@@ -18,17 +20,31 @@ import (
 var quoteRegexp = regexp.MustCompile(`["'][\s\S]+?["']`)
 
 // CreateReader creates a csv.Reader with the given delimiter.
-func CreateReader(rawBytes []byte, delimiter rune) *csv.Reader {
-	rd := csv.NewReader(bytes.NewReader(rawBytes))
+func CreateReader(input io.Reader, delimiter rune) *stdcsv.Reader {
+	rd := stdcsv.NewReader(input)
 	rd.Comma = delimiter
 	rd.TrimLeadingSpace = true
 	return rd
 }
 
 // CreateReaderAndGuessDelimiter tries to guess the field delimiter from the content and creates a csv.Reader.
-func CreateReaderAndGuessDelimiter(rawBytes []byte) *csv.Reader {
-	delimiter := guessDelimiter(rawBytes)
-	return CreateReader(rawBytes, delimiter)
+func CreateReaderAndGuessDelimiter(rd io.Reader) (*stdcsv.Reader, error) {
+	var data = make([]byte, 1e4)
+	size, err := rd.Read(data)
+	if err != nil {
+		return nil, err
+	}
+
+	delimiter := guessDelimiter(data[:size])
+
+	var newInput io.Reader
+	if size < 1e4 {
+		newInput = bytes.NewReader(data[:size])
+	} else {
+		newInput = io.MultiReader(bytes.NewReader(data), rd)
+	}
+
+	return CreateReader(newInput, delimiter), nil
 }
 
 // guessDelimiter scores the input CSV data against delimiters, and returns the best match.

modules/csv/csv_test.go

@@ -5,20 +5,23 @@
 package csv
 
 import (
+	"bytes"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
 func TestCreateReader(t *testing.T) {
-	rd := CreateReader([]byte{}, ',')
+	rd := CreateReader(bytes.NewReader([]byte{}), ',')
 	assert.Equal(t, ',', rd.Comma)
 }
 
 func TestCreateReaderAndGuessDelimiter(t *testing.T) {
 	input := "a;b;c\n1;2;3\n4;5;6"
 
-	rd := CreateReaderAndGuessDelimiter([]byte(input))
+	rd, err := CreateReaderAndGuessDelimiter(strings.NewReader(input))
+	assert.NoError(t, err)
 	assert.Equal(t, ';', rd.Comma)
 }