Improve optimaziation

Author: lunny

modules/charset/charset.go

@@ -23,32 +23,6 @@ import (
 // UTF8BOM is the utf-8 byte-order marker
 var UTF8BOM = []byte{'\xef', '\xbb', '\xbf'}
 
-// ToUTF8WithErr converts content to UTF8 encoding
-func ToUTF8WithErr(content []byte) (string, error) {
-	charsetLabel, err := DetectEncoding(content)
-	if err != nil {
-		return "", err
-	} else if charsetLabel == "UTF-8" {
-		return string(RemoveBOMIfPresent(content)), nil
-	}
-
-	encoding, _ := charset.Lookup(charsetLabel)
-	if encoding == nil {
-		return string(content), fmt.Errorf("Unknown encoding: %s", charsetLabel)
-	}
-
-	// If there is an error, we concatenate the nicely decoded part and the
-	// original left over. This way we won't lose much data.
-	result, n, err := transform.Bytes(encoding.NewDecoder(), content)
-	if err != nil {
-		result = append(result, content[n:]...)
-	}
-
-	result = RemoveBOMIfPresent(result)
-
-	return string(result), err
-}
-
 // ToUTF8WithFallbackReader detects the encoding of content and coverts to UTF-8 reader if possible
 func ToUTF8WithFallbackReader(rd io.Reader) io.Reader {
 	var buf = make([]byte, 2048)
@@ -76,6 +50,32 @@ func ToUTF8WithFallbackReader(rd io.Reader) io.Reader {
 	)
 }
 
+// ToUTF8WithErr converts content to UTF8 encoding
+func ToUTF8WithErr(content []byte) (string, error) {
+	charsetLabel, err := DetectEncoding(content)
+	if err != nil {
+		return "", err
+	} else if charsetLabel == "UTF-8" {
+		return string(RemoveBOMIfPresent(content)), nil
+	}
+
+	encoding, _ := charset.Lookup(charsetLabel)
+	if encoding == nil {
+		return string(content), fmt.Errorf("Unknown encoding: %s", charsetLabel)
+	}
+
+	// If there is an error, we concatenate the nicely decoded part and the
+	// original left over. This way we won't lose much data.
+	result, n, err := transform.Bytes(encoding.NewDecoder(), content)
+	if err != nil {
+		result = append(result, content[n:]...)
+	}
+
+	result = RemoveBOMIfPresent(result)
+
+	return string(result), err
+}
+
 // ToUTF8WithFallback detects the encoding of content and coverts to UTF-8 if possible
 func ToUTF8WithFallback(content []byte) []byte {
 	bs, _ := ioutil.ReadAll(ToUTF8WithFallbackReader(bytes.NewReader(content)))

routers/repo/lfs.go

@@ -296,20 +296,13 @@ func LFSFileGet(ctx *context.Context) {
 			break
 		}
 
-		d, _ := ioutil.ReadAll(dataRc)
-		buf = charset.ToUTF8WithFallback(append(buf, d...))
+		buf := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc))
 
 		// Building code view blocks with line number on server side.
-		var fileContent string
-		if content, err := charset.ToUTF8WithErr(buf); err != nil {
-			log.Error("ToUTF8WithErr: %v", err)
-			fileContent = string(buf)
-		} else {
-			fileContent = content
-		}
+		fileContent, _ := ioutil.ReadAll(buf)
 
 		var output bytes.Buffer
-		lines := strings.Split(fileContent, "\n")
+		lines := strings.Split(string(fileContent), "\n")
 		//Remove blank line at the end of file
 		if len(lines) > 0 && lines[len(lines)-1] == "" {
 			lines = lines[:len(lines)-1]

routers/repo/view.go

@@ -324,8 +324,7 @@ func renderDirectory(ctx *context.Context, treeLink string) {
 				ctx.Data["IsTextFile"] = true
 				ctx.Data["FileSize"] = fileSize
 			} else {
-				d, _ := ioutil.ReadAll(dataRc)
-				buf = charset.ToUTF8WithFallback(append(buf, d...))
+				rd := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc))
 
 				if markupType := markup.Type(readmeFile.name); markupType != "" {
 					ctx.Data["IsMarkup"] = true
@@ -335,12 +334,14 @@ func renderDirectory(ctx *context.Context, treeLink string) {
 						Filename:  readmeFile.name,
 						URLPrefix: readmeTreelink,
 						Metas:     ctx.Repo.Repository.ComposeDocumentMetas(),
-					}, bytes.NewReader(buf), &result)
+					}, rd, &result)
 					if err != nil {
-						ctx.ServerError("Render", err)
-						return
+						log.Error("Render failed: %v then fallback", err)
+						bs, _ := ioutil.ReadAll(rd)
+						ctx.Data["FileContent"] = string(bs)
+					} else {
+						ctx.Data["FileContent"] = result.String()
 					}
-					ctx.Data["FileContent"] = result.String()
 				} else {
 					ctx.Data["IsRenderedHTML"] = true
 					ctx.Data["FileContent"] = strings.ReplaceAll(