Ensure syntax highlighting is the same inside diffs (go-gitea#12205)

Make sure to end up with the same syntax highlighting inside various sections of diffs by processing the code first before detecting specific changes between the lines. Also try and make sure that when highlighting individual lines in a diff that it is tokenized the same as it would be when part of an entire file with more context.

Fixes: go-gitea#12190

Author: mrsdizzie

modules/highlight/highlight.go

@@ -14,7 +14,6 @@ import (
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
-	"github.com/alecthomas/chroma"
 	"github.com/alecthomas/chroma/formatters/html"
 	"github.com/alecthomas/chroma/lexers"
 	"github.com/alecthomas/chroma/styles"
@@ -44,9 +43,12 @@ func NewContext() {
 func Code(fileName, code string) string {
 	NewContext()
 
-	if code == "" {
+	// diff view newline will be passed as empty, change to literal \n so it can be copied
+	// preserve literal newline in blame view
+	if code == "" || code == "\n" {
 		return "\n"
 	}
+
 	if len(code) > sizeLimit {
 		return code
 	}
@@ -72,7 +74,7 @@ func Code(fileName, code string) string {
 		lexer = lexers.Fallback
 	}
 
-	iterator, err := lexer.Tokenise(&chroma.TokeniseOptions{State: "root", Nested: true}, string(code))
+	iterator, err := lexer.Tokenise(nil, string(code))
 	if err != nil {
 		log.Error("Can't tokenize code: %v", err)
 		return code
@@ -85,7 +87,9 @@ func Code(fileName, code string) string {
 	}
 
 	htmlw.Flush()
-	return htmlbuf.String()
+	// Chroma will add newlines for certain lexers in order to highlight them properly
+	// Once highlighted, strip them here so they don't cause copy/paste trouble in HTML output
+	return strings.TrimSuffix(htmlbuf.String(), "\n")
 }
 
 // File returns map with line lumbers and HTML version of code with chroma syntax highlighting classes

services/gitdiff/gitdiff.go

@@ -188,14 +188,14 @@ func diffToHTML(fileName string, diffs []diffmatchpatch.Diff, lineType DiffLineT
 		switch {
 		case diffs[i].Type == diffmatchpatch.DiffInsert && lineType == DiffLineAdd:
 			buf.Write(addedCodePrefix)
-			buf.WriteString(highlight.Code(fileName, diffs[i].Text))
+			buf.WriteString(getLineContent(diffs[i].Text))
 			buf.Write(codeTagSuffix)
 		case diffs[i].Type == diffmatchpatch.DiffDelete && lineType == DiffLineDel:
 			buf.Write(removedCodePrefix)
-			buf.WriteString(highlight.Code(fileName, diffs[i].Text))
+			buf.WriteString(getLineContent(diffs[i].Text))
 			buf.Write(codeTagSuffix)
 		case diffs[i].Type == diffmatchpatch.DiffEqual:
-			buf.WriteString(highlight.Code(fileName, getLineContent(diffs[i].Text)))
+			buf.WriteString(getLineContent(diffs[i].Text))
 		}
 	}
 	return template.HTML(buf.Bytes())
@@ -287,7 +287,7 @@ func (diffSection *DiffSection) GetComputedInlineDiffFor(diffLine *DiffLine) tem
 		return template.HTML(highlight.Code(diffSection.FileName, diffLine.Content))
 	}
 
-	diffRecord := diffMatchPatch.DiffMain(diff1[1:], diff2[1:], true)
+	diffRecord := diffMatchPatch.DiffMain(highlight.Code(diffSection.FileName, diff1[1:]), highlight.Code(diffSection.FileName, diff2[1:]), true)
 	diffRecord = diffMatchPatch.DiffCleanupEfficiency(diffRecord)
 	return diffToHTML(diffSection.FileName, diffRecord, diffLine.Type)
 }