Merge remote-tracking branch 'upstream/main'

Author: HEREYUA

.devcontainer/devcontainer.json

@@ -8,7 +8,9 @@
     },
     "ghcr.io/devcontainers/features/git-lfs:1.1.0": {},
     "ghcr.io/devcontainers-contrib/features/poetry:2": {},
-    "ghcr.io/devcontainers/features/python:1": {}
+    "ghcr.io/devcontainers/features/python:1": {
+      "version": "3.12"
+    }
   },
   "customizations": {
     "vscode": {

docs/content/administration/config-cheat-sheet.en-us.md

@@ -832,7 +832,7 @@ Default templates for project boards:
 ## Issue and pull request attachments (`attachment`)
 
 - `ENABLED`: **true**: Whether issue and pull request attachments are enabled.
-- `ALLOWED_TYPES`: **.csv,.docx,.fodg,.fodp,.fods,.fodt,.gif,.gz,.jpeg,.jpg,.log,.md,.mov,.mp4,.odf,.odg,.odp,.ods,.odt,.patch,.pdf,.png,.pptx,.svg,.tgz,.txt,.webm,.xls,.xlsx,.zip**: Comma-separated list of allowed file extensions (`.zip`), mime types (`text/plain`) or wildcard type (`image/*`, `audio/*`, `video/*`). Empty value or `*/*` allows all types.
+- `ALLOWED_TYPES`: **.cpuprofile,.csv,.dmp,.docx,.fodg,.fodp,.fods,.fodt,.gif,.gz,.jpeg,.jpg,.json,.jsonc,.log,.md,.mov,.mp4,.odf,.odg,.odp,.ods,.odt,.patch,.pdf,.png,.pptx,.svg,.tgz,.txt,.webm,.xls,.xlsx,.zip**: Comma-separated list of allowed file extensions (`.zip`), mime types (`text/plain`) or wildcard type (`image/*`, `audio/*`, `video/*`). Empty value or `*/*` allows all types.
 - `MAX_SIZE`: **2048**: Maximum size (MB).
 - `MAX_FILES`: **5**: Maximum number of attachments that can be uploaded at once.
 - `STORAGE_TYPE`: **local**: Storage type for attachments, `local` for local disk or `minio` for s3 compatible object storage service, default is `local` or other name defined with `[storage.xxx]`

docs/content/administration/config-cheat-sheet.zh-cn.md

@@ -782,7 +782,7 @@ Gitea 创建以下非唯一队列：
 ## 工单和合并请求的附件 (`attachment`)
 
 - `ENABLED`: **true**: 是否允许用户上传附件。
-- `ALLOWED_TYPES`: **.csv,.docx,.fodg,.fodp,.fods,.fodt,.gif,.gz,.jpeg,.jpg,.log,.md,.mov,.mp4,.odf,.odg,.odp,.ods,.odt,.patch,.pdf,.png,.pptx,.svg,.tgz,.txt,.webm,.xls,.xlsx,.zip**: 允许的文件扩展名（`.zip`）、mime 类型（`text/plain`）或通配符类型（`image/*`、`audio/*`、`video/*`）的逗号分隔列表。空值或 `*/*` 允许所有类型。
+- `ALLOWED_TYPES`: **.cpuprofile,.csv,.dmp,.docx,.fodg,.fodp,.fods,.fodt,.gif,.gz,.jpeg,.jpg,.json,.jsonc,.log,.md,.mov,.mp4,.odf,.odg,.odp,.ods,.odt,.patch,.pdf,.png,.pptx,.svg,.tgz,.txt,.webm,.xls,.xlsx,.zip**: 允许的文件扩展名（`.zip`）、mime 类型（`text/plain`）或通配符类型（`image/*`、`audio/*`、`video/*`）的逗号分隔列表。空值或 `*/*` 允许所有类型。
 - `MAX_SIZE`: **2048**: 附件的最大限制（MB）。
 - `MAX_FILES`: **5**: 一次最多上传的附件数量。
 - `STORAGE_TYPE`: **local**: 附件的存储类型，`local` 表示本地磁盘，`minio` 表示兼容 S3 的对象存储服务，如果未设置将使用默认值 `local` 或其他在 `[storage.xxx]` 中定义的名称。

modules/indexer/code/bleve/bleve.go

@@ -233,21 +233,21 @@ func (b *Indexer) Delete(_ context.Context, repoID int64) error {
 
 // Search searches for files in the specified repo.
 // Returns the matching file-paths
-func (b *Indexer) Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
+func (b *Indexer) Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isFuzzy bool) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
 	var (
 		indexerQuery query.Query
 		keywordQuery query.Query
 	)
 
-	if isMatch {
-		prefixQuery := bleve.NewPrefixQuery(keyword)
-		prefixQuery.FieldVal = "Content"
-		keywordQuery = prefixQuery
-	} else {
+	if isFuzzy {
 		phraseQuery := bleve.NewMatchPhraseQuery(keyword)
 		phraseQuery.FieldVal = "Content"
 		phraseQuery.Analyzer = repoIndexerAnalyzer
 		keywordQuery = phraseQuery
+	} else {
+		prefixQuery := bleve.NewPrefixQuery(keyword)
+		prefixQuery.FieldVal = "Content"
+		keywordQuery = prefixQuery
 	}
 
 	if len(repoIDs) > 0 {

modules/indexer/code/elasticsearch/elasticsearch.go

@@ -281,10 +281,10 @@ func extractAggs(searchResult *elastic.SearchResult) []*internal.SearchResultLan
 }
 
 // Search searches for codes and language stats by given conditions.
-func (b *Indexer) Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
-	searchType := esMultiMatchTypeBestFields
-	if isMatch {
-		searchType = esMultiMatchTypePhrasePrefix
+func (b *Indexer) Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isFuzzy bool) (int64, []*internal.SearchResult, []*internal.SearchResultLanguages, error) {
+	searchType := esMultiMatchTypePhrasePrefix
+	if isFuzzy {
+		searchType = esMultiMatchTypeBestFields
 	}
 
 	kwQuery := elastic.NewMultiMatchQuery(keyword, "content").Type(searchType)

modules/indexer/code/git.go

@@ -91,11 +91,9 @@ func genesisChanges(ctx context.Context, repo *repo_model.Repository, revision s
 		return nil, runErr
 	}
 
+	objectFormat := git.ObjectFormatFromName(repo.ObjectFormatName)
+
 	var err error
-	objectFormat, err := git.GetObjectFormatOfRepo(ctx, repo.RepoPath())
-	if err != nil {
-		return nil, err
-	}
 	changes.Updates, err = parseGitLsTreeOutput(objectFormat, stdout)
 	return &changes, err
 }
@@ -174,10 +172,8 @@ func nonGenesisChanges(ctx context.Context, repo *repo_model.Repository, revisio
 		return nil, err
 	}
 
-	objectFormat, err := git.GetObjectFormatOfRepo(ctx, repo.RepoPath())
-	if err != nil {
-		return nil, err
-	}
+	objectFormat := git.ObjectFormatFromName(repo.ObjectFormatName)
+
 	changes.Updates, err = parseGitLsTreeOutput(objectFormat, lsTreeStdout)
 	return &changes, err
 }

modules/indexer/code/indexer_test.go

@@ -70,7 +70,7 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) {
 
 		for _, kw := range keywords {
 			t.Run(kw.Keyword, func(t *testing.T) {
-				total, res, langs, err := indexer.Search(context.TODO(), kw.RepoIDs, "", kw.Keyword, 1, 10, false)
+				total, res, langs, err := indexer.Search(context.TODO(), kw.RepoIDs, "", kw.Keyword, 1, 10, true)
 				assert.NoError(t, err)
 				assert.Len(t, kw.IDs, int(total))
 				assert.Len(t, langs, kw.Langs)

modules/indexer/code/internal/indexer.go

@@ -16,7 +16,7 @@ type Indexer interface {
 	internal.Indexer
 	Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *RepoChanges) error
 	Delete(ctx context.Context, repoID int64) error
-	Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error)
+	Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isFuzzy bool) (int64, []*SearchResult, []*SearchResultLanguages, error)
 }
 
 // NewDummyIndexer returns a dummy indexer
@@ -38,6 +38,6 @@ func (d *dummyIndexer) Delete(ctx context.Context, repoID int64) error {
 	return fmt.Errorf("indexer is not ready")
 }
 
-func (d *dummyIndexer) Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
+func (d *dummyIndexer) Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isFuzzy bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
 	return 0, nil, nil, fmt.Errorf("indexer is not ready")
 }

modules/indexer/code/search.go

@@ -124,12 +124,13 @@ func searchResult(result *internal.SearchResult, startIndex, endIndex int) (*Res
 }
 
 // PerformSearch perform a search on a repository
-func PerformSearch(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int, []*Result, []*internal.SearchResultLanguages, error) {
+// if isFuzzy is true set the Damerau-Levenshtein distance from 0 to 2
+func PerformSearch(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isFuzzy bool) (int, []*Result, []*internal.SearchResultLanguages, error) {
 	if len(keyword) == 0 {
 		return 0, nil, nil, nil
 	}
 
-	total, results, resultLanguages, err := (*globalIndexer.Load()).Search(ctx, repoIDs, language, keyword, page, pageSize, isMatch)
+	total, results, resultLanguages, err := (*globalIndexer.Load()).Search(ctx, repoIDs, language, keyword, page, pageSize, isFuzzy)
 	if err != nil {
 		return 0, nil, nil, err
 	}

modules/indexer/internal/bleve/query.go

@@ -25,6 +25,13 @@ func MatchPhraseQuery(matchPhrase, field, analyzer string) *query.MatchPhraseQue
 	return q
 }
 
+// PrefixQuery generates a match prefix query for the given prefix and field
+func PrefixQuery(matchPrefix, field string) *query.PrefixQuery {
+	q := bleve.NewPrefixQuery(matchPrefix)
+	q.FieldVal = field
+	return q
+}
+
 // BoolFieldQuery generates a bool field query for the given value and field
 func BoolFieldQuery(value bool, field string) *query.BoolFieldQuery {
 	q := bleve.NewBoolFieldQuery(value)

modules/indexer/issues/bleve/bleve.go

@@ -156,12 +156,19 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
 	var queries []query.Query
 
 	if options.Keyword != "" {
-		keywordQueries := []query.Query{
-			inner_bleve.MatchPhraseQuery(options.Keyword, "title", issueIndexerAnalyzer),
-			inner_bleve.MatchPhraseQuery(options.Keyword, "content", issueIndexerAnalyzer),
-			inner_bleve.MatchPhraseQuery(options.Keyword, "comments", issueIndexerAnalyzer),
+		if options.IsFuzzyKeyword {
+			queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{
+				inner_bleve.MatchPhraseQuery(options.Keyword, "title", issueIndexerAnalyzer),
+				inner_bleve.MatchPhraseQuery(options.Keyword, "content", issueIndexerAnalyzer),
+				inner_bleve.MatchPhraseQuery(options.Keyword, "comments", issueIndexerAnalyzer),
+			}...))
+		} else {
+			queries = append(queries, bleve.NewDisjunctionQuery([]query.Query{
+				inner_bleve.PrefixQuery(options.Keyword, "title"),
+				inner_bleve.PrefixQuery(options.Keyword, "content"),
+				inner_bleve.PrefixQuery(options.Keyword, "comments"),
+			}...))
 		}
-		queries = append(queries, bleve.NewDisjunctionQuery(keywordQueries...))
 	}
 
 	if len(options.RepoIDs) > 0 || options.AllPublic {

modules/indexer/issues/elasticsearch/elasticsearch.go

@@ -19,6 +19,10 @@ import (
 
 const (
 	issueIndexerLatestVersion = 1
+	// multi-match-types, currently only 2 types are used
+	// Reference: https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-multi-match-query.html#multi-match-types
+	esMultiMatchTypeBestFields   = "best_fields"
+	esMultiMatchTypePhrasePrefix = "phrase_prefix"
 )
 
 var _ internal.Indexer = &Indexer{}
@@ -141,7 +145,13 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
 	query := elastic.NewBoolQuery()
 
 	if options.Keyword != "" {
-		query.Must(elastic.NewMultiMatchQuery(options.Keyword, "title", "content", "comments"))
+
+		searchType := esMultiMatchTypePhrasePrefix
+		if options.IsFuzzyKeyword {
+			searchType = esMultiMatchTypeBestFields
+		}
+
+		query.Must(elastic.NewMultiMatchQuery(options.Keyword, "title", "content", "comments").Type(searchType))
 	}
 
 	if len(options.RepoIDs) > 0 {

modules/indexer/issues/internal/model.go

@@ -74,6 +74,8 @@ type SearchResult struct {
 type SearchOptions struct {
 	Keyword string // keyword to search
 
+	IsFuzzyKeyword bool // if false the levenshtein distance is 0
+
 	RepoIDs   []int64 // repository IDs which the issues belong to
 	AllPublic bool    // if include all public repositories
 

modules/indexer/issues/meilisearch/meilisearch.go

@@ -5,6 +5,7 @@ package meilisearch
 
 import (
 	"context"
+	"errors"
 	"strconv"
 	"strings"
 
@@ -16,12 +17,15 @@ import (
 )
 
 const (
-	issueIndexerLatestVersion = 2
+	issueIndexerLatestVersion = 3
 
 	// TODO: make this configurable if necessary
 	maxTotalHits = 10000
 )
 
+// ErrMalformedResponse is never expected as we initialize the indexer ourself and so define the types.
+var ErrMalformedResponse = errors.New("meilisearch returned unexpected malformed content")
+
 var _ internal.Indexer = &Indexer{}
 
 // Indexer implements Indexer interface
@@ -47,6 +51,9 @@ func NewIndexer(url, apiKey, indexerName string) *Indexer {
 		},
 		DisplayedAttributes: []string{
 			"id",
+			"title",
+			"content",
+			"comments",
 		},
 		FilterableAttributes: []string{
 			"repo_id",
@@ -221,11 +228,9 @@ func (b *Indexer) Search(ctx context.Context, options *internal.SearchOptions) (
 		return nil, err
 	}
 
-	hits := make([]internal.Match, 0, len(searchRes.Hits))
-	for _, hit := range searchRes.Hits {
-		hits = append(hits, internal.Match{
-			ID: int64(hit.(map[string]any)["id"].(float64)),
-		})
+	hits, err := nonFuzzyWorkaround(searchRes, options.Keyword, options.IsFuzzyKeyword)
+	if err != nil {
+		return nil, err
 	}
 
 	return &internal.SearchResult{
@@ -241,3 +246,77 @@ func parseSortBy(sortBy internal.SortBy) string {
 	}
 	return field + ":asc"
 }
+
+// nonFuzzyWorkaround is needed as meilisearch does not have an exact search
+// and you can only change "typo tolerance" per index. So we have to post-filter the results
+// https://www.meilisearch.com/docs/learn/configuration/typo_tolerance#configuring-typo-tolerance
+// TODO: remove once https://github.com/orgs/meilisearch/discussions/377 is addressed
+func nonFuzzyWorkaround(searchRes *meilisearch.SearchResponse, keyword string, isFuzzy bool) ([]internal.Match, error) {
+	hits := make([]internal.Match, 0, len(searchRes.Hits))
+	for _, hit := range searchRes.Hits {
+		hit, ok := hit.(map[string]any)
+		if !ok {
+			return nil, ErrMalformedResponse
+		}
+
+		if !isFuzzy {
+			keyword = strings.ToLower(keyword)
+
+			// declare a anon func to check if the title, content or at least one comment contains the keyword
+			found, err := func() (bool, error) {
+				// check if title match first
+				title, ok := hit["title"].(string)
+				if !ok {
+					return false, ErrMalformedResponse
+				} else if strings.Contains(strings.ToLower(title), keyword) {
+					return true, nil
+				}
+
+				// check if content has a match
+				content, ok := hit["content"].(string)
+				if !ok {
+					return false, ErrMalformedResponse
+				} else if strings.Contains(strings.ToLower(content), keyword) {
+					return true, nil
+				}
+
+				// now check for each comment if one has a match
+				// so we first try to cast and skip if there are no comments
+				comments, ok := hit["comments"].([]any)
+				if !ok {
+					return false, ErrMalformedResponse
+				} else if len(comments) == 0 {
+					return false, nil
+				}
+
+				// now we iterate over all and report as soon as we detect one match
+				for i := range comments {
+					comment, ok := comments[i].(string)
+					if !ok {
+						return false, ErrMalformedResponse
+					}
+					if strings.Contains(strings.ToLower(comment), keyword) {
+						return true, nil
+					}
+				}
+
+				// we got no match
+				return false, nil
+			}()
+
+			if err != nil {
+				return nil, err
+			} else if !found {
+				continue
+			}
+		}
+		issueID, ok := hit["id"].(float64)
+		if !ok {
+			return nil, ErrMalformedResponse
+		}
+		hits = append(hits, internal.Match{
+			ID: int64(issueID),
+		})
+	}
+	return hits, nil
+}

modules/indexer/issues/meilisearch/meilisearch_test.go

@@ -10,7 +10,11 @@ import (
 	"testing"
 	"time"
 
+	"code.gitea.io/gitea/modules/indexer/issues/internal"
 	"code.gitea.io/gitea/modules/indexer/issues/internal/tests"
+
+	"github.com/meilisearch/meilisearch-go"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestMeilisearchIndexer(t *testing.T) {
@@ -48,3 +52,44 @@ func TestMeilisearchIndexer(t *testing.T) {
 
 	tests.TestIndexer(t, indexer)
 }
+
+func TestNonFuzzyWorkaround(t *testing.T) {
+	// get unexpected return
+	_, err := nonFuzzyWorkaround(&meilisearch.SearchResponse{
+		Hits: []any{"aa", "bb", "cc", "dd"},
+	}, "bowling", false)
+	assert.ErrorIs(t, err, ErrMalformedResponse)
+
+	validResponse := &meilisearch.SearchResponse{
+		Hits: []any{
+			map[string]any{
+				"id":       float64(11),
+				"title":    "a title",
+				"content":  "issue body with no match",
+				"comments": []any{"hey whats up?", "I'm currently bowling", "nice"},
+			},
+			map[string]any{
+				"id":       float64(22),
+				"title":    "Bowling as title",
+				"content":  "",
+				"comments": []any{},
+			},
+			map[string]any{
+				"id":       float64(33),
+				"title":    "Bowl-ing as fuzzy match",
+				"content":  "",
+				"comments": []any{},
+			},
+		},
+	}
+
+	// nonFuzzy
+	hits, err := nonFuzzyWorkaround(validResponse, "bowling", false)
+	assert.NoError(t, err)
+	assert.EqualValues(t, []internal.Match{{ID: 11}, {ID: 22}}, hits)
+
+	// fuzzy
+	hits, err = nonFuzzyWorkaround(validResponse, "bowling", true)
+	assert.NoError(t, err)
+	assert.EqualValues(t, []internal.Match{{ID: 11}, {ID: 22}, {ID: 33}}, hits)
+}