Issue search support elasticsearch

Author: lunny

.drone.yml

@@ -86,6 +86,12 @@ services:
     pull: default
     image: gitea/test-openldap:latest
 
+  - name: elasticsearch
+    pull: default
+    environment:
+      discovery.type: single-node
+    image: elasticsearch:7.5.0
+
 steps:
   - name: fetch-tags
     pull: default

custom/conf/app.ini.sample

@@ -368,8 +368,10 @@ CONN_MAX_LIFETIME = 3s
 MAX_OPEN_CONNS = 0
 
 [indexer]
-; Issue indexer type, currently support: bleve or db, default is bleve
+; Issue indexer type, currently support: bleve, db or elasticsearch, default is bleve
 ISSUE_INDEXER_TYPE = bleve
+; Issue indexer connection string, available when ISSUE_INDEXER_TYPE is elasticsearch
+ISSUE_INDEXER_CONN_STR = http://elastic:changeme@localhost:9200
 ; Issue indexer storage path, available when ISSUE_INDEXER_TYPE is bleve
 ISSUE_INDEXER_PATH = indexers/issues.bleve
 ; Issue indexer queue, currently support: channel, levelqueue or redis, default is levelqueue

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -228,7 +228,8 @@ relation to port exhaustion.
 
 ## Indexer (`indexer`)
 
-- `ISSUE_INDEXER_TYPE`: **bleve**: Issue indexer type, currently support: bleve or db, if it's db, below issue indexer item will be invalid.
+- `ISSUE_INDEXER_TYPE`: **bleve**: Issue indexer type, currently support: bleve, db or elasticsearch, if it's db, below issue indexer item will be invalid.
+- `ISSUE_INDEXER_CONN_STR`: ****: Issue indexer connection string, available when ISSUE_INDEXER_TYPE is elasticsearch. i.e. http://elastic:changeme@localhost:9200
 - `ISSUE_INDEXER_PATH`: **indexers/issues.bleve**: Index file used for issue search.
 - The next 4 configuration values are deprecated and should be set in `queue.issue_indexer` however are kept for backwards compatibility:
 - `ISSUE_INDEXER_QUEUE_TYPE`: **levelqueue**: Issue indexer queue, currently supports:`channel`, `levelqueue`, `redis`.

go.mod

@@ -74,6 +74,7 @@ require (
 	github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5
 	github.com/niklasfasching/go-org v0.1.9
 	github.com/oliamb/cutter v0.2.2
+	github.com/olivere/elastic/v7 v7.0.9
 	github.com/pkg/errors v0.8.1
 	github.com/pquerna/otp v0.0.0-20160912161815-54653902c20e
 	github.com/prometheus/client_golang v1.1.0

go.sum

@@ -68,6 +68,7 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
+github.com/aws/aws-sdk-go v1.25.25/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -154,6 +155,8 @@ github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 h1:E2s37DuLxFhQD
 github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
@@ -248,6 +251,7 @@ github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZ
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
@@ -322,6 +326,7 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOl
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
 github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U=
 github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ=
 github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
@@ -411,13 +416,16 @@ github.com/niklasfasching/go-org v0.1.9/go.mod h1:AsLD6X7djzRIz4/RFZu8vwRL0VGjUv
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/oliamb/cutter v0.2.2 h1:Lfwkya0HHNU1YLnGv2hTkzHfasrSMkgv4Dn+5rmlk3k=
 github.com/oliamb/cutter v0.2.2/go.mod h1:4BenG2/4GuRBDbVm/OPahDVqbrOemzpPiG5mi1iryBU=
+github.com/olivere/elastic/v7 v7.0.9 h1:+bTR1xJbfLYD8WnTBt9672mFlKxjfWRJpEQ1y8BMS3g=
+github.com/olivere/elastic/v7 v7.0.9/go.mod h1:2TeRd0vhLRTK9zqm5xP0uLiVeZ5yUoL7kZ+8SZA9r9Y=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
 github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
@@ -486,6 +494,7 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
 github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w=
 github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
+github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
 github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s=
 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8=
@@ -573,6 +582,7 @@ go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qL
 go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=

integrations/mysql.ini.tmpl

@@ -10,6 +10,8 @@ PASSWD   = {{TEST_MYSQL_PASSWORD}}
 SSL_MODE = disable
 
 [indexer]
+ISSUE_INDEXER_TYPE = elasticsearch
+ISSUE_INDEXER_CONN_STR = http://elastic:changeme@elasticsearch:9200
 ISSUE_INDEXER_PATH = integrations/indexers-mysql/issues.bleve
 REPO_INDEXER_ENABLED = true
 REPO_INDEXER_PATH = integrations/indexers-mysql/repos.bleve

modules/indexer/issues/elastic_search.go

@@ -0,0 +1,233 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package issues
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strconv"
+	"time"
+
+	"code.gitea.io/gitea/modules/log"
+
+	"github.com/olivere/elastic/v7"
+)
+
+var (
+	_ Indexer = &ElasticSearchIndexer{}
+)
+
+// ElasticSearchIndexer implements Indexer interface
+type ElasticSearchIndexer struct {
+	client      *elastic.Client
+	indexerName string
+	typeName    string
+}
+
+type elasticLogger struct {
+	*log.Logger
+}
+
+func (l elasticLogger) Printf(format string, args ...interface{}) {
+	l.Logger.Log(2, l.Logger.GetLevel(), format, args...)
+}
+
+// NewElasticSearchIndexer creates a new elasticsearch indexer
+func NewElasticSearchIndexer(url, indexerName string) (*ElasticSearchIndexer, error) {
+	opts := []elastic.ClientOptionFunc{
+		elastic.SetURL(url),
+		elastic.SetSniff(false),
+		elastic.SetHealthcheckInterval(10 * time.Second),
+		elastic.SetGzip(false),
+	}
+
+	logger := elasticLogger{log.GetLogger(log.DEFAULT)}
+
+	if logger.GetLevel() == log.TRACE || logger.GetLevel() == log.DEBUG {
+		opts = append(opts, elastic.SetTraceLog(logger))
+	} else if logger.GetLevel() == log.ERROR || logger.GetLevel() == log.CRITICAL || logger.GetLevel() == log.FATAL {
+		opts = append(opts, elastic.SetErrorLog(logger))
+	} else if logger.GetLevel() == log.INFO || logger.GetLevel() == log.WARN {
+		opts = append(opts, elastic.SetInfoLog(logger))
+	}
+
+	client, err := elastic.NewClient(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	return &ElasticSearchIndexer{
+		client:      client,
+		indexerName: indexerName,
+		typeName:    "indexer_data",
+	}, nil
+}
+
+const (
+	defaultMapping = `{
+		"mappings": {
+			"properties": {
+				"id": {
+					"type": "integer",
+					"index": true
+				},
+				"repo_id": {
+					"type": "integer",
+					"index": true
+				},
+				"title": {
+					"type": "text",
+					"index": true
+				},
+				"content": {
+					"type": "text",
+					"index": true
+				},
+				"comments": {
+					"type" : "text",
+					"index": true
+				}
+			}
+		}
+	}`
+)
+
+// Init will initial the indexer
+func (b *ElasticSearchIndexer) Init() (bool, error) {
+	ctx := context.Background()
+	exists, err := b.client.IndexExists(b.indexerName).Do(ctx)
+	if err != nil {
+		return false, err
+	}
+
+	if !exists {
+		var mapping = defaultMapping
+
+		createIndex, err := b.client.CreateIndex(b.indexerName).BodyString(mapping).Do(ctx)
+		if err != nil {
+			return false, err
+		}
+		if !createIndex.Acknowledged {
+			return false, errors.New("init failed")
+		}
+
+		return false, nil
+	}
+	return true, nil
+}
+
+// Index will save the index data
+func (b *ElasticSearchIndexer) Index(issues []*IndexerData) error {
+	if len(issues) == 0 {
+		return nil
+	} else if len(issues) == 1 {
+		issue := issues[0]
+		_, err := b.client.Index().
+			Index(b.indexerName).
+			Id(fmt.Sprintf("%d", issue.ID)).
+			BodyJson(map[string]interface{}{
+				"id":       issue.ID,
+				"repo_id":  issue.RepoID,
+				"title":    issue.Title,
+				"content":  issue.Content,
+				"comments": issue.Comments,
+			}).
+			Do(context.Background())
+		return err
+	}
+
+	reqs := make([]elastic.BulkableRequest, 0)
+	for _, issue := range issues {
+		reqs = append(reqs,
+			elastic.NewBulkIndexRequest().
+				Index(b.indexerName).
+				//Type(b.typeName).
+				Id(fmt.Sprintf("%d", issue.ID)).
+				Doc(map[string]interface{}{
+					"id":       issue.ID,
+					"repo_id":  issue.RepoID,
+					"title":    issue.Title,
+					"content":  issue.Content,
+					"comments": issue.Comments,
+				}),
+		)
+	}
+
+	_, err := b.client.Bulk().
+		Index(b.indexerName).
+		Add(reqs...).
+		Do(context.Background())
+	return err
+}
+
+// Delete deletes indexes by ids
+func (b *ElasticSearchIndexer) Delete(ids ...int64) error {
+	if len(ids) == 0 {
+		return nil
+	} else if len(ids) == 1 {
+		_, err := b.client.Delete().
+			Index(b.indexerName).
+			Id(fmt.Sprintf("%d", ids[0])).
+			Do(context.Background())
+		return err
+	}
+
+	reqs := make([]elastic.BulkableRequest, 0)
+	for _, id := range ids {
+		reqs = append(reqs,
+			elastic.NewBulkDeleteRequest().
+				Index(b.indexerName).
+				Id(fmt.Sprintf("%d", id)),
+		)
+	}
+
+	_, err := b.client.Bulk().
+		Index(b.indexerName).
+		Add(reqs...).
+		Do(context.Background())
+	return err
+}
+
+// Search searches for issues by given conditions.
+// Returns the matching issue IDs
+func (b *ElasticSearchIndexer) Search(keyword string, repoIDs []int64, limit, start int) (*SearchResult, error) {
+	kwQuery := elastic.NewMultiMatchQuery(keyword, "title", "content", "comments")
+	query := elastic.NewBoolQuery()
+	query = query.Must(kwQuery)
+	if len(repoIDs) > 0 {
+		var repoStrs = make([]interface{}, 0, len(repoIDs))
+		for _, repoID := range repoIDs {
+			repoStrs = append(repoStrs, repoID)
+		}
+		repoQuery := elastic.NewTermsQuery("repo_id", repoStrs...)
+		query = query.Must(repoQuery)
+	}
+	searchResult, err := b.client.Search().
+		Index(b.indexerName).
+		Query(query).
+		Sort("id", true).
+		From(start).Size(limit).
+		Do(context.Background())
+	if err != nil {
+		return nil, err
+	}
+
+	hits := make([]Match, 0, limit)
+	for _, hit := range searchResult.Hits.Hits {
+		id, _ := strconv.ParseInt(hit.Id, 10, 64)
+		hits = append(hits, Match{
+			ID: id,
+		})
+	}
+
+	return &SearchResult{
+		Total: searchResult.TotalHits(),
+		Hits:  hits,
+	}, nil
+}
+
+// Close implements indexer
+func (b *ElasticSearchIndexer) Close() {}