Code/repo search (go-gitea#2582)

Indexed search of repository contents (for default branch only)

Author: ethantkoenig

conf/app.ini

@@ -192,7 +192,11 @@ ITERATE_BUFFER_SIZE = 50
 
 [indexer]
 ISSUE_INDEXER_PATH = indexers/issues.bleve
+; repo indexer by default disabled, since it uses a lot of disk space
+REPO_INDEXER_ENABLED = false
+REPO_INDEXER_PATH = indexers/repos.bleve
 UPDATE_BUFFER_LEN = 20
+MAX_FILE_SIZE = 1048576
 
 [admin]
 ; Disable regular (non-admin) users to create organizations

integrations/integration_test.go

@@ -63,6 +63,10 @@ func TestMain(m *testing.M) {
 		fmt.Printf("os.RemoveAll: %v\n", err)
 		os.Exit(1)
 	}
+	if err = os.RemoveAll(setting.Indexer.RepoPath); err != nil {
+		fmt.Printf("Unable to remove repo indexer: %v\n", err)
+		os.Exit(1)
+	}
 
 	os.Exit(exitCode)
 }

integrations/mysql.ini.tmpl

@@ -12,6 +12,8 @@ PATH     = data/gitea.db
 
 [indexer]
 ISSUE_INDEXER_PATH = integrations/indexers-mysql/issues.bleve
+REPO_INDEXER_ENABLED = true
+REPO_INDEXER_PATH = integrations/indexers-mysql/repos.bleve
 
 [repository]
 ROOT = integrations/gitea-integration-mysql/gitea-repositories

integrations/pgsql.ini.tmpl

@@ -12,6 +12,8 @@ PATH     = data/gitea.db
 
 [indexer]
 ISSUE_INDEXER_PATH = integrations/indexers-pgsql/issues.bleve
+REPO_INDEXER_ENABLED = true
+REPO_INDEXER_PATH = integrations/indexers-pgsql/repos.bleve
 
 [repository]
 ROOT = integrations/gitea-integration-pgsql/gitea-repositories

integrations/repo_search_test.go

@@ -0,0 +1,35 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package integrations
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/PuerkitoBio/goquery"
+	"github.com/stretchr/testify/assert"
+)
+
+func resultFilenames(t testing.TB, doc *HTMLDoc) []string {
+	resultsSelection := doc.doc.Find(".repository.search")
+	assert.EqualValues(t, 1, resultsSelection.Length(),
+		"Invalid template (repo search template has changed?)")
+	filenameSelections := resultsSelection.Find(".repo-search-result").Find(".header").Find("span.file")
+	result := make([]string, filenameSelections.Length())
+	filenameSelections.Each(func(i int, selection *goquery.Selection) {
+		result[i] = selection.Text()
+	})
+	return result
+}
+
+func TestSearchRepo(t *testing.T) {
+	prepareTestEnv(t)
+
+	req := NewRequestf(t, "GET", "/user2/repo1/search?q=Description&page=1")
+	resp := MakeRequest(t, req, http.StatusOK)
+
+	filenames := resultFilenames(t, NewHTMLParser(t, resp.Body))
+	assert.EqualValues(t, []string{"README.md"}, filenames)
+}

integrations/sqlite.ini

@@ -7,6 +7,8 @@ PATH     = :memory:
 
 [indexer]
 ISSUE_INDEXER_PATH = integrations/indexers-sqlite/issues.bleve
+REPO_INDEXER_ENABLED = true
+REPO_INDEXER_PATH = integrations/indexers-sqlite/repos.bleve
 
 [repository]
 ROOT = integrations/gitea-integration-sqlite/gitea-repositories

models/fixtures/repo_indexer_status.yml

@@ -0,0 +1 @@
+[] # empty

models/migrations/migrations.go

@@ -144,6 +144,8 @@ var migrations = []Migration{
 	NewMigration("remove organization watch repositories", removeOrganizationWatchRepo),
 	// v47 -> v48
 	NewMigration("add deleted branches", addDeletedBranch),
+	// v48 -> v49
+	NewMigration("add repo indexer status", addRepoIndexerStatus),
 }
 
 // Migrate database to current version

models/migrations/v48.go

@@ -0,0 +1,25 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"fmt"
+
+	"github.com/go-xorm/xorm"
+)
+
+func addRepoIndexerStatus(x *xorm.Engine) error {
+	// RepoIndexerStatus see models/repo_indexer.go
+	type RepoIndexerStatus struct {
+		ID        int64  `xorm:"pk autoincr"`
+		RepoID    int64  `xorm:"INDEX NOT NULL"`
+		CommitSha string `xorm:"VARCHAR(40)"`
+	}
+
+	if err := x.Sync2(new(RepoIndexerStatus)); err != nil {
+		return fmt.Errorf("Sync2: %v", err)
+	}
+	return nil
+}

models/models.go

@@ -13,6 +13,10 @@ import (
 	"path"
 	"strings"
 
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/util"
+
 	// Needed for the MySQL driver
 	_ "github.com/go-sql-driver/mysql"
 	"github.com/go-xorm/core"
@@ -23,9 +27,6 @@ import (
 
 	// Needed for the MSSSQL driver
 	_ "github.com/denisenkom/go-mssqldb"
-
-	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/setting"
 )
 
 // Engine represents a xorm engine or session.
@@ -115,6 +116,7 @@ func init() {
 		new(Stopwatch),
 		new(TrackedTime),
 		new(DeletedBranch),
+		new(RepoIndexerStatus),
 	)
 
 	gonicNames := []string{"SSL", "UID"}
@@ -150,8 +152,13 @@ func LoadConfigs() {
 	DbCfg.Timeout = sec.Key("SQLITE_TIMEOUT").MustInt(500)
 
 	sec = setting.Cfg.Section("indexer")
-	setting.Indexer.IssuePath = sec.Key("ISSUE_INDEXER_PATH").MustString("indexers/issues.bleve")
+	setting.Indexer.IssuePath = absolutePath(
+		sec.Key("ISSUE_INDEXER_PATH").MustString("indexers/issues.bleve"))
+	setting.Indexer.RepoIndexerEnabled = sec.Key("REPO_INDEXER_ENABLED").MustBool(false)
+	setting.Indexer.RepoPath = absolutePath(
+		sec.Key("REPO_INDEXER_PATH").MustString("indexers/repos.bleve"))
 	setting.Indexer.UpdateQueueLength = sec.Key("UPDATE_BUFFER_LEN").MustInt(20)
+	setting.Indexer.MaxIndexerFileSize = sec.Key("MAX_FILE_SIZE").MustInt64(512 * 1024 * 1024)
 }
 
 // parsePostgreSQLHostPort parses given input in various forms defined in
@@ -336,3 +343,12 @@ func DumpDatabase(filePath string, dbType string) error {
 	}
 	return x.DumpTablesToFile(tbs, filePath)
 }
+
+// absolutePath make path absolute if it is relative
+func absolutePath(path string) string {
+	workDir, err := setting.WorkDir()
+	if err != nil {
+		log.Fatal(4, "Failed to get work directory: %v", err)
+	}
+	return util.EnsureAbsolutePath(path, workDir)
+}

models/repo.go

@@ -205,10 +205,11 @@ type Repository struct {
 	ExternalMetas map[string]string `xorm:"-"`
 	Units         []*RepoUnit       `xorm:"-"`
 
-	IsFork   bool        `xorm:"INDEX NOT NULL DEFAULT false"`
-	ForkID   int64       `xorm:"INDEX"`
-	BaseRepo *Repository `xorm:"-"`
-	Size     int64       `xorm:"NOT NULL DEFAULT 0"`
+	IsFork        bool               `xorm:"INDEX NOT NULL DEFAULT false"`
+	ForkID        int64              `xorm:"INDEX"`
+	BaseRepo      *Repository        `xorm:"-"`
+	Size          int64              `xorm:"NOT NULL DEFAULT 0"`
+	IndexerStatus *RepoIndexerStatus `xorm:"-"`
 
 	Created     time.Time `xorm:"-"`
 	CreatedUnix int64     `xorm:"INDEX created"`
@@ -782,8 +783,10 @@ func UpdateLocalCopyBranch(repoPath, localPath, branch string) error {
 		if err != nil {
 			return fmt.Errorf("git fetch origin: %v", err)
 		}
-		if err := git.ResetHEAD(localPath, true, "origin/"+branch); err != nil {
-			return fmt.Errorf("git reset --hard origin/%s: %v", branch, err)
+		if len(branch) > 0 {
+			if err := git.ResetHEAD(localPath, true, "origin/"+branch); err != nil {
+				return fmt.Errorf("git reset --hard origin/%s: %v", branch, err)
+			}
 		}
 	}
 	return nil
@@ -989,6 +992,7 @@ func MigrateRepository(doer, u *User, opts MigrateRepoOptions) (*Repository, err
 		if err = SyncReleasesWithTags(repo, gitRepo); err != nil {
 			log.Error(4, "Failed to synchronize tags to releases for repository: %v", err)
 		}
+		UpdateRepoIndexer(repo)
 	}
 
 	if err = repo.UpdateSize(); err != nil {
@@ -1883,6 +1887,7 @@ func DeleteRepository(doer *User, uid, repoID int64) error {
 		go HookQueue.Add(repo.ID)
 	}
 
+	DeleteRepoFromIndexer(repo)
 	return nil
 }
 

models/repo_editor.go

@@ -178,6 +178,8 @@ func (repo *Repository) UpdateRepoFile(doer *User, opts UpdateRepoFileOptions) (
 	if err != nil {
 		return fmt.Errorf("PushUpdate: %v", err)
 	}
+	UpdateRepoIndexer(repo)
+
 	return nil
 }