Merge branch 'main' into ava-cleanup

Author: GiteaBot

.github/workflows/pull-db-tests.yml

@@ -102,6 +102,13 @@ jobs:
           --health-retries 10
         ports:
           - 6379:6379
+      minio:
+        image: bitnami/minio:2021.3.17
+        env:
+          MINIO_ACCESS_KEY: 123456
+          MINIO_SECRET_KEY: 12345678
+        ports:
+          - "9000:9000"
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4

Dockerfile

@@ -1,5 +1,5 @@
 #Build stage
-FROM docker.io/library/golang:1.20-alpine3.17 AS build-env
+FROM docker.io/library/golang:1.20-alpine3.18 AS build-env
 
 ARG GOPROXY
 ENV GOPROXY ${GOPROXY:-direct}
@@ -23,7 +23,7 @@ RUN if [ -n "${GITEA_VERSION}" ]; then git checkout "${GITEA_VERSION}"; fi \
 # Begin env-to-ini build
 RUN go build contrib/environment-to-ini/environment-to-ini.go
 
-FROM docker.io/library/alpine:3.17
+FROM docker.io/library/alpine:3.18
 LABEL maintainer="[email protected]"
 
 EXPOSE 22 3000

Dockerfile.rootless

@@ -1,5 +1,5 @@
 #Build stage
-FROM docker.io/library/golang:1.20-alpine3.17 AS build-env
+FROM docker.io/library/golang:1.20-alpine3.18 AS build-env
 
 ARG GOPROXY
 ENV GOPROXY ${GOPROXY:-direct}
@@ -23,7 +23,7 @@ RUN if [ -n "${GITEA_VERSION}" ]; then git checkout "${GITEA_VERSION}"; fi \
 # Begin env-to-ini build
 RUN go build contrib/environment-to-ini/environment-to-ini.go
 
-FROM docker.io/library/alpine:3.17
+FROM docker.io/library/alpine:3.18
 LABEL maintainer="[email protected]"
 
 EXPOSE 2222 3000

models/git/commit_status.go

@@ -23,6 +23,7 @@ import (
 	api "code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/timeutil"
 
+	"xorm.io/builder"
 	"xorm.io/xorm"
 )
 
@@ -240,6 +241,55 @@ func GetLatestCommitStatus(ctx context.Context, repoID int64, sha string, listOp
 	return statuses, count, db.GetEngine(ctx).In("id", ids).Find(&statuses)
 }
 
+// GetLatestCommitStatusForPairs returns all statuses with a unique context for a given list of repo-sha pairs
+func GetLatestCommitStatusForPairs(ctx context.Context, repoIDsToLatestCommitSHAs map[int64]string, listOptions db.ListOptions) (map[int64][]*CommitStatus, error) {
+	type result struct {
+		ID     int64
+		RepoID int64
+	}
+
+	results := make([]result, 0, len(repoIDsToLatestCommitSHAs))
+
+	sess := db.GetEngine(ctx).Table(&CommitStatus{})
+
+	// Create a disjunction of conditions for each repoID and SHA pair
+	conds := make([]builder.Cond, 0, len(repoIDsToLatestCommitSHAs))
+	for repoID, sha := range repoIDsToLatestCommitSHAs {
+		conds = append(conds, builder.Eq{"repo_id": repoID, "sha": sha})
+	}
+	sess = sess.Where(builder.Or(conds...)).
+		Select("max( id ) as id, repo_id").
+		GroupBy("context_hash, repo_id").OrderBy("max( id ) desc")
+
+	sess = db.SetSessionPagination(sess, &listOptions)
+
+	err := sess.Find(&results)
+	if err != nil {
+		return nil, err
+	}
+
+	ids := make([]int64, 0, len(results))
+	repoStatuses := make(map[int64][]*CommitStatus)
+	for _, result := range results {
+		ids = append(ids, result.ID)
+	}
+
+	statuses := make([]*CommitStatus, 0, len(ids))
+	if len(ids) > 0 {
+		err = db.GetEngine(ctx).In("id", ids).Find(&statuses)
+		if err != nil {
+			return nil, err
+		}
+
+		// Group the statuses by repo ID
+		for _, status := range statuses {
+			repoStatuses[status.RepoID] = append(repoStatuses[status.RepoID], status)
+		}
+	}
+
+	return repoStatuses, nil
+}
+
 // FindRepoRecentCommitStatusContexts returns repository's recent commit status contexts
 func FindRepoRecentCommitStatusContexts(ctx context.Context, repoID int64, before time.Duration) ([]string, error) {
 	start := timeutil.TimeStampNow().AddDuration(-before)

modules/git/repo_branch.go

@@ -106,6 +106,17 @@ func GetBranchesByPath(ctx context.Context, path string, skip, limit int) ([]*Br
 	return gitRepo.GetBranches(skip, limit)
 }
 
+// GetBranchCommitID returns a branch commit ID by its name
+func GetBranchCommitID(ctx context.Context, path, branch string) (string, error) {
+	gitRepo, err := OpenRepository(ctx, path)
+	if err != nil {
+		return "", err
+	}
+	defer gitRepo.Close()
+
+	return gitRepo.GetBranchCommitID(branch)
+}
+
 // GetBranches returns a slice of *git.Branch
 func (repo *Repository) GetBranches(skip, limit int) ([]*Branch, int, error) {
 	brs, countAll, err := repo.GetBranchNames(skip, limit)

modules/storage/local.go

@@ -133,8 +133,8 @@ func (l *LocalStorage) URL(path, name string) (*url.URL, error) {
 }
 
 // IterateObjects iterates across the objects in the local storage
-func (l *LocalStorage) IterateObjects(prefix string, fn func(path string, obj Object) error) error {
-	dir := l.buildLocalPath(prefix)
+func (l *LocalStorage) IterateObjects(dirName string, fn func(path string, obj Object) error) error {
+	dir := l.buildLocalPath(dirName)
 	return filepath.WalkDir(dir, func(path string, d os.DirEntry, err error) error {
 		if err != nil {
 			return err

modules/storage/local_test.go

@@ -4,8 +4,6 @@
 package storage
 
 import (
-	"bytes"
-	"context"
 	"os"
 	"path/filepath"
 	"testing"
@@ -57,38 +55,5 @@ func TestBuildLocalPath(t *testing.T) {
 
 func TestLocalStorageIterator(t *testing.T) {
 	dir := filepath.Join(os.TempDir(), "TestLocalStorageIteratorTestDir")
-	l, err := NewLocalStorage(context.Background(), LocalStorageConfig{Path: dir})
-	assert.NoError(t, err)
-
-	testFiles := [][]string{
-		{"a/1.txt", "a1"},
-		{"/a/1.txt", "aa1"}, // same as above, but with leading slash that will be trim
-		{"b/1.txt", "b1"},
-		{"b/2.txt", "b2"},
-		{"b/3.txt", "b3"},
-		{"b/x 4.txt", "bx4"},
-	}
-	for _, f := range testFiles {
-		_, err = l.Save(f[0], bytes.NewBufferString(f[1]), -1)
-		assert.NoError(t, err)
-	}
-
-	expectedList := map[string][]string{
-		"a":           {"a/1.txt"},
-		"b":           {"b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"},
-		"":            {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"},
-		"/":           {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"},
-		"a/b/../../a": {"a/1.txt"},
-	}
-	for dir, expected := range expectedList {
-		count := 0
-		err = l.IterateObjects(dir, func(path string, f Object) error {
-			defer f.Close()
-			assert.Contains(t, expected, path)
-			count++
-			return nil
-		})
-		assert.NoError(t, err)
-		assert.Len(t, expected, count)
-	}
+	testStorageIterator(t, string(LocalStorageType), LocalStorageConfig{Path: dir})
 }

modules/storage/minio.go

@@ -129,7 +129,11 @@ func NewMinioStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error
 }
 
 func (m *MinioStorage) buildMinioPath(p string) string {
-	return util.PathJoinRelX(m.basePath, p)
+	p = util.PathJoinRelX(m.basePath, p)
+	if p == "." {
+		p = "" // minio doesn't use dot as relative path
+	}
+	return p
 }
 
 // Open opens a file
@@ -224,14 +228,15 @@ func (m *MinioStorage) URL(path, name string) (*url.URL, error) {
 }
 
 // IterateObjects iterates across the objects in the miniostorage
-func (m *MinioStorage) IterateObjects(prefix string, fn func(path string, obj Object) error) error {
+func (m *MinioStorage) IterateObjects(dirName string, fn func(path string, obj Object) error) error {
 	opts := minio.GetObjectOptions{}
 	lobjectCtx, cancel := context.WithCancel(m.ctx)
 	defer cancel()
 
 	basePath := m.basePath
-	if prefix != "" {
-		basePath = m.buildMinioPath(prefix)
+	if dirName != "" {
+		// ending slash is required for avoiding matching like "foo/" and "foobar/" with prefix "foo"
+		basePath = m.buildMinioPath(dirName) + "/"
 	}
 
 	for mObjInfo := range m.client.ListObjects(lobjectCtx, m.bucket, minio.ListObjectsOptions{
@@ -244,7 +249,7 @@ func (m *MinioStorage) IterateObjects(prefix string, fn func(path string, obj Ob
 		}
 		if err := func(object *minio.Object, fn func(path string, obj Object) error) error {
 			defer object.Close()
-			return fn(strings.TrimPrefix(mObjInfo.Key, basePath), &minioObject{object})
+			return fn(strings.TrimPrefix(mObjInfo.Key, m.basePath), &minioObject{object})
 		}(object, fn); err != nil {
 			return convertMinioErr(err)
 		}

modules/storage/minio_test.go

@@ -0,0 +1,18 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package storage
+
+import (
+	"testing"
+)
+
+func TestMinioStorageIterator(t *testing.T) {
+	testStorageIterator(t, string(MinioStorageType), MinioStorageConfig{
+		Endpoint:        "127.0.0.1:9000",
+		AccessKeyID:     "123456",
+		SecretAccessKey: "12345678",
+		Bucket:          "gitea",
+		Location:        "us-east-1",
+	})
+}

modules/storage/storage_test.go

@@ -0,0 +1,49 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package storage
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func testStorageIterator(t *testing.T, typStr string, cfg interface{}) {
+	l, err := NewStorage(typStr, cfg)
+	assert.NoError(t, err)
+
+	testFiles := [][]string{
+		{"a/1.txt", "a1"},
+		{"/a/1.txt", "aa1"}, // same as above, but with leading slash that will be trim
+		{"ab/1.txt", "ab1"},
+		{"b/1.txt", "b1"},
+		{"b/2.txt", "b2"},
+		{"b/3.txt", "b3"},
+		{"b/x 4.txt", "bx4"},
+	}
+	for _, f := range testFiles {
+		_, err = l.Save(f[0], bytes.NewBufferString(f[1]), -1)
+		assert.NoError(t, err)
+	}
+
+	expectedList := map[string][]string{
+		"a":           {"a/1.txt"},
+		"b":           {"b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt"},
+		"":            {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"},
+		"/":           {"a/1.txt", "b/1.txt", "b/2.txt", "b/3.txt", "b/x 4.txt", "ab/1.txt"},
+		"a/b/../../a": {"a/1.txt"},
+	}
+	for dir, expected := range expectedList {
+		count := 0
+		err = l.IterateObjects(dir, func(path string, f Object) error {
+			defer f.Close()
+			assert.Contains(t, expected, path)
+			count++
+			return nil
+		})
+		assert.NoError(t, err)
+		assert.Len(t, expected, count)
+	}
+}

routers/web/repo/repo.go

@@ -9,9 +9,11 @@ import (
 	"fmt"
 	"net/http"
 	"strings"
+	"sync"
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/models/db"
+	git_model "code.gitea.io/gitea/models/git"
 	"code.gitea.io/gitea/models/organization"
 	access_model "code.gitea.io/gitea/models/perm/access"
 	repo_model "code.gitea.io/gitea/models/repo"
@@ -576,23 +578,49 @@ func SearchRepo(ctx *context.Context) {
 		return
 	}
 
-	results := make([]*api.Repository, len(repos))
+	// collect the latest commit of each repo
+	repoIDsToLatestCommitSHAs := make(map[int64]string)
+	wg := sync.WaitGroup{}
+	wg.Add(len(repos))
+	for _, repo := range repos {
+		go func(repo *repo_model.Repository) {
+			defer wg.Done()
+			commitID, err := repo_service.GetBranchCommitID(ctx, repo, repo.DefaultBranch)
+			if err != nil {
+				return
+			}
+			repoIDsToLatestCommitSHAs[repo.ID] = commitID
+		}(repo)
+	}
+	wg.Wait()
+
+	// call the database O(1) times to get the commit statuses for all repos
+	repoToItsLatestCommitStatuses, err := git_model.GetLatestCommitStatusForPairs(ctx, repoIDsToLatestCommitSHAs, db.ListOptions{})
+	if err != nil {
+		log.Error("GetLatestCommitStatusForPairs: %v", err)
+		return
+	}
+
+	results := make([]*repo_service.WebSearchRepository, len(repos))
 	for i, repo := range repos {
-		results[i] = &api.Repository{
-			ID:       repo.ID,
-			FullName: repo.FullName(),
-			Fork:     repo.IsFork,
-			Private:  repo.IsPrivate,
-			Template: repo.IsTemplate,
-			Mirror:   repo.IsMirror,
-			Stars:    repo.NumStars,
-			HTMLURL:  repo.HTMLURL(),
-			Link:     repo.Link(),
-			Internal: !repo.IsPrivate && repo.Owner.Visibility == api.VisibleTypePrivate,
+		results[i] = &repo_service.WebSearchRepository{
+			Repository: &api.Repository{
+				ID:       repo.ID,
+				FullName: repo.FullName(),
+				Fork:     repo.IsFork,
+				Private:  repo.IsPrivate,
+				Template: repo.IsTemplate,
+				Mirror:   repo.IsMirror,
+				Stars:    repo.NumStars,
+				HTMLURL:  repo.HTMLURL(),
+				Link:     repo.Link(),
+				Internal: !repo.IsPrivate && repo.Owner.Visibility == api.VisibleTypePrivate,
+			},
+			LatestCommitStatus: git_model.CalcCommitStatus(repoToItsLatestCommitStatuses[repo.ID]),
 		}
 	}
 
-	ctx.JSON(http.StatusOK, api.SearchResults{
+	ctx.JSON(http.StatusOK, repo_service.WebSearchResults{
 		OK:   true,
 		Data: results,
 	})

services/repository/branch.go

@@ -53,6 +53,10 @@ func GetBranches(ctx context.Context, repo *repo_model.Repository, skip, limit i
 	return git.GetBranchesByPath(ctx, repo.RepoPath(), skip, limit)
 }
 
+func GetBranchCommitID(ctx context.Context, repo *repo_model.Repository, branch string) (string, error) {
+	return git.GetBranchCommitID(ctx, repo.RepoPath(), branch)
+}
+
 // checkBranchName validates branch name with existing repository branches
 func checkBranchName(ctx context.Context, repo *repo_model.Repository, name string) error {
 	_, err := git.WalkReferences(ctx, repo.RepoPath(), func(_, refName string) error {