feat(code insights): language stats speed improvements by using archive loading (#62946)

We previously improved the performance of Language Stats Insights by
introducing parallel requests to gitserver:
https://github.com/sourcegraph/sourcegraph/pull/62011

This PR replaces the previous approach where we would iterate through
and request each file from gitserver with an approach where we request
just one archive. This eliminates a lot of network traffic, and gives us
an additional(!) performance improvement of 70-90%.

Even repositories like chromium (42GB) can now be processed (on my
machine in just one minute).

---

Caching: We dropped most of the caching, and kept only the top-level
caching (repo@commit). This means that we only need to compute the
language stats once per commit, and subsequent users/requests can see
the cached data. We dropped the file/directory level caching, because
(1) the code to do that got very complex and (2) we can assume that most
repositories are able to compute within the 5 minutes timeout (which can
be increase via the environment variable `GET_INVENTORY_TIMEOUT`). The
timeout is not bound to the user's request anymore. Before, the frontend
would request the stats up to three times to let the computation move
forward and pick up where the last request aborted. While we still have
this frontend retry mechanism, we don't have to worry about an
abort-and-continue mechanism in the backend.

---

Credits for the code to @eseliger:
https://github.com/sourcegraph/sourcegraph/issues/62019#issuecomment-2119278481

I've taken the diff, and updated the caching methods to allow for more
advanced use cases should we decide to introduce more caching. We can
take that out again if the current caching is sufficient.

Todos:

- [x] Check if CI passes, manual testing seems to be fine
- [x] Verify that insights are cached at the top level

---

Test data:

- sourcegraph/sourcegraph: 9.07s (main) -> 1.44s (current): 74% better
- facebook/react: 17.52s (main) -> 0.87s (current): 95% better
- godotengine/godot: 28.92s (main) -> 1.98s (current): 93% better
- chromium/chromium: ~1 minute: 100% better, because it didn't compute
before

## Changelog

- Language stats queries now request one archive from gitserver instead
of individual file requests. This leads to a huge performance
improvement. Even extra large repositories like chromium are now able to
compute within one minute. Previously they timed out.

## Test plan

- New unit tests
- Plenty of manual testing

Author: bahrmichael

CHANGELOG.md

@@ -37,6 +37,7 @@ All notable changes to Sourcegraph are documented in this file.
 
 - The default and recommended chat model for Anthropic and Cody Gateway configurations is now `claude-3-sonnet-20240229`. [#62757](https://github.com/sourcegraph/sourcegraph/pull/62757)
 - The default and recommended autocomplete model for Cody Gateway configurations is now `fireworks/starcoder`. [#62757](https://github.com/sourcegraph/sourcegraph/pull/62757)
+- Code Insights: Language Stats Insights performance improved by another 70-90%. It's now able to handle repositories above 40 GB. [#62946](https://github.com/sourcegraph/sourcegraph/pull/62946)
 - The keyword search toggle has been removed from the search results page. [Keyword search](https://sourcegraph.com/docs/code-search/queries#keyword-search-default) is now enabled by default for all searches in the Sourcegraph web app. [#63584](https://github.com/sourcegraph/sourcegraph/pull/63584)
 
 ### Fixed

cmd/frontend/backend/inventory.go

@@ -52,20 +52,16 @@ func InventoryContext(logger log.Logger, repo api.RepoName, gsClient gitserver.C
 		return inventory.Context{}, errors.Errorf("refusing to compute inventory for non-absolute commit ID %q", commitID)
 	}
 
-	cacheKey := func(e fs.FileInfo) string {
-		info, ok := e.Sys().(gitdomain.ObjectInfo)
-		if !ok {
-			return "" // not cacheable
-		}
-		return info.OID().String()
-	}
-
 	gitServerSemaphore := semaphore.NewWeighted(int64(gitServerConcurrency))
 	redisSemaphore := semaphore.NewWeighted(int64(redisConcurrency))
 
 	logger = logger.Scoped("InventoryContext").
 		With(log.String("repo", string(repo)), log.String("commitID", string(commitID)))
 	invCtx := inventory.Context{
+		Repo:                                repo,
+		CommitID:                            commitID,
+		ShouldSkipEnhancedLanguageDetection: !useEnhancedLanguageDetection && !forceEnhancedLanguageDetection,
+		GitServerClient:                     gsClient,
 		ReadTree: func(ctx context.Context, path string) ([]fs.FileInfo, error) {
 			trc, ctx := trace.New(ctx, "ReadTree waits for semaphore")
 			err := gitServerSemaphore.Acquire(ctx, 1)
@@ -109,55 +105,52 @@ func InventoryContext(logger log.Logger, repo api.RepoName, gsClient gitserver.C
 				gitServerSemaphore.Release(1)
 			}}, nil
 		},
-		CacheGet: func(ctx context.Context, e fs.FileInfo) (inventory.Inventory, bool) {
-			cacheKey := cacheKey(e)
+		CacheKey: func(e fs.FileInfo) string {
+			info, ok := e.Sys().(gitdomain.ObjectInfo)
+			if !ok {
+				return "" // not cacheable
+			}
+			return info.OID().String()
+		},
+		CacheGet: func(ctx context.Context, cacheKey string) (inventory.Inventory, bool) {
 			if cacheKey == "" {
 				return inventory.Inventory{}, false // not cacheable
 			}
 
 			if err := redisSemaphore.Acquire(ctx, 1); err != nil {
-				logger.Warn("Failed to acquire semaphore for redis cache.", log.String("path", e.Name()), log.Error(err))
+				logger.Warn("Failed to acquire semaphore for redis cache.", log.String("cacheKey", cacheKey), log.Error(err))
 				return inventory.Inventory{}, false
 			}
 			defer redisSemaphore.Release(1)
 
 			if b, ok := inventoryCache.Get(cacheKey); ok {
 				var inv inventory.Inventory
 				if err := json.Unmarshal(b, &inv); err != nil {
-					logger.Warn("Failed to unmarshal cached JSON inventory.", log.String("path", e.Name()), log.Error(err))
+					logger.Warn("Failed to unmarshal cached JSON inventory.", log.String("cacheKey", cacheKey), log.Error(err))
 					return inventory.Inventory{}, false
 				}
 				return inv, true
 			}
 			return inventory.Inventory{}, false
 		},
-		CacheSet: func(ctx context.Context, e fs.FileInfo, inv inventory.Inventory) {
-			cacheKey := cacheKey(e)
+		CacheSet: func(ctx context.Context, cacheKey string, inv inventory.Inventory) {
 			if cacheKey == "" {
 				return // not cacheable
 			}
 			b, err := json.Marshal(&inv)
 			if err != nil {
-				logger.Warn("Failed to marshal JSON inventory for cache.", log.String("path", e.Name()), log.Error(err))
+				logger.Warn("Failed to marshal JSON inventory for cache.", log.String("cacheKey", cacheKey), log.Error(err))
 				return
 			}
 
 			if err := redisSemaphore.Acquire(ctx, 1); err != nil {
-				logger.Warn("Failed to acquire semaphore for redis cache.", log.String("path", e.Name()), log.Error(err))
+				logger.Warn("Failed to acquire semaphore for redis cache.", log.String("cacheKey", cacheKey), log.Error(err))
 				return
 			}
 			defer redisSemaphore.Release(1)
 			inventoryCache.Set(cacheKey, b)
 		},
 	}
 
-	if !useEnhancedLanguageDetection && !forceEnhancedLanguageDetection {
-		// If USE_ENHANCED_LANGUAGE_DETECTION is disabled, do not read file contents to determine
-		// the language. This means we won't calculate the number of lines per language.
-		invCtx.NewFileReader = func(ctx context.Context, path string) (io.ReadCloser, error) {
-			return nil, nil
-		}
-	}
-
 	return invCtx, nil
 }

cmd/frontend/backend/repos.go

@@ -3,14 +3,7 @@ package backend
 import (
 	"context"
 	"fmt"
-	"strconv"
-	"time"
-
-	"github.com/sourcegraph/sourcegraph/internal/env"
-
 	"github.com/sourcegraph/log"
-	"go.opentelemetry.io/otel/attribute"
-
 	"github.com/sourcegraph/sourcegraph/cmd/frontend/internal/inventory"
 	"github.com/sourcegraph/sourcegraph/internal/api"
 	"github.com/sourcegraph/sourcegraph/internal/database"
@@ -19,6 +12,7 @@ import (
 	"github.com/sourcegraph/sourcegraph/internal/trace"
 	"github.com/sourcegraph/sourcegraph/internal/types"
 	"github.com/sourcegraph/sourcegraph/lib/errors"
+	"go.opentelemetry.io/otel/attribute"
 )
 
 type ReposService interface {
@@ -112,8 +106,6 @@ func (s *repos) ListIndexable(ctx context.Context) (repos []types.MinimalRepo, e
 	})
 }
 
-var getInventoryTimeout, _ = strconv.Atoi(env.Get("GET_INVENTORY_TIMEOUT", "5", "Time in minutes before cancelling getInventory requests. Raise this if your repositories are large and need a long time to process."))
-
 func (s *repos) GetInventory(ctx context.Context, repo api.RepoName, commitID api.CommitID, forceEnhancedLanguageDetection bool) (res *inventory.Inventory, err error) {
 	if Mocks.Repos.GetInventory != nil {
 		return Mocks.Repos.GetInventory(ctx, repo, commitID)
@@ -122,24 +114,12 @@ func (s *repos) GetInventory(ctx context.Context, repo api.RepoName, commitID ap
 	ctx, done := startTrace(ctx, "GetInventory", map[string]any{"repo": repo, "commitID": commitID}, &err)
 	defer done()
 
-	ctx, cancel := context.WithTimeout(ctx, time.Duration(getInventoryTimeout)*time.Minute)
-	defer cancel()
-
 	invCtx, err := InventoryContext(s.logger, repo, s.gitserverClient, commitID, forceEnhancedLanguageDetection)
 	if err != nil {
 		return nil, err
 	}
 
-	root, err := s.gitserverClient.Stat(ctx, repo, commitID, "")
-	if err != nil {
-		return nil, err
-	}
-
-	// In computing the inventory, sub-tree inventories are cached based on the OID of the Git
-	// tree. Compared to per-blob caching, this creates many fewer cache entries, which means fewer
-	// stores, fewer lookups, and less cache storage overhead. Compared to per-commit caching, this
-	// yields a higher cache hit rate because most trees are unchanged across commits.
-	inv, err := invCtx.Entries(ctx, root)
+	inv, err := invCtx.All(ctx)
 	if err != nil {
 		return nil, err
 	}

cmd/frontend/backend/repos_test.go

@@ -1,6 +1,7 @@
 package backend
 
 import (
+	"archive/tar"
 	"bytes"
 	"context"
 	"flag"
@@ -69,6 +70,43 @@ func (oid gitObjectInfo) OID() gitdomain.OID {
 	return v
 }
 
+type FileData struct {
+	Name    string
+	Content string
+}
+
+// createInMemoryTarArchive creates a tar archive in memory containing multiple files with their given content.
+func createInMemoryTarArchive(files []FileData) ([]byte, error) {
+	buf := new(bytes.Buffer)
+	tarWriter := tar.NewWriter(buf)
+
+	for _, file := range files {
+		header := &tar.Header{
+			Name: file.Name,
+			Size: int64(len(file.Content)),
+		}
+
+		err := tarWriter.WriteHeader(header)
+		if err != nil {
+			return nil, err
+		}
+
+		// Write the content to the tar archive.
+		_, err = io.WriteString(tarWriter, file.Content)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Close the tar writer to flush the data to the buffer.
+	err := tarWriter.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}
+
 func TestReposGetInventory(t *testing.T) {
 	ctx := testContext()
 
@@ -108,13 +146,27 @@ func TestReposGetInventory(t *testing.T) {
 		switch name {
 		case "b.go":
 			data = []byte("package main")
-		case "a/c.m":
+		case "c.m":
 			data = []byte("@interface X:NSObject {}")
 		default:
 			panic("unhandled mock ReadFile " + name)
 		}
 		return io.NopCloser(bytes.NewReader(data)), nil
 	})
+	gitserverClient.ArchiveReaderFunc.SetDefaultHook(func(_ context.Context, _ api.RepoName, archiveOptions gitserver.ArchiveOptions) (io.ReadCloser, error) {
+		files := []FileData{
+			{Name: "b.go", Content: "package main"},
+			{Name: "a/c.m", Content: "@interface X:NSObject {}"},
+		}
+
+		// Create the in-memory tar archive.
+		archiveData, err := createInMemoryTarArchive(files)
+		if err != nil {
+			t.Fatalf("Failed to create in-memory tar archive: %v", err)
+		}
+
+		return io.NopCloser(bytes.NewReader(archiveData)), nil
+	})
 	s := repos{
 		logger:          logtest.Scoped(t),
 		gitserverClient: gitserverClient,

cmd/frontend/graphqlbackend/search_results_stats_languages_test.go

@@ -1,6 +1,7 @@
 package graphqlbackend
 
 import (
+	"archive/tar"
 	"bytes"
 	"context"
 	"io"
@@ -22,6 +23,43 @@ import (
 	"github.com/sourcegraph/sourcegraph/internal/types"
 )
 
+type FileData struct {
+	Name    string
+	Content string
+}
+
+// createInMemoryTarArchive creates a tar archive in memory containing multiple files with their given content.
+func createInMemoryTarArchive(files []FileData) ([]byte, error) {
+	buf := new(bytes.Buffer)
+	tarWriter := tar.NewWriter(buf)
+
+	for _, file := range files {
+		header := &tar.Header{
+			Name: file.Name,
+			Size: int64(len(file.Content)),
+		}
+
+		err := tarWriter.WriteHeader(header)
+		if err != nil {
+			return nil, err
+		}
+
+		// Write the content to the tar archive.
+		_, err = io.WriteString(tarWriter, file.Content)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Close the tar writer to flush the data to the buffer.
+	err := tarWriter.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}
+
 func TestSearchResultsStatsLanguages(t *testing.T) {
 	logger := logtest.Scoped(t)
 	wantCommitID := api.CommitID(strings.Repeat("a", 40))
@@ -58,6 +96,20 @@ func TestSearchResultsStatsLanguages(t *testing.T) {
 	gsClient.StatFunc.SetDefaultHook(func(_ context.Context, _ api.RepoName, _ api.CommitID, path string) (fs.FileInfo, error) {
 		return &fileutil.FileInfo{Name_: path, Mode_: os.ModeDir}, nil
 	})
+	gsClient.ArchiveReaderFunc.SetDefaultHook(func(_ context.Context, _ api.RepoName, archiveOptions gitserver.ArchiveOptions) (io.ReadCloser, error) {
+		files := []FileData{
+			{Name: "two.go", Content: "a\nb\n"},
+			{Name: "three.go", Content: "a\nb\nc\n"},
+		}
+
+		// Create the in-memory tar archive.
+		archiveData, err := createInMemoryTarArchive(files)
+		if err != nil {
+			t.Fatalf("Failed to create in-memory tar archive: %v", err)
+		}
+
+		return io.NopCloser(bytes.NewReader(archiveData)), nil
+	})
 
 	mkResult := func(path string, lineNumbers ...int) *result.FileMatch {
 		rn := types.MinimalRepo{

cmd/frontend/internal/inventory/BUILD.bazel

@@ -12,14 +12,15 @@ go_library(
     tags = [TAG_SEARCHSUITE],
     visibility = ["//cmd/frontend:__subpackages__"],
     deps = [
-        "//internal/trace",
+        "//internal/api",
+        "//internal/env",
+        "//internal/gitserver",
         "//lib/codeintel/languages",
         "//lib/errors",
         "@com_github_go_enry_go_enry_v2//:go-enry",
         "@com_github_go_enry_go_enry_v2//data",
         "@com_github_sourcegraph_conc//iter",
         "@com_github_sourcegraph_log//:log",
-        "@io_opentelemetry_go_otel//attribute",
     ],
 )
 
@@ -32,9 +33,7 @@ go_test(
     embed = [":inventory"],
     tags = [TAG_SEARCHSUITE],
     deps = [
-        "//internal/fileutil",
         "//lib/errors",
         "@com_github_go_enry_go_enry_v2//:go-enry",
-        "@com_github_google_go_cmp//cmp",
     ],
 )

cmd/frontend/internal/inventory/context.go

@@ -2,12 +2,23 @@ package inventory
 
 import (
 	"context"
+	"github.com/sourcegraph/sourcegraph/internal/gitserver"
 	"io"
 	"io/fs"
+
+	"github.com/sourcegraph/sourcegraph/internal/api"
 )
 
 // Context defines the environment in which the inventory is computed.
 type Context struct {
+	Repo api.RepoName
+
+	CommitID api.CommitID
+
+	ShouldSkipEnhancedLanguageDetection bool
+
+	GitServerClient gitserver.Client
+
 	// ReadTree is called to list the immediate children of a tree at path. The returned fs.FileInfo
 	// values' Name method must return the full path (that can be passed to another ReadTree or
 	// ReadFile call), not just the basename.
@@ -16,9 +27,11 @@ type Context struct {
 	// NewFileReader is called to get an io.ReadCloser from the file at path.
 	NewFileReader func(ctx context.Context, path string) (io.ReadCloser, error)
 
+	CacheKey func(e fs.FileInfo) string
+
 	// CacheGet, if set, returns the cached inventory and true for the given tree, or false for a cache miss.
-	CacheGet func(context.Context, fs.FileInfo) (Inventory, bool)
+	CacheGet func(context.Context, string) (Inventory, bool)
 
 	// CacheSet, if set, stores the inventory in the cache for the given tree.
-	CacheSet func(context.Context, fs.FileInfo, Inventory)
+	CacheSet func(context.Context, string, Inventory)
 }