Search: fix case-sensitive queries with lang filters (#62157)

This PR fixes a regression where language filters might not match when case
sensitivity is enabled. Before, we just appended `(?i)` to the file filters to
indicate case insensitivity, but still passed a case-sensitive file query to
Zoekt (called `case_file_regex`). Now, we make sure that the Zoekt queries are
also case insensitive so there is no conflict in how to interpret them.

Co-authored-by: Matthew Manela <[email protected]>

Author: jtibshirani

dev/gqltest/search_test.go

@@ -254,7 +254,28 @@ func testSearchClient(t *testing.T, client searchClient) {
 		}
 		// Make sure we only got .go files
 		for _, r := range results.Results {
-			if !strings.Contains(r.File.Name, ".go") {
+			if !strings.Contains(strings.ToLower(r.File.Name), ".go") {
+				t.Fatalf("Found file name does not end with .go: %s", r.File.Name)
+			}
+		}
+	})
+
+	t.Run("lang: filter with case sensitivity", func(t *testing.T) {
+		// Guard against a previous regression where case sensitivity broke lang filters. This search
+		// query closely mimics the one the web client ues for search-based code navigation.
+		results, err := client.SearchFiles("type:symbol ^readLine$ lang:go case:yes patterntype:regexp")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Ensure some results are returned
+		if len(results.Results) == 0 {
+			t.Fatal("Want non-zero results but got none")
+		}
+
+		// Make sure we only got .go files
+		for _, r := range results.Results {
+			if !strings.Contains(strings.ToLower(r.File.Name), ".go") {
 				t.Fatalf("Found file name does not end with .go: %s", r.File.Name)
 			}
 		}

internal/search/zoekt/query.go

@@ -52,27 +52,23 @@ func QueryToZoektQuery(b query.Basic, resultTypes result.Types, feat *search.Fea
 			and = append(and, &zoekt.Not{Child: filter})
 		}
 	} else {
-		filesInclude = append(filesInclude, mapSlice(langInclude, query.LangToFileRegexp)...)
-		filesExclude = append(filesExclude, mapSlice(langExclude, query.LangToFileRegexp)...)
-	}
-
-	// zoekt also uses regular expressions for file paths
-	// TODO PathPatternsAreCaseSensitive
-	// TODO whitespace in file path patterns?
-	for _, i := range filesInclude {
-		q, err := zoektquery.FileRe(i, isCaseSensitive)
+		// By default, convert the language filters to file regexp patterns. Note: these
+		// are always case-insensitive, to match broadly on extensions like file.C.
+		langFilters, err := toFileFilters(
+			mapSlice(langInclude, query.LangToFileRegexp),
+			mapSlice(langExclude, query.LangToFileRegexp),
+			false)
 		if err != nil {
 			return nil, err
 		}
-		and = append(and, q)
+		and = append(and, langFilters...)
 	}
-	if len(filesExclude) > 0 {
-		q, err := zoektquery.FileRe(query.UnionRegExps(filesExclude), isCaseSensitive)
-		if err != nil {
-			return nil, err
-		}
-		and = append(and, &zoekt.Not{Child: q})
+
+	fileFilters, err := toFileFilters(filesInclude, filesExclude, isCaseSensitive)
+	if err != nil {
+		return nil, err
 	}
+	and = append(and, fileFilters...)
 
 	var repoHasFilters []zoekt.Q
 	for _, filter := range b.RepoHasFileContent() {
@@ -90,6 +86,27 @@ func toLangFilter(lang string) zoekt.Q {
 	return &zoekt.Language{Language: lang}
 }
 
+// toFileFilters converts a list of file regexp patterns to Zoekt file filters.
+func toFileFilters(filesInclude []string, filesExclude []string, isCaseSensitive bool) ([]zoekt.Q, error) {
+	var filters []zoekt.Q
+	// TODO whitespace in file path patterns?
+	for _, i := range filesInclude {
+		q, err := zoektquery.FileRe(i, isCaseSensitive)
+		if err != nil {
+			return nil, err
+		}
+		filters = append(filters, q)
+	}
+	if len(filesExclude) > 0 {
+		q, err := zoektquery.FileRe(query.UnionRegExps(filesExclude), isCaseSensitive)
+		if err != nil {
+			return nil, err
+		}
+		filters = append(filters, &zoekt.Not{Child: q})
+	}
+	return filters, nil
+}
+
 func QueryForFileContentArgs(opt query.RepoHasFileContentArgs, caseSensitive bool) zoekt.Q {
 	var children []zoekt.Q
 	if opt.Path != "" {

internal/search/zoekt/query_test.go

@@ -82,13 +82,19 @@ func TestQueryToZoektQuery(t *testing.T) {
 			Name:            "Languages get passed as file filter",
 			Type:            search.TextRequest,
 			Query:           `file:\.go$ lang:go`,
-			WantZoektOutput: autogold.Expect(`(and file_regex:"\\.go(?m:$)" file_regex:"(?i:\\.GO)(?m:$)")`),
+			WantZoektOutput: autogold.Expect(`(and file_regex:"(?i:\\.GO)(?m:$)" file_regex:"\\.go(?m:$)")`),
 		},
 		{
-			Name:            "Languages still use case_insensitive in case sensitivity mode",
+			Name:            "Languages still use case_insensitive in case sensitivity mode (Go)",
 			Type:            search.TextRequest,
 			Query:           `file:\.go$ lang:go case:true`,
-			WantZoektOutput: autogold.Expect(`(and case_file_regex:"\\.go(?m:$)" case_file_regex:"(?i:\\.GO)(?m:$)")`),
+			WantZoektOutput: autogold.Expect(`(and file_regex:"(?i:\\.GO)(?m:$)" case_file_regex:"\\.go(?m:$)")`),
+		},
+		{
+			Name:            "Languages still use case_insensitive in case sensitivity mode (Typescript)",
+			Type:            search.TextRequest,
+			Query:           `lang:typescript case:true`,
+			WantZoektOutput: autogold.Expect(`file_regex:"(?i:\\.)(?:(?i:TS)(?m:$)|(?i:CTS)(?m:$)|(?i:MTS)(?m:$))"`),
 		},
 		{
 			Name:  "Language get passed as lang: query",