[dashboard] deduplicate repositories

components/dashboard/src/data/git-providers/unified-repositories-search-query.test.ts

@@ -0,0 +1,77 @@
+/**
+ * Copyright (c) 2023 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License.AGPL.txt in the project root for license information.
+ */
+
+import { SuggestedRepository } from "@gitpod/gitpod-protocol";
+import { deduplicateAndFilterRepositories } from "./unified-repositories-search-query";
+
+function repo(name: string, project?: string): SuggestedRepository {
+    return {
+        url: `http://github.com/efu3he4rf/${name}`,
+        repositoryName: name,
+        projectName: project,
+        projectId: project,
+    };
+}
+
+test("it should deduplicate non-project entries", () => {
+    const suggestedRepos: SuggestedRepository[] = [repo("foo"), repo("foo2"), repo("foo", "project-foo")];
+    const deduplicated = deduplicateAndFilterRepositories("foo", false, suggestedRepos);
+    expect(deduplicated.length).toEqual(2);
+    expect(deduplicated[1].projectName).toEqual("project-foo");
+});
+
+test("it should not deduplicate project entries", () => {
+    const suggestedRepos: SuggestedRepository[] = [
+        repo("foo", "project-foo2"),
+        repo("foo2"),
+        repo("foo", "project-foo"),
+    ];
+    const deduplicated = deduplicateAndFilterRepositories("foo", false, suggestedRepos);
+    expect(deduplicated.length).toEqual(3);
+});
+
+test("it should exclude project entries", () => {
+    const suggestedRepos: SuggestedRepository[] = [
+        repo("foo", "project-foo2"),
+        repo("foo2"),
+        repo("foo", "project-foo"),
+    ];
+    const deduplicated = deduplicateAndFilterRepositories("foo", true, suggestedRepos);
+    expect(deduplicated.length).toEqual(1);
+});
+
+test("it should match entries in url as well as poject name", () => {
+    const suggestedRepos: SuggestedRepository[] = [
+        repo("somefOOtest"),
+        repo("Footest"),
+        repo("somefoO"),
+        repo("bar", "somefOO"),
+        repo("bar", "someFootest"),
+        repo("bar", "FOOtest"),
+    ];
+    var deduplicated = deduplicateAndFilterRepositories("foo", false, suggestedRepos);
+    expect(deduplicated.length).toEqual(6);
+    deduplicated = deduplicateAndFilterRepositories("foot", false, suggestedRepos);
+    expect(deduplicated.length).toEqual(4);
+    deduplicated = deduplicateAndFilterRepositories("FOOT", false, suggestedRepos);
+    expect(deduplicated.length).toEqual(4);
+});
+
+test("it keeps the order", () => {
+    const suggestedRepos: SuggestedRepository[] = [
+        repo("somefOOtest"),
+        repo("Footest"),
+        repo("somefoO"),
+        repo("bar", "somefOO"),
+        repo("bar", "someFootest"),
+        repo("bar", "FOOtest"),
+    ];
+    const deduplicated = deduplicateAndFilterRepositories("foot", false, suggestedRepos);
+    expect(deduplicated[0].repositoryName).toEqual("somefOOtest");
+    expect(deduplicated[1].repositoryName).toEqual("Footest");
+    expect(deduplicated[2].projectName).toEqual("someFootest");
+    expect(deduplicated[3].projectName).toEqual("FOOtest");
+});

components/dashboard/src/data/git-providers/unified-repositories-search-query.ts

@@ -20,26 +20,8 @@ export const useUnifiedRepositorySearch = ({ searchString, excludeProjects = fal
     const searchQuery = useSearchRepositories({ searchString });
 
     const filteredRepos = useMemo(() => {
-        const repoMap = new Map<string, SuggestedRepository>();
-        // combine & flatten suggestions and search results, then merge them into a map
         const flattenedRepos = [suggestedQuery.data || [], searchQuery.data || []].flat();
-
-        for (const repo of flattenedRepos) {
-            const key = excludeProjects ? repo.url : `${repo.url}:${repo.projectId || ""}`;
-
-            const newEntry = {
-                ...(repoMap.get(key) || {}),
-                ...repo,
-            };
-            if (excludeProjects) {
-                // TODO: would be great if we can always include repositoryName on SuggestedRepository entities, then we could remove this
-                newEntry.repositoryName = newEntry.repositoryName || newEntry.projectName;
-                newEntry.projectName = undefined;
-            }
-            repoMap.set(key, newEntry);
-        }
-
-        return filterRepos(searchString, Array.from(repoMap.values()));
+        return deduplicateAndFilterRepositories(searchString, excludeProjects, flattenedRepos);
     }, [excludeProjects, searchQuery.data, searchString, suggestedQuery.data]);
 
     return {
@@ -51,24 +33,52 @@ export const useUnifiedRepositorySearch = ({ searchString, excludeProjects = fal
     };
 };
 
-export const filterRepos = (searchString: string, suggestedRepos: SuggestedRepository[]) => {
-    let results = suggestedRepos;
-    const normalizedSearchString = searchString.trim();
-
-    if (normalizedSearchString.length > 1) {
-        results = suggestedRepos.filter((r) => {
-            return `${r.url}${r.projectName || ""}`.toLowerCase().includes(normalizedSearchString);
+export function deduplicateAndFilterRepositories(
+    searchString: string,
+    excludeProjects = false,
+    suggestedRepos: SuggestedRepository[],
+): SuggestedRepository[] {
+    const normalizedSearchString = searchString.trim().toLowerCase();
+    const collected = new Set<string>();
+    const results: SuggestedRepository[] = [];
+    const reposWithProject = new Set<string>();
+    if (!excludeProjects) {
+        suggestedRepos.forEach((r) => {
+            if (r.projectId) {
+                reposWithProject.add(r.url);
+            }
         });
-
-        if (results.length === 0) {
-            try {
-                // If the normalizedSearchString is a URL, and it's not present in the proposed results, "artificially" add it here.
-                new URL(normalizedSearchString);
-                results.push({ url: normalizedSearchString });
-            } catch {}
+    }
+    for (const repo of suggestedRepos) {
+        // filter out project entries if excludeProjects is true
+        if (repo.projectId && excludeProjects) {
+            continue;
+        }
+        // filter out project-less entries if an entry with a project exists
+        if (!repo.projectId && reposWithProject.has(repo.url)) {
+            continue;
+        }
+        // filter out entries that don't match the search string
+        if (!`${repo.url}${repo.projectName || ""}`.toLowerCase().includes(normalizedSearchString)) {
+            continue;
+        }
+        // filter out duplicates
+        const key = `${repo.url}:${repo.projectId || "no-project"}`;
+        if (collected.has(key)) {
+            continue;
         }
+        collected.add(key);
+        results.push(repo);
+    }
+
+    if (results.length === 0) {
+        try {
+            // If the normalizedSearchString is a URL, and it's not present in the proposed results, "artificially" add it here.
+            new URL(normalizedSearchString);
+            results.push({ url: normalizedSearchString });
+        } catch {}
     }
 
     // Limit what we show to 200 results
-    return results.length > 200 ? results.slice(0, 200) : results;
-};
+    return results.slice(0, 200);
+}

components/server/src/bitbucket-server/bitbucket-server-repository-provider.ts

@@ -210,7 +210,7 @@ export class BitbucketServerRepositoryProvider implements RepositoryProvider {
 
     public async searchRepos(user: User, searchString: string): Promise<RepositoryInfo[]> {
         // Only load 1 page of 10 results for our searchString
-        const results = await this.api.getRepos(user, { maxPages: 1, limit: 10, searchString });
+        const results = await this.api.getRepos(user, { maxPages: 1, limit: 30, searchString });
 
         const repos: RepositoryInfo[] = [];
         results.forEach((r) => {