Adding searchRepositories jsonrpc method (#18827)

* adding searchRepositories method w/ github impl

* fix a few things and add bbs impl for searchRepos

* integrate searchRepositories in RepositoryFinder

* adjusting loading indicators

* cleanup

* wrap repository search behind feature flag

* Changing queries for github repo search

* fix enabled to use debounced value

* fix org filters

* drop variables for query

* fixing encoding

Author: selfcontained

components/dashboard/src/components/DropDown2.tsx

@@ -17,6 +17,7 @@ import React, {
 } from "react";
 import Arrow from "./Arrow";
 import classNames from "classnames";
+import { ReactComponent as Spinner } from "../icons/Spinner.svg";
 
 export interface DropDown2Element {
     id: string;
@@ -32,6 +33,8 @@ export interface DropDown2Props {
     disableSearch?: boolean;
     expanded?: boolean;
     onSelectionChange: (id: string) => void;
+    // Meant to allow consumers to react to search changes even though state is managed internally
+    onSearchChange?: (searchString: string) => void;
     allOptions?: string;
 }
 
@@ -45,6 +48,7 @@ export const DropDown2: FunctionComponent<DropDown2Props> = ({
     disableSearch,
     children,
     onSelectionChange,
+    onSearchChange,
 }) => {
     const [showDropDown, setShowDropDown] = useState<boolean>(!disabled && !!expanded);
     const nodeRef: RefObject<HTMLDivElement> = useRef(null);
@@ -61,7 +65,7 @@ export const DropDown2: FunctionComponent<DropDown2Props> = ({
 
     // reset search when the drop down is expanded or closed
     useEffect(() => {
-        setSearch("");
+        updateSearch("");
         if (allOptions) {
             setSelectedElementTemp(allOptions);
         }
@@ -72,6 +76,16 @@ export const DropDown2: FunctionComponent<DropDown2Props> = ({
         // eslint-disable-next-line react-hooks/exhaustive-deps
     }, [showDropDown]);
 
+    const updateSearch = useCallback(
+        (value: string) => {
+            setSearch(value);
+            if (onSearchChange) {
+                onSearchChange(value);
+            }
+        },
+        [onSearchChange],
+    );
+
     const toggleDropDown = useCallback(() => {
         if (disabled) {
             return;
@@ -158,6 +172,9 @@ export const DropDown2: FunctionComponent<DropDown2Props> = ({
         [setShowDropDown],
     );
 
+    const showInputLoadingIndicator = filteredOptions.length > 0 && loading;
+    const showResultsLoadingIndicator = filteredOptions.length === 0 && loading;
+
     return (
         <div
             onKeyDown={onKeyDown}
@@ -191,25 +208,30 @@ export const DropDown2: FunctionComponent<DropDown2Props> = ({
             {showDropDown && (
                 <div className="absolute w-full top-12 bg-gray-100 dark:bg-gray-800 rounded-b-lg mt-3 z-50 p-2 filter drop-shadow-xl">
                     {!disableSearch && (
-                        <div className="h-12">
+                        <div className="relative mb-2">
                             <input
                                 type="text"
                                 autoFocus
                                 className={"w-full focus rounded-lg"}
                                 placeholder={searchPlaceholder}
                                 value={search}
-                                onChange={(e) => setSearch(e.target.value)}
+                                onChange={(e) => updateSearch(e.target.value)}
                             />
+                            {showInputLoadingIndicator && (
+                                <div className="absolute top-0 right-0 h-full flex items-center pr-2">
+                                    <Spinner className="h-4 w-4 opacity-25 animate-spin" />
+                                </div>
+                            )}
                         </div>
                     )}
                     <ul className="max-h-60 overflow-auto">
-                        {loading && (
+                        {showResultsLoadingIndicator && (
                             <div className="flex-col space-y-2 animate-pulse">
                                 <div className="bg-gray-300 dark:bg-gray-500 h-4 rounded" />
                                 <div className="bg-gray-300 dark:bg-gray-500 h-4 rounded" />
                             </div>
                         )}
-                        {!loading && filteredOptions.length > 0 ? (
+                        {!showResultsLoadingIndicator && filteredOptions.length > 0 ? (
                             filteredOptions.map((element) => {
                                 let selectionClasses = `dark:bg-gray-800 cursor-pointer`;
                                 if (element.id === selectedElementTemp) {
@@ -237,7 +259,7 @@ export const DropDown2: FunctionComponent<DropDown2Props> = ({
                                     </li>
                                 );
                             })
-                        ) : !loading ? (
+                        ) : !showResultsLoadingIndicator ? (
                             <li key="no-elements" className={"rounded-md "}>
                                 <div className="h-12 pl-8 py-3 text-gray-800 dark:text-gray-200">No results</div>
                             </li>

components/dashboard/src/components/RepositoryFinder.tsx

@@ -9,10 +9,10 @@ import { getGitpodService } from "../service/service";
 import { DropDown2, DropDown2Element, DropDown2SelectedElement } from "./DropDown2";
 import RepositorySVG from "../icons/Repository.svg";
 import { ReactComponent as RepositoryIcon } from "../icons/RepositoryWithColor.svg";
-import { useSuggestedRepositories } from "../data/git-providers/suggested-repositories-query";
 import { useFeatureFlag } from "../data/featureflag-query";
 import { SuggestedRepository } from "@gitpod/gitpod-protocol";
 import { MiddleDot } from "./typography/MiddleDot";
+import { filterRepos, useUnifiedRepositorySearch } from "../data/git-providers/unified-repositories-search-query";
 
 // TODO: Remove this once we've fully enabled `includeProjectsOnCreateWorkspace`
 // flag (caches w/ react-query instead of local storage)
@@ -31,10 +31,12 @@ export default function RepositoryFinder(props: RepositoryFinderProps) {
     const includeProjectsOnCreateWorkspace = useFeatureFlag("includeProjectsOnCreateWorkspace");
 
     const [suggestedContextURLs, setSuggestedContextURLs] = useState<string[]>(loadSearchData());
-    const { data: suggestedRepos, isLoading } = useSuggestedRepositories();
+
+    const [searchString, setSearchString] = useState("");
+    const { data: repos, isLoading, isSearching } = useUnifiedRepositorySearch({ searchString });
 
     // TODO: remove this once includeProjectsOnCreateWorkspace is fully enabled
-    const suggestedRepoURLs = useMemo(() => {
+    const normalizedRepos = useMemo(() => {
         // If the flag is disabled continue to use suggestedContextURLs, but convert into SuggestedRepository objects
         if (!includeProjectsOnCreateWorkspace) {
             return suggestedContextURLs.map(
@@ -44,8 +46,8 @@ export default function RepositoryFinder(props: RepositoryFinderProps) {
             );
         }
 
-        return suggestedRepos || [];
-    }, [suggestedContextURLs, suggestedRepos, includeProjectsOnCreateWorkspace]);
+        return repos;
+    }, [includeProjectsOnCreateWorkspace, repos, suggestedContextURLs]);
 
     // TODO: remove this once includeProjectsOnCreateWorkspace is fully enabled
     useEffect(() => {
@@ -60,7 +62,7 @@ export default function RepositoryFinder(props: RepositoryFinderProps) {
     const handleSelectionChange = useCallback(
         (selectedID: string) => {
             // selectedId is either projectId or repo url
-            const matchingSuggestion = suggestedRepos?.find((repo) => {
+            const matchingSuggestion = normalizedRepos?.find((repo) => {
                 if (repo.projectId) {
                     return repo.projectId === selectedID;
                 }
@@ -75,12 +77,12 @@ export default function RepositoryFinder(props: RepositoryFinderProps) {
             // If we have no matching suggestion, it's a context URL they typed/pasted in, so just use that as the url
             props.setSelection(selectedID);
         },
-        [props, suggestedRepos],
+        [props, normalizedRepos],
     );
 
     // Resolve the selected context url & project id props to a suggestion entry
     const selectedSuggestion = useMemo(() => {
-        let match = suggestedRepos?.find((repo) => {
+        let match = normalizedRepos?.find((repo) => {
             if (props.selectedProjectID) {
                 return repo.projectId === props.selectedProjectID;
             }
@@ -105,15 +107,16 @@ export default function RepositoryFinder(props: RepositoryFinderProps) {
         }
 
         return match;
-    }, [props.selectedProjectID, props.selectedContextURL, suggestedRepos]);
+    }, [normalizedRepos, props.selectedContextURL, props.selectedProjectID]);
 
     const getElements = useCallback(
         (searchString: string) => {
-            const results = filterRepos(searchString, suggestedRepoURLs);
-
+            // TODO: remove once includeProjectsOnCreateWorkspace is fully enabled
             // With the flag off, we only want to show the suggestedContextURLs
             if (!includeProjectsOnCreateWorkspace) {
-                return results.map(
+                // w/o the flag on we still need to filter the repo as the search string changes
+                const filteredResults = filterRepos(searchString, normalizedRepos);
+                return filteredResults.map(
                     (repo) =>
                         ({
                             id: repo.url,
@@ -133,27 +136,29 @@ export default function RepositoryFinder(props: RepositoryFinderProps) {
                 );
             }
 
-            // Otherwise we show the suggestedRepos
-            return results.map((repo) => {
+            // Otherwise we show the suggestedRepos (already filtered)
+            return normalizedRepos.map((repo) => {
                 return {
                     id: repo.projectId || repo.url,
                     element: <SuggestedRepositoryOption repo={repo} />,
                     isSelectable: true,
                 } as DropDown2Element;
             });
         },
-        [includeProjectsOnCreateWorkspace, suggestedRepoURLs],
+        [includeProjectsOnCreateWorkspace, normalizedRepos],
     );
 
     return (
         <DropDown2
             getElements={getElements}
             expanded={!props.selectedContextURL}
+            // we use this to track the search string so we can search for repos via the api
             onSelectionChange={handleSelectionChange}
             disabled={props.disabled}
             // Only consider the isLoading prop if we're including projects in list
-            loading={isLoading && includeProjectsOnCreateWorkspace}
+            loading={(isLoading || isSearching) && includeProjectsOnCreateWorkspace}
             searchPlaceholder="Paste repository URL or type to find suggestions"
+            onSearchChange={setSearchString}
         >
             <DropDown2SelectedElement
                 icon={RepositorySVG}
@@ -201,7 +206,6 @@ const SuggestedRepositoryOption: FC<SuggestedRepositoryOptionProps> = ({ repo })
                 </>
             )}
 
-            {/* TODO: refine some Text* components a bit to make it easy to set the right colors for dark/light mode */}
             <span className="text-sm whitespace-nowrap truncate overflow-ellipsis text-gray-500 dark:text-gray-400">
                 {stripOffProtocol(repo.url)}
             </span>
@@ -238,28 +242,6 @@ function saveSearchData(searchData: string[]): void {
     }
 }
 
-function filterRepos(searchString: string, suggestedRepos: SuggestedRepository[]) {
-    let results = suggestedRepos;
-    const normalizedSearchString = searchString.trim().toLowerCase();
-
-    if (normalizedSearchString.length > 1) {
-        results = suggestedRepos.filter((r) => {
-            return `${r.url}${r.projectName || ""}`.toLowerCase().includes(normalizedSearchString);
-        });
-
-        if (results.length === 0) {
-            try {
-                // If the normalizedSearchString is a URL, and it's not present in the proposed results, "artificially" add it here.
-                new URL(normalizedSearchString);
-                results.push({ url: normalizedSearchString });
-            } catch {}
-        }
-    }
-
-    // Limit what we show to 200 results
-    return results.length > 200 ? results.slice(0, 200) : results;
-}
-
 function stripOffProtocol(url: string): string {
     if (!url.startsWith("http")) {
         return url;

components/dashboard/src/data/featureflag-query.ts

@@ -28,6 +28,7 @@ const featureFlags = {
     enabledOrbitalDiscoveries: "",
     newProjectIncrementalRepoSearchBBS: false,
     includeProjectsOnCreateWorkspace: false,
+    repositoryFinderSearch: false,
 };
 
 type FeatureFlags = typeof featureFlags;

components/dashboard/src/data/git-providers/search-repositories-query.ts

@@ -0,0 +1,36 @@
+/**
+ * Copyright (c) 2023 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License.AGPL.txt in the project root for license information.
+ */
+
+import { useQuery } from "@tanstack/react-query";
+import { getGitpodService } from "../../service/service";
+import { useCurrentOrg } from "../organizations/orgs-query";
+import { useDebounce } from "../../hooks/use-debounce";
+import { useFeatureFlag } from "../featureflag-query";
+
+export const useSearchRepositories = ({ searchString }: { searchString: string }) => {
+    // This disables the search behavior when flag is disabled
+    const repositoryFinderSearchEnabled = useFeatureFlag("repositoryFinderSearch");
+    const { data: org } = useCurrentOrg();
+    const debouncedSearchString = useDebounce(searchString);
+
+    return useQuery(
+        ["search-repositories", { organizationId: org?.id || "", searchString: debouncedSearchString }],
+        async () => {
+            return await getGitpodService().server.searchRepositories({
+                searchString,
+                organizationId: org?.id ?? "",
+            });
+        },
+        {
+            enabled: repositoryFinderSearchEnabled && !!org && debouncedSearchString.length >= 3,
+            // Need this to keep previous results while we wait for a new search to complete since debouncedSearchString changes and updates the key
+            keepPreviousData: true,
+            // We intentionally don't want to trigger refetches here to avoid a loading state side effect of focusing
+            refetchOnWindowFocus: false,
+            refetchOnReconnect: false,
+        },
+    );
+};

components/dashboard/src/data/git-providers/unified-repositories-search-query.ts

@@ -0,0 +1,59 @@
+/**
+ * Copyright (c) 2023 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License.AGPL.txt in the project root for license information.
+ */
+
+import { SuggestedRepository } from "@gitpod/gitpod-protocol";
+import { useSearchRepositories } from "./search-repositories-query";
+import { useSuggestedRepositories } from "./suggested-repositories-query";
+import { useMemo } from "react";
+
+// Combines the suggested repositories and the search repositories query into one hook
+export const useUnifiedRepositorySearch = ({ searchString }: { searchString: string }) => {
+    const suggestedQuery = useSuggestedRepositories();
+    const searchQuery = useSearchRepositories({ searchString });
+
+    const filteredRepos = useMemo(() => {
+        const repoMap = new Map<string, SuggestedRepository>((suggestedQuery.data || []).map((r) => [r.url, r]));
+
+        // Merge the search results into the suggested results
+        for (const repo of searchQuery.data || []) {
+            if (!repoMap.has(repo.url)) {
+                repoMap.set(repo.url, repo);
+            }
+        }
+
+        return filterRepos(searchString, Array.from(repoMap.values()));
+    }, [searchQuery.data, searchString, suggestedQuery.data]);
+
+    return {
+        data: filteredRepos,
+        isLoading: suggestedQuery.isLoading,
+        isSearching: searchQuery.isFetching,
+        isError: suggestedQuery.isError || searchQuery.isError,
+        error: suggestedQuery.error || searchQuery.error,
+    };
+};
+
+export const filterRepos = (searchString: string, suggestedRepos: SuggestedRepository[]) => {
+    let results = suggestedRepos;
+    const normalizedSearchString = searchString.trim().toLowerCase();
+
+    if (normalizedSearchString.length > 1) {
+        results = suggestedRepos.filter((r) => {
+            return `${r.url}${r.projectName || ""}`.toLowerCase().includes(normalizedSearchString);
+        });
+
+        if (results.length === 0) {
+            try {
+                // If the normalizedSearchString is a URL, and it's not present in the proposed results, "artificially" add it here.
+                new URL(normalizedSearchString);
+                results.push({ url: normalizedSearchString });
+            } catch {}
+        }
+    }
+
+    // Limit what we show to 200 results
+    return results.length > 200 ? results.slice(0, 200) : results;
+};

components/dashboard/src/hooks/use-debounce.ts

@@ -0,0 +1,32 @@
+/**
+ * Copyright (c) 2023 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License.AGPL.txt in the project root for license information.
+ */
+
+import { useEffect, useMemo, useState } from "react";
+import debounce from "lodash.debounce";
+
+type DebounceOptions = {
+    leading?: boolean;
+    trailing?: boolean;
+    maxWait?: number;
+};
+
+export const useDebounce = <T>(value: T, delay = 500, options?: DebounceOptions): T => {
+    const [debouncedValue, setDebouncedValue] = useState<T>(value);
+
+    const debouncedSetValue = useMemo(() => {
+        return debounce(setDebouncedValue, delay, {
+            leading: options?.leading || false,
+            trailing: options?.trailing || true,
+            maxWait: options?.maxWait ?? undefined,
+        });
+    }, [delay, options?.leading, options?.maxWait, options?.trailing]);
+
+    useEffect(() => {
+        debouncedSetValue(value);
+    }, [value, debouncedSetValue]);
+
+    return debouncedValue;
+};