[WIP] Move towards sync deletion (away from PeriodicDeleter) - step II/II

WORKSPACE.yaml

@@ -7,8 +7,8 @@ defaultArgs:
   publishToNPM: true
   publishToJBMarketplace: true
   localAppVersion: unknown
-  codeCommit: 8bcdbe5e6e097191649984c8329af15a3e6b3066
-  codeVersion: 1.83.0
+  codeCommit: 7e9dd80fe5c75101c3b3e9ba71c0ff63cbd09768
+  codeVersion: 1.82.2
   codeQuality: stable
   noVerifyJBPlugin: false
   intellijDownloadUrl: "https://download.jetbrains.com/idea/ideaIU-2023.2.2.tar.gz"

components/blobserve/leeway.Dockerfile

@@ -2,7 +2,7 @@
 # Licensed under the GNU Affero General Public License (AGPL).
 # See License.AGPL.txt in the project root for license information.
 
-FROM cgr.dev/chainguard/wolfi-base:latest@sha256:a60ae0886c94287da6d54c9df735e87937d589dcdc6c556df1341caef5c3f6ba
+FROM cgr.dev/chainguard/wolfi-base:latest@sha256:ccc5551b5dd1fdcff5fc76ac1605b4c217f77f43410e0bd8a56599d6504dbbdd
 
 # Ensure latest packages are present, like security updates.
 RUN  apk upgrade --no-cache \

components/content-service/leeway.Dockerfile

@@ -2,7 +2,7 @@
 # Licensed under the GNU Affero General Public License (AGPL).
 # See License.AGPL.txt in the project root for license information.
 
-FROM cgr.dev/chainguard/wolfi-base:latest@sha256:a60ae0886c94287da6d54c9df735e87937d589dcdc6c556df1341caef5c3f6ba
+FROM cgr.dev/chainguard/wolfi-base:latest@sha256:ccc5551b5dd1fdcff5fc76ac1605b4c217f77f43410e0bd8a56599d6504dbbdd
 
 # Ensure latest packages are present, like security updates.
 RUN  apk upgrade --no-cache \

components/content-service/pkg/storage/s3.go

@@ -9,7 +9,6 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"strings"
 
@@ -290,39 +289,33 @@ func (s3st *s3Storage) DownloadSnapshot(ctx context.Context, destination string,
 	return s3st.download(ctx, destination, name, mappings)
 }
 
-// download object using s5cmd (prior to which we used aws sdk)
 func (s3st *s3Storage) download(ctx context.Context, destination string, obj string, mappings []archive.IDMapping) (found bool, err error) {
-	tempFile, err := os.CreateTemp("", "temporal-s3-file")
+	downloader := s3manager.NewDownloader(s3st.client, func(d *s3manager.Downloader) {
+		d.Concurrency = defaultCopyConcurrency
+		d.PartSize = defaultPartSize * megabytes
+		d.BufferProvider = s3manager.NewPooledBufferedWriterReadFromProvider(25 * megabytes)
+	})
+
+	s3File, err := os.CreateTemp("", "temporal-s3-file")
 	if err != nil {
 		return true, xerrors.Errorf("creating temporal file: %s", err.Error())
 	}
-	tempFile.Close()
-
-	args := []string{
-		"cp",
-		// # of file parts to download at once
-		"--concurrency", "20",
-		// size in MB of each part
-		"--part-size", "25",
-		destination,
-		tempFile.Name(),
-	}
-	cmd := exec.Command("s5cmd", args...)
-	out, err := cmd.CombinedOutput()
+	defer os.Remove(s3File.Name())
+
+	_, err = downloader.Download(ctx, s3File, &s3.GetObjectInput{
+		Bucket: aws.String(s3st.Config.Bucket),
+		Key:    aws.String(obj),
+	})
 	if err != nil {
-		log.WithError(err).WithField("out", string(out)).Error("unexpected error downloading file")
-		return true, xerrors.Errorf("unexpected error downloading file")
+		return false, err
 	}
 
-	tempFile, err = os.Open(tempFile.Name())
+	_, err = s3File.Seek(0, 0)
 	if err != nil {
-		return true, xerrors.Errorf("unexpected error opening downloaded file")
+		return false, err
 	}
 
-	defer os.Remove(tempFile.Name())
-	defer tempFile.Close()
-
-	err = archive.ExtractTarbal(ctx, tempFile, destination, archive.WithUIDMapping(mappings), archive.WithGIDMapping(mappings))
+	err = archive.ExtractTarbal(ctx, s3File, destination, archive.WithUIDMapping(mappings), archive.WithGIDMapping(mappings))
 	if err != nil {
 		return true, xerrors.Errorf("tar %s: %s", destination, err.Error())
 	}

components/dashboard/leeway.Dockerfile

@@ -2,7 +2,7 @@
 # Licensed under the GNU Affero General Public License (AGPL).
 # See License.AGPL.txt in the project root for license information.
 
-FROM cgr.dev/chainguard/wolfi-base:latest@sha256:a60ae0886c94287da6d54c9df735e87937d589dcdc6c556df1341caef5c3f6ba as compress
+FROM cgr.dev/chainguard/wolfi-base:latest@sha256:ccc5551b5dd1fdcff5fc76ac1605b4c217f77f43410e0bd8a56599d6504dbbdd as compress
 
 RUN apk add brotli gzip
 

components/dashboard/src/components/DropDown2.tsx

@@ -17,6 +17,7 @@ import React, {
 } from "react";
 import Arrow from "./Arrow";
 import classNames from "classnames";
+import { ReactComponent as Spinner } from "../icons/Spinner.svg";
 
 export interface DropDown2Element {
     id: string;
@@ -32,6 +33,8 @@ export interface DropDown2Props {
     disableSearch?: boolean;
     expanded?: boolean;
     onSelectionChange: (id: string) => void;
+    // Meant to allow consumers to react to search changes even though state is managed internally
+    onSearchChange?: (searchString: string) => void;
     allOptions?: string;
 }
 
@@ -45,6 +48,7 @@ export const DropDown2: FunctionComponent<DropDown2Props> = ({
     disableSearch,
     children,
     onSelectionChange,
+    onSearchChange,
 }) => {
     const [showDropDown, setShowDropDown] = useState<boolean>(!disabled && !!expanded);
     const nodeRef: RefObject<HTMLDivElement> = useRef(null);
@@ -61,7 +65,7 @@ export const DropDown2: FunctionComponent<DropDown2Props> = ({
 
     // reset search when the drop down is expanded or closed
     useEffect(() => {
-        setSearch("");
+        updateSearch("");
         if (allOptions) {
             setSelectedElementTemp(allOptions);
         }
@@ -72,6 +76,16 @@ export const DropDown2: FunctionComponent<DropDown2Props> = ({
         // eslint-disable-next-line react-hooks/exhaustive-deps
     }, [showDropDown]);
 
+    const updateSearch = useCallback(
+        (value: string) => {
+            setSearch(value);
+            if (onSearchChange) {
+                onSearchChange(value);
+            }
+        },
+        [onSearchChange],
+    );
+
     const toggleDropDown = useCallback(() => {
         if (disabled) {
             return;
@@ -158,6 +172,9 @@ export const DropDown2: FunctionComponent<DropDown2Props> = ({
         [setShowDropDown],
     );
 
+    const showInputLoadingIndicator = filteredOptions.length > 0 && loading;
+    const showResultsLoadingIndicator = filteredOptions.length === 0 && loading;
+
     return (
         <div
             onKeyDown={onKeyDown}
@@ -191,25 +208,30 @@ export const DropDown2: FunctionComponent<DropDown2Props> = ({
             {showDropDown && (
                 <div className="absolute w-full top-12 bg-gray-100 dark:bg-gray-800 rounded-b-lg mt-3 z-50 p-2 filter drop-shadow-xl">
                     {!disableSearch && (
-                        <div className="h-12">
+                        <div className="relative mb-2">
                             <input
                                 type="text"
                                 autoFocus
                                 className={"w-full focus rounded-lg"}
                                 placeholder={searchPlaceholder}
                                 value={search}
-                                onChange={(e) => setSearch(e.target.value)}
+                                onChange={(e) => updateSearch(e.target.value)}
                             />
+                            {showInputLoadingIndicator && (
+                                <div className="absolute top-0 right-0 h-full flex items-center pr-2">
+                                    <Spinner className="h-4 w-4 opacity-25 animate-spin" />
+                                </div>
+                            )}
                         </div>
                     )}
                     <ul className="max-h-60 overflow-auto">
-                        {loading && (
+                        {showResultsLoadingIndicator && (
                             <div className="flex-col space-y-2 animate-pulse">
                                 <div className="bg-gray-300 dark:bg-gray-500 h-4 rounded" />
                                 <div className="bg-gray-300 dark:bg-gray-500 h-4 rounded" />
                             </div>
                         )}
-                        {!loading && filteredOptions.length > 0 ? (
+                        {!showResultsLoadingIndicator && filteredOptions.length > 0 ? (
                             filteredOptions.map((element) => {
                                 let selectionClasses = `dark:bg-gray-800 cursor-pointer`;
                                 if (element.id === selectedElementTemp) {
@@ -237,7 +259,7 @@ export const DropDown2: FunctionComponent<DropDown2Props> = ({
                                     </li>
                                 );
                             })
-                        ) : !loading ? (
+                        ) : !showResultsLoadingIndicator ? (
                             <li key="no-elements" className={"rounded-md "}>
                                 <div className="h-12 pl-8 py-3 text-gray-800 dark:text-gray-200">No results</div>
                             </li>
@@ -279,7 +301,7 @@ export const DropDown2SelectedElement: FC<DropDown2SelectedElementProps> = ({
                     <>{icon}</>
                 )}
             </div>
-            <div className="flex-col ml-1 flex-grow">
+            <div className="flex-col ml-1 flex-grow max-w-xs">
                 {loading ? (
                     <div className="flex-col space-y-2">
                         <div className="bg-gray-300 dark:bg-gray-500 h-4 w-24 rounded" />
@@ -288,7 +310,7 @@ export const DropDown2SelectedElement: FC<DropDown2SelectedElementProps> = ({
                 ) : (
                     <>
                         <div className="text-gray-700 dark:text-gray-300 font-semibold">{title}</div>
-                        <div className="text-xs text-gray-500 dark:text-gray-400">{subtitle}</div>
+                        <div className="text-xs text-gray-500 dark:text-gray-400 truncate">{subtitle}</div>
                     </>
                 )}
             </div>

components/dashboard/src/components/RepositoryFinder.tsx

@@ -9,10 +9,10 @@ import { getGitpodService } from "../service/service";
 import { DropDown2, DropDown2Element, DropDown2SelectedElement } from "./DropDown2";
 import RepositorySVG from "../icons/Repository.svg";
 import { ReactComponent as RepositoryIcon } from "../icons/RepositoryWithColor.svg";
-import { useSuggestedRepositories } from "../data/git-providers/suggested-repositories-query";
 import { useFeatureFlag } from "../data/featureflag-query";
 import { SuggestedRepository } from "@gitpod/gitpod-protocol";
 import { MiddleDot } from "./typography/MiddleDot";
+import { filterRepos, useUnifiedRepositorySearch } from "../data/git-providers/unified-repositories-search-query";
 
 // TODO: Remove this once we've fully enabled `includeProjectsOnCreateWorkspace`
 // flag (caches w/ react-query instead of local storage)
@@ -31,10 +31,12 @@ export default function RepositoryFinder(props: RepositoryFinderProps) {
     const includeProjectsOnCreateWorkspace = useFeatureFlag("includeProjectsOnCreateWorkspace");
 
     const [suggestedContextURLs, setSuggestedContextURLs] = useState<string[]>(loadSearchData());
-    const { data: suggestedRepos, isLoading } = useSuggestedRepositories();
+
+    const [searchString, setSearchString] = useState("");
+    const { data: repos, isLoading, isSearching } = useUnifiedRepositorySearch({ searchString });
 
     // TODO: remove this once includeProjectsOnCreateWorkspace is fully enabled
-    const suggestedRepoURLs = useMemo(() => {
+    const normalizedRepos = useMemo(() => {
         // If the flag is disabled continue to use suggestedContextURLs, but convert into SuggestedRepository objects
         if (!includeProjectsOnCreateWorkspace) {
             return suggestedContextURLs.map(
@@ -44,8 +46,8 @@ export default function RepositoryFinder(props: RepositoryFinderProps) {
             );
         }
 
-        return suggestedRepos || [];
-    }, [suggestedContextURLs, suggestedRepos, includeProjectsOnCreateWorkspace]);
+        return repos;
+    }, [includeProjectsOnCreateWorkspace, repos, suggestedContextURLs]);
 
     // TODO: remove this once includeProjectsOnCreateWorkspace is fully enabled
     useEffect(() => {
@@ -60,7 +62,7 @@ export default function RepositoryFinder(props: RepositoryFinderProps) {
     const handleSelectionChange = useCallback(
         (selectedID: string) => {
             // selectedId is either projectId or repo url
-            const matchingSuggestion = suggestedRepos?.find((repo) => {
+            const matchingSuggestion = normalizedRepos?.find((repo) => {
                 if (repo.projectId) {
                     return repo.projectId === selectedID;
                 }
@@ -75,12 +77,12 @@ export default function RepositoryFinder(props: RepositoryFinderProps) {
             // If we have no matching suggestion, it's a context URL they typed/pasted in, so just use that as the url
             props.setSelection(selectedID);
         },
-        [props, suggestedRepos],
+        [props, normalizedRepos],
     );
 
     // Resolve the selected context url & project id props to a suggestion entry
     const selectedSuggestion = useMemo(() => {
-        let match = suggestedRepos?.find((repo) => {
+        let match = normalizedRepos?.find((repo) => {
             if (props.selectedProjectID) {
                 return repo.projectId === props.selectedProjectID;
             }
@@ -105,15 +107,16 @@ export default function RepositoryFinder(props: RepositoryFinderProps) {
         }
 
         return match;
-    }, [props.selectedProjectID, props.selectedContextURL, suggestedRepos]);
+    }, [normalizedRepos, props.selectedContextURL, props.selectedProjectID]);
 
     const getElements = useCallback(
         (searchString: string) => {
-            const results = filterRepos(searchString, suggestedRepoURLs);
-
+            // TODO: remove once includeProjectsOnCreateWorkspace is fully enabled
             // With the flag off, we only want to show the suggestedContextURLs
             if (!includeProjectsOnCreateWorkspace) {
-                return results.map(
+                // w/o the flag on we still need to filter the repo as the search string changes
+                const filteredResults = filterRepos(searchString, normalizedRepos);
+                return filteredResults.map(
                     (repo) =>
                         ({
                             id: repo.url,
@@ -133,27 +136,29 @@ export default function RepositoryFinder(props: RepositoryFinderProps) {
                 );
             }
 
-            // Otherwise we show the suggestedRepos
-            return results.map((repo) => {
+            // Otherwise we show the suggestedRepos (already filtered)
+            return normalizedRepos.map((repo) => {
                 return {
                     id: repo.projectId || repo.url,
                     element: <SuggestedRepositoryOption repo={repo} />,
                     isSelectable: true,
                 } as DropDown2Element;
             });
         },
-        [includeProjectsOnCreateWorkspace, suggestedRepoURLs],
+        [includeProjectsOnCreateWorkspace, normalizedRepos],
     );
 
     return (
         <DropDown2
             getElements={getElements}
             expanded={!props.selectedContextURL}
+            // we use this to track the search string so we can search for repos via the api
             onSelectionChange={handleSelectionChange}
             disabled={props.disabled}
             // Only consider the isLoading prop if we're including projects in list
-            loading={isLoading && includeProjectsOnCreateWorkspace}
+            loading={(isLoading || isSearching) && includeProjectsOnCreateWorkspace}
             searchPlaceholder="Paste repository URL or type to find suggestions"
+            onSearchChange={setSearchString}
         >
             <DropDown2SelectedElement
                 icon={RepositorySVG}
@@ -201,7 +206,6 @@ const SuggestedRepositoryOption: FC<SuggestedRepositoryOptionProps> = ({ repo })
                 </>
             )}
 
-            {/* TODO: refine some Text* components a bit to make it easy to set the right colors for dark/light mode */}
             <span className="text-sm whitespace-nowrap truncate overflow-ellipsis text-gray-500 dark:text-gray-400">
                 {stripOffProtocol(repo.url)}
             </span>
@@ -238,28 +242,6 @@ function saveSearchData(searchData: string[]): void {
     }
 }
 
-function filterRepos(searchString: string, suggestedRepos: SuggestedRepository[]) {
-    let results = suggestedRepos;
-    const normalizedSearchString = searchString.trim().toLowerCase();
-
-    if (normalizedSearchString.length > 1) {
-        results = suggestedRepos.filter((r) => {
-            return `${r.url}${r.projectName || ""}`.toLowerCase().includes(normalizedSearchString);
-        });
-
-        if (results.length === 0) {
-            try {
-                // If the normalizedSearchString is a URL, and it's not present in the proposed results, "artificially" add it here.
-                new URL(normalizedSearchString);
-                results.push({ url: normalizedSearchString });
-            } catch {}
-        }
-    }
-
-    // Limit what we show to 200 results
-    return results.length > 200 ? results.slice(0, 200) : results;
-}
-
 function stripOffProtocol(url: string): string {
     if (!url.startsWith("http")) {
         return url;

components/dashboard/src/data/featureflag-query.ts

@@ -28,6 +28,7 @@ const featureFlags = {
     enabledOrbitalDiscoveries: "",
     newProjectIncrementalRepoSearchBBS: false,
     includeProjectsOnCreateWorkspace: false,
+    repositoryFinderSearch: false,
 };
 
 type FeatureFlags = typeof featureFlags;