[content-service] remove s5cmd integration (#18863)

* [ws-daemon] log download and extract times

* Revert "[content-service] log duration for s3 download and tar extract (#18829)"

This reverts commit ac8d3e2.

* Revert "[content-service] download s3 content using s5cmd (#18783)"

This reverts commit 4fb2677.

Author: kylos101

components/content-service/pkg/storage/s3.go

@@ -9,10 +9,8 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/gitpod-io/gitpod/common-go/log"
 	"github.com/gitpod-io/gitpod/content-service/pkg/archive"
@@ -291,48 +289,36 @@ func (s3st *s3Storage) DownloadSnapshot(ctx context.Context, destination string,
 	return s3st.download(ctx, destination, name, mappings)
 }
 
-// download object using s5cmd (prior to which we used aws sdk)
 func (s3st *s3Storage) download(ctx context.Context, destination string, obj string, mappings []archive.IDMapping) (found bool, err error) {
-	tempFile, err := os.CreateTemp("", "temporal-s3-file")
+	downloader := s3manager.NewDownloader(s3st.client, func(d *s3manager.Downloader) {
+		d.Concurrency = defaultCopyConcurrency
+		d.PartSize = defaultPartSize * megabytes
+		d.BufferProvider = s3manager.NewPooledBufferedWriterReadFromProvider(25 * megabytes)
+	})
+
+	s3File, err := os.CreateTemp("", "temporal-s3-file")
 	if err != nil {
 		return true, xerrors.Errorf("creating temporal file: %s", err.Error())
 	}
-	tempFile.Close()
-
-	args := []string{
-		"cp",
-		// # of file parts to download at once
-		"--concurrency", "20",
-		// size in MB of each part
-		"--part-size", "25",
-		destination,
-		tempFile.Name(),
-	}
-	cmd := exec.Command("s5cmd", args...)
-	downloadStart := time.Now()
-	out, err := cmd.CombinedOutput()
+	defer os.Remove(s3File.Name())
+
+	_, err = downloader.Download(ctx, s3File, &s3.GetObjectInput{
+		Bucket: aws.String(s3st.Config.Bucket),
+		Key:    aws.String(obj),
+	})
 	if err != nil {
-		log.WithError(err).WithField("out", string(out)).Error("unexpected error downloading file")
-		return false, xerrors.Errorf("unexpected error downloading file")
+		return false, err
 	}
-	downloadDuration := time.Since(downloadStart)
-	log.WithField("downloadDuration", downloadDuration.String()).Info("S3 download duration")
 
-	tempFile, err = os.Open(tempFile.Name())
+	_, err = s3File.Seek(0, 0)
 	if err != nil {
-		return true, xerrors.Errorf("unexpected error opening downloaded file")
+		return false, err
 	}
 
-	defer os.Remove(tempFile.Name())
-	defer tempFile.Close()
-
-	extractStart := time.Now()
-	err = archive.ExtractTarbal(ctx, tempFile, destination, archive.WithUIDMapping(mappings), archive.WithGIDMapping(mappings))
+	err = archive.ExtractTarbal(ctx, s3File, destination, archive.WithUIDMapping(mappings), archive.WithGIDMapping(mappings))
 	if err != nil {
 		return true, xerrors.Errorf("tar %s: %s", destination, err.Error())
 	}
-	extractDuration := time.Since(extractStart)
-	log.WithField("extractdDuration", extractDuration.String()).Info("tarbar extraction duration")
 
 	return true, nil
 }

components/ws-daemon/leeway.Dockerfile

@@ -9,11 +9,6 @@ RUN apk add --no-cache curl file \
   && chmod +x runc.amd64 \
   && if ! file runc.amd64 | grep -iq "ELF 64-bit LSB pie executable"; then echo "runc.amd64 is not a binary file"; exit 1;fi
 
-RUN curl -OsSL https://github.com/peak/s5cmd/releases/download/v2.2.2/s5cmd_2.2.2_Linux-64bit.tar.gz \
- && tar -xzvf s5cmd_2.2.2_Linux-64bit.tar.gz s5cmd \
- && chmod +x s5cmd \
- && if ! file s5cmd | grep -iq "ELF 64-bit LSB executable"; then echo "s5cmd is not a binary file"; exit 1;fi
-
 FROM ubuntu:22.04
 
 # trigger manual rebuild increasing the value
@@ -51,7 +46,6 @@ RUN apt update \
     /var/tmp/*
 
 COPY --from=dl /dl/runc.amd64 /usr/bin/runc
-COPY --from=dl /dl/s5cmd /usr/bin/s5cmd
 
 # Add gitpod user for operations (e.g. checkout because of the post-checkout hook!)
 RUN groupadd -r -g 33333 gitpod \

components/ws-daemon/pkg/content/initializer.go

@@ -406,12 +406,15 @@ func (rs *remoteContentStorage) Download(ctx context.Context, destination string
 		"-o", tempFile.Name(),
 	}
 
+	downloadStart := time.Now()
 	cmd := exec.Command("aria2c", args...)
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		log.WithError(err).WithField("out", string(out)).Error("unexpected error downloading file")
 		return true, xerrors.Errorf("unexpected error downloading file")
 	}
+	downloadDuration := time.Since(downloadStart)
+	log.WithField("downloadDuration", downloadDuration.String()).Info("aria2c download duration")
 
 	tempFile, err = os.Open(tempFile.Name())
 	if err != nil {
@@ -421,10 +424,13 @@ func (rs *remoteContentStorage) Download(ctx context.Context, destination string
 	defer os.Remove(tempFile.Name())
 	defer tempFile.Close()
 
+	extractStart := time.Now()
 	err = archive.ExtractTarbal(ctx, tempFile, destination, archive.WithUIDMapping(mappings), archive.WithGIDMapping(mappings))
 	if err != nil {
 		return true, xerrors.Errorf("tar %s: %s", destination, err.Error())
 	}
+	extractDuration := time.Since(extractStart)
+	log.WithField("extractDuration", extractDuration.String()).Info("extract tarbal duration")
 
 	return true, nil
 }