[content-service] download s3 content using s5cmd

Fixes ENG-884

Author: kylos101

components/content-service/pkg/storage/s3.go

@@ -9,6 +9,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 
@@ -281,12 +282,12 @@ func (s3st *s3Storage) BackupObject(name string) string {
 
 // Download implements DirectAccess
 func (s3st *s3Storage) Download(ctx context.Context, destination string, name string, mappings []archive.IDMapping) (found bool, err error) {
-	return s3st.download(ctx, destination, s3st.objectName(name), mappings)
+	return s3st.downloadS5cmd(ctx, destination, s3st.objectName(name), mappings)
 }
 
 // DownloadSnapshot implements DirectAccess
 func (s3st *s3Storage) DownloadSnapshot(ctx context.Context, destination string, name string, mappings []archive.IDMapping) (found bool, err error) {
-	return s3st.download(ctx, destination, name, mappings)
+	return s3st.downloadS5cmd(ctx, destination, name, mappings)
 }
 
 func (s3st *s3Storage) download(ctx context.Context, destination string, obj string, mappings []archive.IDMapping) (found bool, err error) {
@@ -323,6 +324,43 @@ func (s3st *s3Storage) download(ctx context.Context, destination string, obj str
 	return true, nil
 }
 
+// download object using s5cmd
+func (s3st *s3Storage) downloadS5cmd(ctx context.Context, destination string, obj string, mappings []archive.IDMapping) (found bool, err error) {
+	tempFile, err := os.CreateTemp("", "temporal-s3-file")
+	if err != nil {
+		return true, xerrors.Errorf("creating temporal file: %s", err.Error())
+	}
+	tempFile.Close()
+
+	args := []string{
+		"--numworkers", "20",
+		"cp", "--part-size", "25",
+		destination,
+		tempFile.Name(),
+	}
+	cmd := exec.Command("s5cmd", args...)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		log.WithError(err).WithField("out", string(out)).Error("unexpected error downloading file")
+		return true, xerrors.Errorf("unexpected error downloading file")
+	}
+
+	tempFile, err = os.Open(tempFile.Name())
+	if err != nil {
+		return true, xerrors.Errorf("unexpected error opening downloaded file")
+	}
+
+	defer os.Remove(tempFile.Name())
+	defer tempFile.Close()
+
+	err = archive.ExtractTarbal(ctx, tempFile, destination, archive.WithUIDMapping(mappings), archive.WithGIDMapping(mappings))
+	if err != nil {
+		return true, xerrors.Errorf("tar %s: %s", destination, err.Error())
+	}
+
+	return true, nil
+}
+
 // EnsureExists implements DirectAccess
 func (*s3Storage) EnsureExists(ctx context.Context) error {
 	return nil

components/ws-daemon/leeway.Dockerfile

@@ -9,6 +9,11 @@ RUN apk add --no-cache curl file \
   && chmod +x runc.amd64 \
   && if ! file runc.amd64 | grep -iq "ELF 64-bit LSB pie executable"; then echo "runc.amd64 is not a binary file"; exit 1;fi
 
+RUN curl -OsSL https://github.com/peak/s5cmd/releases/download/v2.2.2/s5cmd_2.2.2_Linux-64bit.tar.gz \
+ && tar -xzvf s5cmd_2.2.2_Linux-64bit.tar.gz s5cmd \
+ && chmod +x s5cmd \
+ && if ! file s5cmd | grep -iq "ELF 64-bit LSB pie executable"; then echo "s5cmd is not a binary file"; exit 1;fi
+
 FROM ubuntu:22.04
 
 # trigger manual rebuild increasing the value
@@ -46,6 +51,7 @@ RUN apt update \
     /var/tmp/*
 
 COPY --from=dl /dl/runc.amd64 /usr/bin/runc
+COPY --from=dl /dl/s5cmd /usr/bin/s5cmd
 
 # Add gitpod user for operations (e.g. checkout because of the post-checkout hook!)
 RUN groupadd -r -g 33333 gitpod \