add soci convert feature

Signed-off-by: Arjun Raja Yogidas <[email protected]>

Author: coderbirju

cmd/nerdctl/image/image_convert.go

@@ -89,6 +89,12 @@ func convertCommand() *cobra.Command {
 	cmd.Flags().String("overlaybd-dbstr", "", "Database config string for overlaybd")
 	// #endregion
 
+	// #region soci flags
+	cmd.Flags().Bool("soci", false, "Convert image to SOCI Index V2 format.")
+	cmd.Flags().Int64("soci-min-layer-size", -1, "The minimum size of layers that will be converted to SOCI Index V2 format")
+	cmd.Flags().Int64("soci-span-size", -1, "The size of SOCI spans")
+	// #endregion
+
 	// #region generic flags
 	cmd.Flags().Bool("uncompress", false, "Convert tar.gz layers to uncompressed tar layers")
 	cmd.Flags().Bool("oci", false, "Convert Docker media types to OCI media types")
@@ -213,6 +219,21 @@ func convertOptions(cmd *cobra.Command) (types.ImageConvertOptions, error) {
 	}
 	// #endregion
 
+	// #region soci flags
+	soci, err := cmd.Flags().GetBool("soci")
+	if err != nil {
+		return types.ImageConvertOptions{}, err
+	}
+	sociMinLayerSize, err := cmd.Flags().GetInt64("soci-min-layer-size")
+	if err != nil {
+		return types.ImageConvertOptions{}, err
+	}
+	sociSpanSize, err := cmd.Flags().GetInt64("soci-span-size")
+	if err != nil {
+		return types.ImageConvertOptions{}, err
+	}
+	// #endregion
+
 	// #region generic flags
 	uncompress, err := cmd.Flags().GetBool("uncompress")
 	if err != nil {
@@ -268,6 +289,13 @@ func convertOptions(cmd *cobra.Command) (types.ImageConvertOptions, error) {
 		OverlayFsType:  overlaybdFsType,
 		OverlaydbDBStr: overlaybdDbstr,
 		// #endregion
+		// #region soci flags
+		Soci: soci,
+		SociOptions: types.SociOptions{
+			SpanSize:     sociSpanSize,
+			MinLayerSize: sociMinLayerSize,
+		},
+		// #endregion
 		// #region generic flags
 		Uncompress: uncompress,
 		Oci:        oci,

cmd/nerdctl/image/image_convert_linux_test.go

@@ -88,6 +88,23 @@ func TestImageConvert(t *testing.T) {
 				},
 				Expected: test.Expects(0, nil, nil),
 			},
+			{
+				Description: "soci",
+				Require: require.All(
+					require.Not(nerdtest.Docker),
+					nerdtest.Soci,
+				),
+				Cleanup: func(data test.Data, helpers test.Helpers) {
+					helpers.Anyhow("rmi", "-f", data.Identifier("converted-image"))
+				},
+				Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
+					return helpers.Command("image", "convert", "--soci",
+						"--soci-span-size", "2097152",
+						"--soci-min-layer-size", "20971520",
+						testutil.CommonImage, data.Identifier("converted-image"))
+				},
+				Expected: test.Expects(0, nil, nil),
+			},
 		},
 	}
 

pkg/api/types/image_types.go

@@ -19,7 +19,7 @@ package types
 import (
 	"io"
 
-	"github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 // ImageListOptions specifies options for `nerdctl image list`.
@@ -124,6 +124,12 @@ type ImageConvertOptions struct {
 	OverlaydbDBStr string
 	// #endregion
 
+	// #region soci flags
+	// Soci convert image to SOCI format. Should be used in conjunction with '--oci'
+	Soci bool
+	// SociOptions contains SOCI-specific options
+	SociOptions SociOptions
+	// #endregion
 }
 
 // ImageCryptOptions specifies options for `nerdctl image encrypt` and `nerdctl image decrypt`.
@@ -200,7 +206,7 @@ type ImagePullOptions struct {
 	// If nil, it will unpack automatically if only 1 platform is specified.
 	Unpack *bool
 	// Content for specific platforms. Empty if `--all-platforms` is true
-	OCISpecPlatform []v1.Platform
+	OCISpecPlatform []ocispec.Platform
 	// Pull mode
 	Mode string
 	// Suppress verbose output

pkg/cmd/image/convert.go

@@ -47,6 +47,7 @@ import (
 	converterutil "github.com/containerd/nerdctl/v2/pkg/imgutil/converter"
 	"github.com/containerd/nerdctl/v2/pkg/platformutil"
 	"github.com/containerd/nerdctl/v2/pkg/referenceutil"
+	"github.com/containerd/nerdctl/v2/pkg/snapshotterutil"
 )
 
 func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRawRef string, options types.ImageConvertOptions) error {
@@ -86,8 +87,9 @@ func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRa
 	zstdchunked := options.ZstdChunked
 	overlaybd := options.Overlaybd
 	nydus := options.Nydus
+	soci := options.Soci
 	var finalize func(ctx context.Context, cs content.Store, ref string, desc *ocispec.Descriptor) (*images.Image, error)
-	if estargz || zstd || zstdchunked || overlaybd || nydus {
+	if estargz || zstd || zstdchunked || overlaybd || nydus || soci {
 		convertCount := 0
 		if estargz {
 			convertCount++
@@ -104,9 +106,12 @@ func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRa
 		if nydus {
 			convertCount++
 		}
+		if soci {
+			convertCount++
+		}
 
 		if convertCount > 1 {
-			return errors.New("options --estargz, --zstdchunked, --overlaybd and --nydus lead to conflict, only one of them can be used")
+			return errors.New("options --estargz, --zstdchunked, --overlaybd, --nydus and --soci lead to conflict, only one of them can be used")
 		}
 
 		var convertFunc converter.ConvertFunc
@@ -164,6 +169,16 @@ func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRa
 				)),
 			)
 			convertType = "nydus"
+		case soci:
+			// Convert image to SOCI format
+			convertedRef, err := snapshotterutil.ConvertSoci(ctx, client, srcRef, targetRef, options.GOptions, options.Platforms, options.SociOptions)
+			if err != nil {
+				return fmt.Errorf("failed to convert image to SOCI format: %w", err)
+			}
+			res := converterutil.ConvertedImageInfo{
+				Image: convertedRef,
+			}
+			return printConvertedImage(options.Stdout, options, res)
 		}
 
 		if convertType != "overlaybd" {

pkg/snapshotterutil/sociutil.go

@@ -18,23 +18,26 @@ package snapshotterutil
 
 import (
 	"bufio"
+	"context"
+	"fmt"
 	"os"
 	"os/exec"
 	"strconv"
 	"strings"
 
+	"github.com/containerd/containerd/v2/client"
 	"github.com/containerd/log"
 
 	"github.com/containerd/nerdctl/v2/pkg/api/types"
 )
 
-// CreateSoci creates a SOCI index(`rawRef`)
-func CreateSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform bool, platforms []string, sOpts types.SociOptions) error {
+// setupSociCommand creates and sets up a SOCI command with common configuration
+func setupSociCommand(gOpts types.GlobalCommandOptions) (*exec.Cmd, error) {
 	sociExecutable, err := exec.LookPath("soci")
 	if err != nil {
 		log.L.WithError(err).Error("soci executable not found in path $PATH")
 		log.L.Info("you might consider installing soci from: https://github.com/awslabs/soci-snapshotter/blob/main/docs/install.md")
-		return err
+		return nil, err
 	}
 
 	sociCmd := exec.Command(sociExecutable)
@@ -47,7 +50,77 @@ func CreateSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform boo
 	if gOpts.Namespace != "" {
 		sociCmd.Args = append(sociCmd.Args, "--namespace", gOpts.Namespace)
 	}
-	// #endregion
+
+	return sociCmd, nil
+}
+
+// ConvertSoci converts an image to SOCI format and returns the converted image reference with digest
+func ConvertSoci(ctx context.Context, client *client.Client, srcRef string, destRef string, gOpts types.GlobalCommandOptions, platforms []string, sOpts types.SociOptions) (string, error) {
+	// Check minimum required version
+	// const minRequiredVersion = "0.9.0"
+	// if err := checkSociVersion(minRequiredVersion); err != nil {
+	// 	return err
+	// }
+
+	sociCmd, err := setupSociCommand(gOpts)
+	if err != nil {
+		return "", err
+	}
+
+	// TODO: Implement conversion logic
+	sociCmd.Args = append(sociCmd.Args, "convert")
+
+	if len(platforms) > 0 {
+		// multiple values need to be passed as separate, repeating flags in soci as it uses urfave
+		// https://github.com/urfave/cli/blob/main/docs/v2/examples/flags.md#multiple-values-per-single-flag
+		for _, p := range platforms {
+			sociCmd.Args = append(sociCmd.Args, "--platform", p)
+		}
+	}
+
+	if sOpts.SpanSize != -1 {
+		sociCmd.Args = append(sociCmd.Args, "--span-size", strconv.FormatInt(sOpts.SpanSize, 10))
+	}
+
+	if sOpts.MinLayerSize != -1 {
+		sociCmd.Args = append(sociCmd.Args, "--min-layer-size", strconv.FormatInt(sOpts.MinLayerSize, 10))
+	}
+
+	sociCmd.Args = append(sociCmd.Args, srcRef, destRef)
+
+	log.L.Infof("Converting image from %s to %s using SOCI format", srcRef, destRef)
+
+	err = processSociIO(sociCmd)
+	if err != nil {
+		return "", err
+	}
+	err = sociCmd.Wait()
+	if err != nil {
+		return "", err
+	}
+
+	// Get the converted image's digest
+	img, err := client.GetImage(ctx, destRef)
+	if err != nil {
+		return "", fmt.Errorf("failed to get converted image: %w", err)
+	}
+
+	// Return the full reference with digest
+	return fmt.Sprintf("%s@%s", destRef, img.Target().Digest), nil
+}
+
+// CreateSoci creates a SOCI index(`rawRef`)
+func CreateSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform bool, platforms []string, sOpts types.SociOptions) error {
+	// Check minimum required version
+	// const minRequiredVersion = "0.9.0"
+	// if err := checkSociVersion(minRequiredVersion); err != nil {
+	// 	return err
+	// }
+
+	sociCmd, err := setupSociCommand(gOpts)
+	if err != nil {
+		return err
+	}
 
 	// Global flags have to be put before subcommand before soci upgrades to urfave v3.
 	// https://github.com/urfave/cli/issues/1113
@@ -73,7 +146,7 @@ func CreateSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform boo
 	// --timeout, --debug, --content-store
 	sociCmd.Args = append(sociCmd.Args, rawRef)
 
-	log.L.Debugf("running %s %v", sociExecutable, sociCmd.Args)
+	log.L.Debugf("running soci %v", sociCmd.Args)
 
 	err = processSociIO(sociCmd)
 	if err != nil {
@@ -88,25 +161,11 @@ func CreateSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform boo
 func PushSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform bool, platforms []string) error {
 	log.L.Debugf("pushing SOCI index: %s", rawRef)
 
-	sociExecutable, err := exec.LookPath("soci")
+	sociCmd, err := setupSociCommand(gOpts)
 	if err != nil {
-		log.L.WithError(err).Error("soci executable not found in path $PATH")
-		log.L.Info("you might consider installing soci from: https://github.com/awslabs/soci-snapshotter/blob/main/docs/install.md")
 		return err
 	}
 
-	sociCmd := exec.Command(sociExecutable)
-	sociCmd.Env = os.Environ()
-
-	// #region for global flags.
-	if gOpts.Address != "" {
-		sociCmd.Args = append(sociCmd.Args, "--address", gOpts.Address)
-	}
-	if gOpts.Namespace != "" {
-		sociCmd.Args = append(sociCmd.Args, "--namespace", gOpts.Namespace)
-	}
-	// #endregion
-
 	// Global flags have to be put before subcommand before soci upgrades to urfave v3.
 	// https://github.com/urfave/cli/issues/1113
 	sociCmd.Args = append(sociCmd.Args, "push")
@@ -131,7 +190,7 @@ func PushSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform bool,
 	}
 	sociCmd.Args = append(sociCmd.Args, rawRef)
 
-	log.L.Debugf("running %s %v", sociExecutable, sociCmd.Args)
+	log.L.Debugf("running soci %v", sociCmd.Args)
 
 	err = processSociIO(sociCmd)
 	if err != nil {
@@ -140,6 +199,55 @@ func PushSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform bool,
 	return sociCmd.Wait()
 }
 
+// checkSociVersion checks if the installed SOCI version meets the minimum required version
+// func checkSociVersion(minVersion string) error {
+// 	sociExecutable, err := exec.LookPath("soci")
+// 	if err != nil {
+// 		log.L.WithError(err).Error("soci executable not found in path $PATH")
+// 		log.L.Info("you might consider installing soci from: https://github.com/awslabs/soci-snapshotter/blob/main/docs/install.md")
+// 		return err
+// 	}
+
+// 	cmd := exec.Command(sociExecutable, "--version")
+// 	output, err := cmd.Output()
+// 	if err != nil {
+// 		return fmt.Errorf("failed to get soci version: %w", err)
+// 	}
+
+// 	// Parse version from output
+// 	// Example output format: "soci version v0.9.0 737f61a3db40c386f997c1f126344158aa3ad43c"
+// 	versionStr := strings.TrimSpace(string(output))
+// 	parts := strings.Fields(versionStr)
+// 	if len(parts) < 3 {
+// 		return fmt.Errorf("unexpected soci version output format: %s", versionStr)
+// 	}
+
+// 	// Extract version number without 'v' prefix
+// 	installedVersion := strings.TrimPrefix(parts[2], "v")
+
+// 	// Compare versions
+// 	v1, err := semver.NewVersion(installedVersion)
+// 	if err != nil {
+// 		return fmt.Errorf("failed to parse installed version %s: %w", installedVersion, err)
+// 	}
+
+// 	v2, err := semver.NewVersion(minVersion)
+// 	if err != nil {
+// 		return fmt.Errorf("failed to parse minimum required version %s: %w", minVersion, err)
+// 	}
+
+// 	if v1.LessThan(v2) {
+// 		return fmt.Errorf("installed soci version %s is older than required version %s", installedVersion, minVersion)
+// 	}
+
+// 	// Log the full version info including commit hash for debugging purposes
+// 	if len(parts) > 3 {
+// 		log.L.Debugf("soci version: %s (commit: %s)", installedVersion, parts[3])
+// 	}
+
+// 	return nil
+// }
+
 func processSociIO(sociCmd *exec.Cmd) error {
 	stdout, err := sociCmd.StdoutPipe()
 	if err != nil {