Merge pull request #17 from coderbirju/add-soci-convert

add soci convert feature

Author: coderbirju

.github/workflows/ghcr-image-build-and-publish.yml

@@ -61,10 +61,12 @@ jobs:
       # Build and push Docker image with Buildx (don't push on PR)
       # https://github.com/docker/build-push-action
       - name: Build and push Docker image
-        uses: docker/build-push-action@1dc73863535b631f98b2378be8619f83b136f4a0  # v6.17.0
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83  # v6.18.0
         with:
           context: .
           platforms: linux/amd64,linux/arm64
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          secrets: |
+            github_token=${{ secrets.GITHUB_TOKEN }}

.github/workflows/job-test-dependencies.yml

@@ -39,6 +39,8 @@ jobs:
         uses: crazy-max/ghaction-github-runtime@3cb05d89e1f492524af3d41a1c98c83bc3025124  # v3.1.0
 
       - name: "Run: build dependencies for the integration test environment image"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           # Cache is sharded per-architecture
           arch=${{ env.RUNNER_ARCH == 'ARM64' && 'arm64' || 'amd64' }}
@@ -49,6 +51,7 @@ jobs:
             args=(--build-arg CONTAINERD_VERSION=${{ inputs.containerd-version }})
           fi
           docker buildx build \
+            --secret id=github_token,env=GITHUB_TOKEN \
             --cache-to type=gha,compression=zstd,mode=max,scope=test-integration-dependencies-"$arch" \
             --cache-from type=gha,scope=test-integration-dependencies-"$arch" \
             --target build-dependencies "${args[@]}" .

.github/workflows/job-test-in-container.yml

@@ -86,6 +86,8 @@ jobs:
           canary::build::integration
       - if: ${{ ! inputs.canary }}
         name: "Init: prepare test image"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           buildargs=()
           # If the runner is old, use old ubuntu inside the container as well
@@ -104,6 +106,7 @@ jobs:
           arch=${{ env.RUNNER_ARCH == 'ARM64' && 'arm64' || 'amd64' }}
           docker buildx create --name with-gha --use
           docker buildx build \
+            --secret id=github_token,env=GITHUB_TOKEN \
             --output=type=docker \
             --cache-from type=gha,scope=test-integration-dependencies-"$arch" \
             -t "$target" --target "$target" \

.github/workflows/job-test-in-lima.yml

@@ -31,7 +31,7 @@ jobs:
           fetch-depth: 1
 
       - name: "Init: lima"
-        uses: lima-vm/lima-actions/setup@be564a1408f84557d067b099a475652288074b2e  # v1.0.0
+        uses: lima-vm/lima-actions/setup@03b96d61959e83b2c737e44162c3088e81de0886  # v1.0.1
         id: lima-actions-setup
 
       - name: "Init: Cache"
@@ -79,6 +79,8 @@ jobs:
         uses: crazy-max/ghaction-github-runtime@3cb05d89e1f492524af3d41a1c98c83bc3025124  # v3.1.0
 
       - name: "Init: prepare integration tests"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           set -eux
 
@@ -88,6 +90,7 @@ jobs:
           [ "$TARGET" = "rootless" ] && TARGET=test-integration-rootless || TARGET=test-integration
           docker buildx create --name with-gha --use
           docker buildx build \
+            --secret id=github_token,env=GITHUB_TOKEN \
             --output=type=docker \
             --cache-from type=gha,scope=test-integration-dependencies-amd64 \
             -t test-integration --target "${TARGET}" \

Dockerfile

@@ -61,6 +61,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update -qq && apt-get install -qq --no-install-recommends \
   make \
   git \
+  jq \
   curl \
   dpkg-dev
 ARG TARGETARCH
@@ -75,6 +76,7 @@ RUN xx-apt-get update -qq && xx-apt-get install -qq --no-install-recommends \
   pkg-config
 RUN git config --global advice.detachedHead false
 ADD hack/git-checkout-tag-with-hash.sh /usr/local/bin/
+ADD hack/scripts/lib.sh /usr/local/bin/http::helper
 
 FROM build-base AS build-containerd
 ARG TARGETARCH
@@ -174,10 +176,11 @@ RUN cd /out/lib/systemd/system && \
   echo "" >> buildkit.service && \
   echo "# This file was converted from containerd.service, with \`sed -E '${sedcomm}'\`" >> buildkit.service
 ARG STARGZ_SNAPSHOTTER_VERSION
-RUN STARGZ_SNAPSHOTTER_VERSION=${STARGZ_SNAPSHOTTER_VERSION%%@*}; \
+RUN --mount=type=secret,id=github_token,env=GITHUB_TOKEN \
+  STARGZ_SNAPSHOTTER_VERSION=${STARGZ_SNAPSHOTTER_VERSION%%@*}; \
   fname="stargz-snapshotter-${STARGZ_SNAPSHOTTER_VERSION}-${TARGETOS:-linux}-${TARGETARCH:-amd64}.tar.gz" && \
   curl -o "${fname}" -fsSL --proto '=https' --tlsv1.2 "https://github.com/containerd/stargz-snapshotter/releases/download/${STARGZ_SNAPSHOTTER_VERSION}/${fname}" && \
-  curl -o "stargz-snapshotter.service" -fsSL --proto '=https' --tlsv1.2 "https://raw.githubusercontent.com/containerd/stargz-snapshotter/${STARGZ_SNAPSHOTTER_VERSION}/script/config/etc/systemd/system/stargz-snapshotter.service" && \
+  http::helper github::file containerd/stargz-snapshotter script/config/etc/systemd/system/stargz-snapshotter.service "${STARGZ_SNAPSHOTTER_VERSION}" > "stargz-snapshotter.service" && \
   grep "${fname}" "/SHA256SUMS.d/stargz-snapshotter-${STARGZ_SNAPSHOTTER_VERSION}" | sha256sum -c - && \
   grep "stargz-snapshotter.service" "/SHA256SUMS.d/stargz-snapshotter-${STARGZ_SNAPSHOTTER_VERSION}" | sha256sum -c - && \
   tar xzf "${fname}" -C /out/bin && \
@@ -245,6 +248,10 @@ RUN ROOTLESSKIT_VERSION=${ROOTLESSKIT_VERSION%%@*}; \
 ARG GOMODJAIL_VERSION
 COPY --from=build-gomodjail /out/${TARGETARCH:-amd64}/* /out/bin/
 RUN echo "- gomodjail: ${GOMODJAIL_VERSION}" >> /out/share/doc/nerdctl-full/README.md
+ARG CONTAINERIZED_SYSTEMD_VERSION
+RUN --mount=type=secret,id=github_token,env=GITHUB_TOKEN \
+  http::helper github::file AkihiroSuda/containerized-systemd docker-entrypoint.sh "${CONTAINERIZED_SYSTEMD_VERSION}" > /docker-entrypoint.sh && \
+  chmod +x /docker-entrypoint.sh
 
 RUN echo "" >> /out/share/doc/nerdctl-full/README.md && \
   echo "## License" >> /out/share/doc/nerdctl-full/README.md && \
@@ -281,9 +288,7 @@ RUN apt-get update -qq && apt-get install -qq -y --no-install-recommends \
   iproute2 iptables \
   dbus dbus-user-session systemd systemd-sysv \
   fuse3
-ARG CONTAINERIZED_SYSTEMD_VERSION
-RUN curl -o /docker-entrypoint.sh -fsSL --proto '=https' --tlsv1.2 https://raw.githubusercontent.com/AkihiroSuda/containerized-systemd/${CONTAINERIZED_SYSTEMD_VERSION}/docker-entrypoint.sh && \
-  chmod +x /docker-entrypoint.sh
+COPY --from=build-full /docker-entrypoint.sh /docker-entrypoint.sh
 COPY --from=out-full / /usr/local/
 RUN perl -pi -e 's/multi-user.target/docker-entrypoint.target/g' /usr/local/lib/systemd/system/*.service && \
   systemctl enable containerd buildkit stargz-snapshotter && \

Makefile

@@ -253,7 +253,7 @@ TAR_OWNER0_FLAGS=--owner=0 --group=0
 TAR_FLATTEN_FLAGS=--transform 's/.*\///g'
 
 define make_artifact_full_linux
-	$(DOCKER) build --output type=tar,dest=$(CURDIR)/_output/nerdctl-full-$(VERSION_TRIMMED)-linux-$(1).tar --target out-full --platform $(1) --build-arg GO_VERSION -f $(MAKEFILE_DIR)/Dockerfile $(MAKEFILE_DIR)
+	$(DOCKER) build --secret id=github_token,env=GITHUB_TOKEN --output type=tar,dest=$(CURDIR)/_output/nerdctl-full-$(VERSION_TRIMMED)-linux-$(1).tar --target out-full --platform $(1) --build-arg GO_VERSION -f $(MAKEFILE_DIR)/Dockerfile $(MAKEFILE_DIR)
 	gzip -9 $(CURDIR)/_output/nerdctl-full-$(VERSION_TRIMMED)-linux-$(1).tar
 endef
 

cmd/nerdctl/container/container_create.go

@@ -371,7 +371,6 @@ func createOptions(cmd *cobra.Command) (types.ContainerCreateOptions, error) {
 	// #endregion
 
 	// #region for metadata flags
-	opt.NameChanged = cmd.Flags().Changed("name")
 	opt.Name, err = cmd.Flags().GetString("name")
 	if err != nil {
 		return opt, err

cmd/nerdctl/image/image_convert.go

@@ -89,6 +89,12 @@ func convertCommand() *cobra.Command {
 	cmd.Flags().String("overlaybd-dbstr", "", "Database config string for overlaybd")
 	// #endregion
 
+	// #region soci flags
+	cmd.Flags().Bool("soci", false, "Convert image to SOCI Index V2 format.")
+	cmd.Flags().Int64("soci-min-layer-size", -1, "The minimum size of layers that will be converted to SOCI Index V2 format")
+	cmd.Flags().Int64("soci-span-size", -1, "The size of SOCI spans")
+	// #endregion
+
 	// #region generic flags
 	cmd.Flags().Bool("uncompress", false, "Convert tar.gz layers to uncompressed tar layers")
 	cmd.Flags().Bool("oci", false, "Convert Docker media types to OCI media types")
@@ -213,6 +219,21 @@ func convertOptions(cmd *cobra.Command) (types.ImageConvertOptions, error) {
 	}
 	// #endregion
 
+	// #region soci flags
+	soci, err := cmd.Flags().GetBool("soci")
+	if err != nil {
+		return types.ImageConvertOptions{}, err
+	}
+	sociMinLayerSize, err := cmd.Flags().GetInt64("soci-min-layer-size")
+	if err != nil {
+		return types.ImageConvertOptions{}, err
+	}
+	sociSpanSize, err := cmd.Flags().GetInt64("soci-span-size")
+	if err != nil {
+		return types.ImageConvertOptions{}, err
+	}
+	// #endregion
+
 	// #region generic flags
 	uncompress, err := cmd.Flags().GetBool("uncompress")
 	if err != nil {
@@ -268,6 +289,13 @@ func convertOptions(cmd *cobra.Command) (types.ImageConvertOptions, error) {
 		OverlayFsType:  overlaybdFsType,
 		OverlaydbDBStr: overlaybdDbstr,
 		// #endregion
+		// #region soci flags
+		Soci: soci,
+		SociOptions: types.SociOptions{
+			SpanSize:     sociSpanSize,
+			MinLayerSize: sociMinLayerSize,
+		},
+		// #endregion
 		// #region generic flags
 		Uncompress: uncompress,
 		Oci:        oci,

cmd/nerdctl/image/image_convert_linux_test.go

@@ -88,6 +88,24 @@ func TestImageConvert(t *testing.T) {
 				},
 				Expected: test.Expects(0, nil, nil),
 			},
+			{
+				Description: "soci",
+				Require: require.All(
+					require.Not(nerdtest.Docker),
+					nerdtest.Soci,
+					nerdtest.SociVersion("0.10.0"),
+				),
+				Cleanup: func(data test.Data, helpers test.Helpers) {
+					helpers.Anyhow("rmi", "-f", data.Identifier("converted-image"))
+				},
+				Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
+					return helpers.Command("image", "convert", "--soci",
+						"--soci-span-size", "2097152",
+						"--soci-min-layer-size", "20971520",
+						testutil.CommonImage, data.Identifier("converted-image"))
+				},
+				Expected: test.Expects(0, nil, nil),
+			},
 		},
 	}
 

docs/command-reference.md

@@ -960,6 +960,11 @@ Flags:
 - `--oci`                              : convert Docker media types to OCI media types
 - `--platform=<PLATFORM>`              : convert content for a specific platform
 - `--all-platforms`                    : convert content for all platforms (default: false)
+- `--soci`: generate SOCI v2 Indices to oci images.
+*[**Note**: content is converted for all platforms by default when using this flag, use the `--platorm` flag to limit this behavior]*
+- `--soci-min-layer-size` : Span size in bytes that soci index uses to segment layer data. Default is 4 MiB.
+- `--soci-min-layer-size`: Minimum layer size in bytes to build zTOC for. Smaller layers won't have zTOC and not lazy pulled. Default is 10 MiB.
+
 
 ### :nerd_face: nerdctl image encrypt
 

docs/soci.md

@@ -45,3 +45,18 @@ For images that already have SOCI indices, see https://gallery.ecr.aws/soci-work
 nerdctl push --snapshotter=soci --soci-span-size=2097152 --soci-min-layer-size=20971520 public.ecr.aws/my-registry/my-repo:latest
 ```
 --soci-span-size and --soci-min-layer-size are two properties to customize the SOCI index. See [Command Reference](https://github.com/containerd/nerdctl/blob/377b2077bb616194a8ef1e19ccde32aa1ffd6c84/docs/command-reference.md?plain=1#L773) for further details.
+
+
+## Enable SOCI for `nerdctl image convert`
+
+| :zap: Requirement | nerdctl >= 2.2.0 |
+| ----------------- | ---------------- |
+
+| :zap: Requirement | soci-snapshotter >= 0.10.0 |
+| ----------------- | ---------------- |
+
+- Convert an image to generate SOCI Index artifacts v2. Running the `nerdctl image convert` with the `--soci` flag and a `srcImg` and `dstImg`, `nerdctl` will create the SOCI v2 indices and the new image will be present in the `dstImg` address.
+```console
+nerdctl image convert --soci --soci-span-size=2097152 --soci-min-layer-size=20971520 public.ecr.aws/my-registry/my-repo:latest public.ecr.aws/my-registry/my-repo:soci
+```
+--soci-span-size and --soci-min-layer-size are two properties to customize the SOCI index. See [Command Reference](https://github.com/containerd/nerdctl/blob/377b2077bb616194a8ef1e19ccde32aa1ffd6c84/docs/command-reference.md?plain=1#L773) for further details.

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/Masterminds/semver/v3 v3.3.1
 	github.com/Microsoft/go-winio v0.6.2
 	github.com/Microsoft/hcsshim v0.13.0
-	github.com/compose-spec/compose-go/v2 v2.6.3 //gomodjail:unconfined
+	github.com/compose-spec/compose-go/v2 v2.6.4 //gomodjail:unconfined
 	github.com/containerd/accelerated-container-image v1.3.0
 	github.com/containerd/cgroups/v3 v3.0.5 //gomodjail:unconfined
 	github.com/containerd/console v1.0.5 //gomodjail:unconfined

go.sum

@@ -19,8 +19,8 @@ github.com/cilium/ebpf v0.16.0 h1:+BiEnHL6Z7lXnlGUsXQPPAE7+kenAd4ES8MQ5min0Ok=
 github.com/cilium/ebpf v0.16.0/go.mod h1:L7u2Blt2jMM/vLAVgjxluxtBKlz3/GWjB0dMOEngfwE=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/compose-spec/compose-go/v2 v2.6.3 h1:zfW1Qp605ESySyth/zR+6yLr55XE0AiOAUlZLHKMoW0=
-github.com/compose-spec/compose-go/v2 v2.6.3/go.mod h1:vPlkN0i+0LjLf9rv52lodNMUTJF5YHVfHVGLLIP67NA=
+github.com/compose-spec/compose-go/v2 v2.6.4 h1:Gjv6x8eAhqwwWvoXIo0oZ4bDQBh0OMwdU7LUL9PDLiM=
+github.com/compose-spec/compose-go/v2 v2.6.4/go.mod h1:vPlkN0i+0LjLf9rv52lodNMUTJF5YHVfHVGLLIP67NA=
 github.com/containerd/accelerated-container-image v1.3.0 h1:sFbTgSuMboeKHa9f7MY11hWF1XxVWjFoiTsXYtOtvdU=
 github.com/containerd/accelerated-container-image v1.3.0/go.mod h1:EvKVWor6ZQNUyYp0MZm5hw4k21ropuz7EegM+m/Jb/Q=
 github.com/containerd/cgroups/v3 v3.0.5 h1:44na7Ud+VwyE7LIoJ8JTNQOa549a8543BmzaJHo6Bzo=

hack/scripts/lib.sh

@@ -226,9 +226,10 @@ github::settoken(){
 }
 
 github::request(){
-  local endpoint="$1"
+  local accept="$1"
+  local endpoint="$2"
   local args=(
-    "Accept: application/vnd.github+json"
+    "Accept: $accept"
     "X-GitHub-Api-Version: 2022-11-28"
   )
 
@@ -237,21 +238,30 @@ github::request(){
   http::get /dev/stdout https://api.github.com/"$endpoint" "${args[@]}"
 }
 
+github::file(){
+  local repo="$1"
+  local path="$2"
+  local ref="${3:-main}"
+  github::request "application/vnd.github.v3.raw" "repos/$repo/contents/$path?ref=$ref"
+}
+
 github::tags::latest(){
   local repo="$1"
-  github::request "repos/$repo/tags" | jq -rc .[0].name
+  github::request "application/vnd.github+json" "repos/$repo/tags" | jq -rc .[0].name
 }
 
 github::releases(){
   local repo="$1"
-  github::request "repos/$repo/releases" |
+  github::request "application/vnd.github+json" "repos/$repo/releases" |
     jq -rc .[]
 }
 
 github::releases::latest(){
   local repo="$1"
-  github::request "repos/$repo/releases/latest" | jq -rc .
+  github::request "application/vnd.github+json" "repos/$repo/releases/latest" | jq -rc .
 }
 
 log::init
 host::require jq tar curl shasum
+
+[[ "${1:-}" != "github"* ]] || "$@"

pkg/api/types/container_types.go

@@ -237,8 +237,6 @@ type ContainerCreateOptions struct {
 	// #endregion
 
 	// #region for metadata flags
-	// NameChanged specifies whether the name has been changed
-	NameChanged bool
 	// Name assign a name to the container
 	Name string
 	// Label set meta data on a container

pkg/api/types/image_types.go

@@ -19,7 +19,7 @@ package types
 import (
 	"io"
 
-	"github.com/opencontainers/image-spec/specs-go/v1"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 // ImageListOptions specifies options for `nerdctl image list`.
@@ -124,6 +124,12 @@ type ImageConvertOptions struct {
 	OverlaydbDBStr string
 	// #endregion
 
+	// #region soci flags
+	// Soci convert image to SOCI format.eiifc
+	Soci bool
+	// SociOptions contains SOCI-specific options
+	SociOptions SociOptions
+	// #endregion
 }
 
 // ImageCryptOptions specifies options for `nerdctl image encrypt` and `nerdctl image decrypt`.
@@ -200,7 +206,7 @@ type ImagePullOptions struct {
 	// If nil, it will unpack automatically if only 1 platform is specified.
 	Unpack *bool
 	// Content for specific platforms. Empty if `--all-platforms` is true
-	OCISpecPlatform []v1.Platform
+	OCISpecPlatform []ocispec.Platform
 	// Pull mode
 	Mode string
 	// Suppress verbose output