Skip to content

[RFC-0003] Select layer by OCI media type #871

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Aug 24, 2022
Merged

[RFC-0003] Select layer by OCI media type #871

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
11dc0a3
Select layer by OCI media type
stefanprodan Aug 22, 2022
49dc309
Add tests for OCI layer selector
stefanprodan Aug 24, 2022
e5cb32b
Add OCI layer selector to API docs
stefanprodan Aug 24, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions api/v1beta2/ocirepository_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,11 @@ type OCIRepositorySpec struct {
// +optional
Reference *OCIRepositoryRef `json:"ref,omitempty"`

// LayerSelector specifies which layer should be extracted from the OCI artifact.
// When not specified, the first layer found in the artifact is selected.
// +optional
LayerSelector *OCILayerSelector `json:"layerSelector,omitempty"`

// The provider used for authentication, can be 'aws', 'azure', 'gcp' or 'generic'.
// When not specified, defaults to 'generic'.
// +kubebuilder:validation:Enum=generic;aws;azure;gcp
Expand Down Expand Up @@ -130,6 +135,15 @@ type OCIRepositoryRef struct {
Tag string `json:"tag,omitempty"`
}

// OCILayerSelector specifies which layer should be extracted from an OCI Artifact
type OCILayerSelector struct {
// MediaType specifies the OCI media type of the layer
// which should be extracted from the OCI Artifact. The
// first layer matching this type is selected.
// +optional
MediaType string `json:"mediaType,omitempty"`
}

// OCIRepositoryVerification verifies the authenticity of an OCI Artifact
type OCIRepositoryVerification struct {
// Provider specifies the technology used to sign the OCI Artifact.
Expand Down Expand Up @@ -192,6 +206,15 @@ func (in *OCIRepository) GetArtifact() *Artifact {
return in.Status.Artifact
}

// GetLayerMediaType returns the media type layer selector if found in spec.
func (in *OCIRepository) GetLayerMediaType() string {
if in.Spec.LayerSelector == nil {
return ""
}

return in.Spec.LayerSelector.MediaType
}

// +genclient
// +genclient:Namespaced
// +kubebuilder:storageversion
Expand Down
20 changes: 20 additions & 0 deletions api/v1beta2/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 11 additions & 0 deletions config/crd/bases/source.toolkit.fluxcd.io_ocirepositories.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,17 @@ spec:
interval:
description: The interval at which to check for image updates.
type: string
layerSelector:
description: LayerSelector specifies which layer should be extracted
from the OCI artifact. When not specified, the first layer found
in the artifact is selected.
properties:
mediaType:
description: MediaType specifies the OCI media type of the layer
which should be extracted from the OCI Artifact. The first layer
matching this type is selected.
type: string
type: object
provider:
default: generic
description: The provider used for authentication, can be 'aws', 'azure',
Expand Down
36 changes: 35 additions & 1 deletion controllers/ocirepository_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (
"github.com/google/go-containerregistry/pkg/authn/k8schain"
"github.com/google/go-containerregistry/pkg/crane"
"github.com/google/go-containerregistry/pkg/name"
gcrv1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/remote"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
Expand Down Expand Up @@ -433,7 +434,40 @@ func (r *OCIRepositoryReconciler) reconcileSource(ctx context.Context, obj *sour
return sreconcile.ResultEmpty, e
}

blob, err := layers[0].Compressed()
var layer gcrv1.Layer

switch {
case obj.GetLayerMediaType() != "":
var found bool
for i, l := range layers {
md, err := l.MediaType()
if err != nil {
e := serror.NewGeneric(
fmt.Errorf("failed to determine the media type of layer[%v] from artifact: %w", i, err),
sourcev1.OCILayerOperationFailedReason,
)
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())
return sreconcile.ResultEmpty, e
}
if string(md) == obj.GetLayerMediaType() {
layer = layers[i]
found = true
break
}
}
if !found {
e := serror.NewGeneric(
fmt.Errorf("failed to find layer with media type '%s' in artifact", obj.GetLayerMediaType()),
sourcev1.OCILayerOperationFailedReason,
)
conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())
return sreconcile.ResultEmpty, e
}
default:
layer = layers[0]
}

blob, err := layer.Compressed()
if err != nil {
e := serror.NewGeneric(
fmt.Errorf("failed to extract the first layer from artifact: %w", err),
Expand Down
117 changes: 112 additions & 5 deletions controllers/ocirepository_controller_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,13 +80,15 @@ func TestOCIRepository_Reconcile(t *testing.T) {
tag string
semver string
digest string
mediaType string
assertArtifact []artifactFixture
}{
{
name: "public tag",
url: podinfoVersions["6.1.6"].url,
tag: podinfoVersions["6.1.6"].tag,
digest: podinfoVersions["6.1.6"].digest.Hex,
name: "public tag",
url: podinfoVersions["6.1.6"].url,
tag: podinfoVersions["6.1.6"].tag,
digest: podinfoVersions["6.1.6"].digest.Hex,
mediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip",
assertArtifact: []artifactFixture{
{
expectedPath: "kustomize/deployment.yaml",
Expand Down Expand Up @@ -142,7 +144,9 @@ func TestOCIRepository_Reconcile(t *testing.T) {
if tt.semver != "" {
obj.Spec.Reference.SemVer = tt.semver
}

if tt.mediaType != "" {
obj.Spec.LayerSelector = &sourcev1.OCILayerSelector{MediaType: tt.mediaType}
}
g.Expect(testEnv.Create(ctx, obj)).To(Succeed())

key := client.ObjectKey{Name: obj.Name, Namespace: obj.Namespace}
Expand Down Expand Up @@ -244,6 +248,109 @@ func TestOCIRepository_Reconcile(t *testing.T) {
}
}

func TestOCIRepository_Reconcile_MediaType(t *testing.T) {
g := NewWithT(t)

// Registry server with public images
tmpDir := t.TempDir()
regServer, err := setupRegistryServer(ctx, tmpDir, registryOptions{})
if err != nil {
g.Expect(err).ToNot(HaveOccurred())
}

podinfoVersions, err := pushMultiplePodinfoImages(regServer.registryHost, "6.1.4", "6.1.5", "6.1.6")

tests := []struct {
name string
url string
tag string
mediaType string
wantErr bool
}{
{
name: "Works with no media type",
url: podinfoVersions["6.1.4"].url,
tag: podinfoVersions["6.1.4"].tag,
},
{
name: "Works with Flux CLI media type",
url: podinfoVersions["6.1.5"].url,
tag: podinfoVersions["6.1.5"].tag,
mediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip",
},
{
name: "Fails with unknown media type",
url: podinfoVersions["6.1.6"].url,
tag: podinfoVersions["6.1.6"].tag,
mediaType: "application/invalid.tar.gzip",
wantErr: true,
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {

g := NewWithT(t)

ns, err := testEnv.CreateNamespace(ctx, "ocirepository-mediatype-test")
g.Expect(err).ToNot(HaveOccurred())
defer func() { g.Expect(testEnv.Delete(ctx, ns)).To(Succeed()) }()

obj := &sourcev1.OCIRepository{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "ocirepository-reconcile",
Namespace: ns.Name,
},
Spec: sourcev1.OCIRepositorySpec{
URL: tt.url,
Interval: metav1.Duration{Duration: 60 * time.Minute},
Reference: &sourcev1.OCIRepositoryRef{
Tag: tt.tag,
},
LayerSelector: &sourcev1.OCILayerSelector{
MediaType: tt.mediaType,
},
},
}

g.Expect(testEnv.Create(ctx, obj)).To(Succeed())

key := client.ObjectKey{Name: obj.Name, Namespace: obj.Namespace}

// Wait for the finalizer to be set
g.Eventually(func() bool {
if err := testEnv.Get(ctx, key, obj); err != nil {
return false
}
return len(obj.Finalizers) > 0
}, timeout).Should(BeTrue())

// Wait for the object to be reconciled
g.Eventually(func() bool {
if err := testEnv.Get(ctx, key, obj); err != nil {
return false
}
readyCondition := conditions.Get(obj, meta.ReadyCondition)
return readyCondition != nil
}, timeout).Should(BeTrue())

g.Expect(conditions.IsReady(obj)).To(BeIdenticalTo(!tt.wantErr))
if tt.wantErr {
g.Expect(conditions.Get(obj, meta.ReadyCondition).Message).Should(ContainSubstring("failed to find layer with media type"))
}

// Wait for the object to be deleted
g.Expect(testEnv.Delete(ctx, obj)).To(Succeed())
g.Eventually(func() bool {
if err := testEnv.Get(ctx, key, obj); err != nil {
return apierrors.IsNotFound(err)
}
return false
}, timeout).Should(BeTrue())
})
}
}

func TestOCIRepository_reconcileSource_authStrategy(t *testing.T) {
type secretOptions struct {
username string
Expand Down
65 changes: 65 additions & 0 deletions docs/api/source.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -968,6 +968,21 @@ defaults to the latest tag.</p>
</tr>
<tr>
<td>
<code>layerSelector</code><br>
<em>
<a href="#source.toolkit.fluxcd.io/v1beta2.OCILayerSelector">
OCILayerSelector
</a>
</em>
</td>
<td>
<em>(Optional)</em>
<p>LayerSelector specifies which layer should be extracted from the OCI artifact.
When not specified, the first layer found in the artifact is selected.</p>
</td>
</tr>
<tr>
<td>
<code>provider</code><br>
<em>
string
Expand Down Expand Up @@ -2529,6 +2544,41 @@ string
</table>
</div>
</div>
<h3 id="source.toolkit.fluxcd.io/v1beta2.OCILayerSelector">OCILayerSelector
</h3>
<p>
(<em>Appears on:</em>
<a href="#source.toolkit.fluxcd.io/v1beta2.OCIRepositorySpec">OCIRepositorySpec</a>)
</p>
<p>OCILayerSelector specifies which layer should be extracted from an OCI Artifact</p>
<div class="md-typeset__scrollwrap">
<div class="md-typeset__table">
<table>
<thead>
<tr>
<th>Field</th>
<th>Description</th>
</tr>
</thead>
<tbody>
<tr>
<td>
<code>mediaType</code><br>
<em>
string
</em>
</td>
<td>
<em>(Optional)</em>
<p>MediaType specifies the OCI media type of the layer
which should be extracted from the OCI Artifact. The
first layer matching this type is selected.</p>
</td>
</tr>
</tbody>
</table>
</div>
</div>
<h3 id="source.toolkit.fluxcd.io/v1beta2.OCIRepositoryRef">OCIRepositoryRef
</h3>
<p>
Expand Down Expand Up @@ -2634,6 +2684,21 @@ defaults to the latest tag.</p>
</tr>
<tr>
<td>
<code>layerSelector</code><br>
<em>
<a href="#source.toolkit.fluxcd.io/v1beta2.OCILayerSelector">
OCILayerSelector
</a>
</em>
</td>
<td>
<em>(Optional)</em>
<p>LayerSelector specifies which layer should be extracted from the OCI artifact.
When not specified, the first layer found in the artifact is selected.</p>
</td>
</tr>
<tr>
<td>
<code>provider</code><br>
<em>
string
Expand Down
Loading