Add the OCI metadata to the internal artifact

Signed-off-by: Stefan Prodan <[email protected]>

Author: stefanprodan

api/v1beta2/artifact_types.go

@@ -54,6 +54,10 @@ type Artifact struct {
 	// Size is the number of bytes in the file.
 	// +optional
 	Size *int64 `json:"size,omitempty"`
+
+	// Metadata holds upstream information such as OCI annotations.
+	// +optional
+	Metadata map[string]string `json:"metadata,omitempty"`
 }
 
 // HasRevision returns if the given revision matches the current Revision of

api/v1beta2/ocirepository_types.go

@@ -17,9 +17,11 @@ limitations under the License.
 package v1beta2
 
 import (
-	"github.com/fluxcd/pkg/apis/meta"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/fluxcd/pkg/apis/meta"
 )
 
 const (

api/v1beta2/zz_generated.deepcopy.go

@@ -384,6 +384,11 @@ spec:
                       the last update of the Artifact.
                     format: date-time
                     type: string
+                  metadata:
+                    additionalProperties:
+                      type: string
+                    description: Metadata holds upstream information such as OCI annotations.
+                    type: object
                   path:
                     description: Path is the relative file path of the Artifact. It
                       can be used to locate the file in the root of the Artifact storage

config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml

@@ -559,6 +559,11 @@ spec:
                       the last update of the Artifact.
                     format: date-time
                     type: string
+                  metadata:
+                    additionalProperties:
+                      type: string
+                    description: Metadata holds upstream information such as OCI annotations.
+                    type: object
                   path:
                     description: Path is the relative file path of the Artifact. It
                       can be used to locate the file in the root of the Artifact storage
@@ -677,6 +682,12 @@ spec:
                         the last update of the Artifact.
                       format: date-time
                       type: string
+                    metadata:
+                      additionalProperties:
+                        type: string
+                      description: Metadata holds upstream information such as OCI
+                        annotations.
+                      type: object
                     path:
                       description: Path is the relative file path of the Artifact.
                         It can be used to locate the file in the root of the Artifact

config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml

@@ -432,6 +432,11 @@ spec:
                       the last update of the Artifact.
                     format: date-time
                     type: string
+                  metadata:
+                    additionalProperties:
+                      type: string
+                    description: Metadata holds upstream information such as OCI annotations.
+                    type: object
                   path:
                     description: Path is the relative file path of the Artifact. It
                       can be used to locate the file in the root of the Artifact storage

config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml

@@ -362,6 +362,11 @@ spec:
                       the last update of the Artifact.
                     format: date-time
                     type: string
+                  metadata:
+                    additionalProperties:
+                      type: string
+                    description: Metadata holds upstream information such as OCI annotations.
+                    type: object
                   path:
                     description: Path is the relative file path of the Artifact. It
                       can be used to locate the file in the root of the Artifact storage

config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml

@@ -142,6 +142,11 @@ spec:
                       the last update of the Artifact.
                     format: date-time
                     type: string
+                  metadata:
+                    additionalProperties:
+                      type: string
+                    description: Metadata holds upstream information such as OCI annotations.
+                    type: object
                   path:
                     description: Path is the relative file path of the Artifact. It
                       can be used to locate the file in the root of the Artifact storage

config/crd/bases/source.toolkit.fluxcd.io_ocirepositories.yaml

@@ -33,7 +33,6 @@ import (
 	"github.com/google/go-containerregistry/pkg/authn/k8schain"
 	"github.com/google/go-containerregistry/pkg/crane"
 	"github.com/google/go-containerregistry/pkg/name"
-	gcrv1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -110,7 +109,7 @@ var ociRepositoryFailConditions = []string{
 // ociRepositoryReconcileFunc is the function type for all the v1beta2.OCIRepository
 // (sub)reconcile functions. The type implementations are grouped and
 // executed serially to perform the complete reconcile of the object.
-type ociRepositoryReconcileFunc func(ctx context.Context, obj *sourcev1.OCIRepository, digest *gcrv1.Hash, dir string) (sreconcile.Result, error)
+type ociRepositoryReconcileFunc func(ctx context.Context, obj *sourcev1.OCIRepository, metadata *sourcev1.Artifact, dir string) (sreconcile.Result, error)
 
 // OCIRepositoryReconciler reconciles a v1beta2.OCIRepository object
 type OCIRepositoryReconciler struct {
@@ -261,16 +260,15 @@ func (r *OCIRepositoryReconciler) reconcile(ctx context.Context, obj *sourcev1.O
 	}()
 	conditions.Delete(obj, sourcev1.StorageOperationFailedCondition)
 
-	hs := gcrv1.Hash{}
 	var (
-		res    sreconcile.Result
-		resErr error
-		digest = hs.DeepCopy()
+		res      sreconcile.Result
+		resErr   error
+		metadata = sourcev1.Artifact{}
 	)
 
 	// Run the sub-reconcilers and build the result of reconciliation.
 	for _, rec := range reconcilers {
-		recResult, err := rec(ctx, obj, digest, tmpDir)
+		recResult, err := rec(ctx, obj, &metadata, tmpDir)
 		// Exit immediately on ResultRequeue.
 		if recResult == sreconcile.ResultRequeue {
 			return sreconcile.ResultRequeue, nil
@@ -286,14 +284,14 @@ func (r *OCIRepositoryReconciler) reconcile(ctx context.Context, obj *sourcev1.O
 		res = sreconcile.LowestRequeuingResult(res, recResult)
 	}
 
-	r.notify(ctx, oldObj, obj, digest, res, resErr)
+	r.notify(ctx, oldObj, obj, res, resErr)
 
 	return res, resErr
 }
 
 // reconcileSource fetches the upstream OCI artifact metadata and content.
 // If this fails, it records v1beta2.FetchFailedCondition=True on the object and returns early.
-func (r *OCIRepositoryReconciler) reconcileSource(ctx context.Context, obj *sourcev1.OCIRepository, digest *gcrv1.Hash, dir string) (sreconcile.Result, error) {
+func (r *OCIRepositoryReconciler) reconcileSource(ctx context.Context, obj *sourcev1.OCIRepository, metadata *sourcev1.Artifact, dir string) (sreconcile.Result, error) {
 	ctxTimeout, cancel := context.WithTimeout(ctx, obj.Spec.Timeout.Duration)
 	defer cancel()
 
@@ -352,9 +350,25 @@ func (r *OCIRepositoryReconciler) reconcileSource(ctx context.Context, obj *sour
 	}
 
 	// Set the internal revision to the remote digest hex
-	imgDigest.DeepCopyInto(digest)
 	revision := imgDigest.Hex
 
+	// Copy the OCI annotations to the internal artifact metadata
+	manifest, err := img.Manifest()
+	if err != nil {
+		e := serror.NewGeneric(
+			fmt.Errorf("failed to parse artifact manifest: %w", err),
+			sourcev1.OCIOperationFailedReason,
+		)
+		conditions.MarkTrue(obj, sourcev1.FetchFailedCondition, e.Reason, e.Err.Error())
+		return sreconcile.ResultEmpty, e
+	}
+
+	m := &sourcev1.Artifact{
+		Revision: revision,
+		Metadata: manifest.Annotations,
+	}
+	m.DeepCopyInto(metadata)
+
 	// Mark observations about the revision on the object
 	defer func() {
 		if !obj.GetArtifact().HasRevision(revision) {
@@ -606,7 +620,7 @@ func (r *OCIRepositoryReconciler) craneOptions(ctx context.Context,
 // The hostname of any URL in the Status of the object are updated, to ensure
 // they match the Storage server hostname of current runtime.
 func (r *OCIRepositoryReconciler) reconcileStorage(ctx context.Context,
-	obj *sourcev1.OCIRepository, _ *gcrv1.Hash, _ string) (sreconcile.Result, error) {
+	obj *sourcev1.OCIRepository, _ *sourcev1.Artifact, _ string) (sreconcile.Result, error) {
 	// Garbage collect previous advertised artifact(s) from storage
 	_ = r.garbageCollect(ctx, obj)
 
@@ -642,9 +656,9 @@ func (r *OCIRepositoryReconciler) reconcileStorage(ctx context.Context,
 // On a successful archive, the Artifact in the Status of the object is set,
 // and the symlink in the Storage is updated to its path.
 func (r *OCIRepositoryReconciler) reconcileArtifact(ctx context.Context,
-	obj *sourcev1.OCIRepository, digest *gcrv1.Hash, dir string) (sreconcile.Result, error) {
+	obj *sourcev1.OCIRepository, metadata *sourcev1.Artifact, dir string) (sreconcile.Result, error) {
 	// Calculate revision
-	revision := digest.Hex
+	revision := metadata.Revision
 
 	// Create artifact
 	artifact := r.Storage.NewArtifactFor(obj.Kind, obj, revision, fmt.Sprintf("%s.tar.gz", revision))
@@ -712,6 +726,7 @@ func (r *OCIRepositoryReconciler) reconcileArtifact(ctx context.Context,
 
 	// Record it on the object
 	obj.Status.Artifact = artifact.DeepCopy()
+	obj.Status.Artifact.Metadata = metadata.Metadata
 
 	// Update symlink on a "best effort" basis
 	url, err := r.Storage.Symlink(artifact, "latest.tar.gz")
@@ -798,7 +813,7 @@ func (r *OCIRepositoryReconciler) eventLogf(ctx context.Context,
 
 // notify emits notification related to the reconciliation.
 func (r *OCIRepositoryReconciler) notify(ctx context.Context,
-	oldObj, newObj *sourcev1.OCIRepository, digest *gcrv1.Hash, res sreconcile.Result, resErr error) {
+	oldObj, newObj *sourcev1.OCIRepository, res sreconcile.Result, resErr error) {
 	// Notify successful reconciliation for new artifact and recovery from any
 	// failure.
 	if resErr == nil && res == sreconcile.ResultSuccess && newObj.Status.Artifact != nil {
@@ -812,7 +827,7 @@ func (r *OCIRepositoryReconciler) notify(ctx context.Context,
 			oldChecksum = oldObj.GetArtifact().Checksum
 		}
 
-		message := fmt.Sprintf("stored artifact with digest '%s' from '%s'", digest.String(), newObj.Spec.URL)
+		message := fmt.Sprintf("stored artifact with digest '%s' from '%s'", newObj.Status.Artifact.Revision, newObj.Spec.URL)
 
 		// Notify on new artifact and failure recovery.
 		if oldChecksum != newObj.GetArtifact().Checksum {

controllers/ocirepository_controller.go

@@ -44,7 +44,8 @@ import (
 	"github.com/google/go-containerregistry/pkg/authn"
 	"github.com/google/go-containerregistry/pkg/crane"
 	"github.com/google/go-containerregistry/pkg/registry"
-	v1 "github.com/google/go-containerregistry/pkg/v1"
+	gcrv1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/mutate"
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -163,11 +164,13 @@ func TestOCIRepository_Reconcile(t *testing.T) {
 					obj.Generation == obj.Status.ObservedGeneration
 			}, timeout).Should(BeTrue())
 
-			t.Log(obj.Spec.Reference)
-
 			// Check if the revision matches the expected digest
 			g.Expect(obj.Status.Artifact.Revision).To(Equal(tt.digest))
 
+			// Check if the metadata matches the expected annotations
+			g.Expect(obj.Status.Artifact.Metadata["org.opencontainers.image.source"]).To(ContainSubstring("podinfo"))
+			g.Expect(obj.Status.Artifact.Metadata["org.opencontainers.image.revision"]).To(ContainSubstring(tt.tag))
+
 			// Check if the artifact storage path matches the expected file path
 			localPath := testStorage.LocalPath(*obj.Status.Artifact)
 			t.Logf("artifact local path: %s", localPath)
@@ -252,6 +255,7 @@ func TestOCIRepository_SecretRef(t *testing.T) {
 	ociURL := fmt.Sprintf("oci://%s", repositoryURL)
 
 	// Push Test Image
+	image = setPodinfoImageAnnotations(image, "6.1.6")
 	err = crane.Push(image, repositoryURL, crane.WithAuth(&authn.Basic{
 		Username: testRegistryUsername,
 		Password: testRegistryPassword,
@@ -265,7 +269,7 @@ func TestOCIRepository_SecretRef(t *testing.T) {
 	tests := []struct {
 		name                  string
 		url                   string
-		digest                v1.Hash
+		digest                gcrv1.Hash
 		includeSecretRef      bool
 		includeServiceAccount bool
 	}{
@@ -449,6 +453,7 @@ func TestOCIRepository_FailedAuth(t *testing.T) {
 	ociURL := fmt.Sprintf("oci://%s", repositoryURL)
 
 	// Push Test Image
+	image = setPodinfoImageAnnotations(image, "6.1.6")
 	err = crane.Push(image, repositoryURL, crane.WithAuth(&authn.Basic{
 		Username: testRegistryUsername,
 		Password: testRegistryPassword,
@@ -462,7 +467,7 @@ func TestOCIRepository_FailedAuth(t *testing.T) {
 	tests := []struct {
 		name                  string
 		url                   string
-		digest                v1.Hash
+		digest                gcrv1.Hash
 		repoUsername          string
 		repoPassword          string
 		includeSecretRef      bool
@@ -644,7 +649,7 @@ func TestOCIRepository_CertSecret(t *testing.T) {
 		name                  string
 		url                   string
 		tag                   string
-		digest                v1.Hash
+		digest                gcrv1.Hash
 		certSecret            *corev1.Secret
 		expectreadyconition   bool
 		expectedstatusmessage string
@@ -760,7 +765,7 @@ type artifactFixture struct {
 type podinfoImage struct {
 	url    string
 	tag    string
-	digest v1.Hash
+	digest gcrv1.Hash
 }
 
 func createPodinfoImageFromTar(tarFileName, tag string, imageServer *httptest.Server) (*podinfoImage, error) {
@@ -770,6 +775,8 @@ func createPodinfoImageFromTar(tarFileName, tag string, imageServer *httptest.Se
 		return nil, err
 	}
 
+	image = setPodinfoImageAnnotations(image, tag)
+
 	url, err := url.Parse(imageServer.URL)
 	if err != nil {
 		return nil, err
@@ -784,7 +791,6 @@ func createPodinfoImageFromTar(tarFileName, tag string, imageServer *httptest.Se
 
 	// Push image
 	err = crane.Push(image, repositoryURL, crane.WithTransport(imageServer.Client().Transport))
-
 	if err != nil {
 		return nil, err
 	}
@@ -802,8 +808,15 @@ func createPodinfoImageFromTar(tarFileName, tag string, imageServer *httptest.Se
 	}, nil
 }
 
-// These two taken verbatim from https://ericchiang.github.io/post/go-tls/
+func setPodinfoImageAnnotations(img gcrv1.Image, tag string) gcrv1.Image {
+	metadata := map[string]string{
+		"org.opencontainers.image.source":   "https://github.com/stefanprodan/podinfo",
+		"org.opencontainers.image.revision": fmt.Sprintf("%s/SHA", tag),
+	}
+	return mutate.Annotations(img, metadata).(gcrv1.Image)
+}
 
+// These two taken verbatim from https://ericchiang.github.io/post/go-tls/
 func certTemplate() (*x509.Certificate, error) {
 	// generate a random serial number (a real cert authority would
 	// have some logic behind this)
@@ -842,8 +855,6 @@ func createCert(template, parent *x509.Certificate, pub interface{}, parentPriv
 	return
 }
 
-// ----
-
 func createTLSServer() (*httptest.Server, []byte, []byte, []byte, tls.Certificate, error) {
 	var clientTLSCert tls.Certificate
 	var rootCertPEM, clientCertPEM, clientKeyPEM []byte

controllers/ocirepository_controller_test.go

@@ -161,6 +161,8 @@ func setupRegistryServer(ctx context.Context) (*registryClientTestServer, error)
 	server.registryHost = fmt.Sprintf("localhost:%d", port)
 	config.HTTP.Addr = fmt.Sprintf("127.0.0.1:%d", port)
 	config.HTTP.DrainTimeout = time.Duration(10) * time.Second
+	config.Log.AccessLog.Disabled = true
+	config.Log.Level = "error"
 	config.Storage = map[string]configuration.Parameters{"inmemory": map[string]interface{}{}}
 	config.Auth = configuration.Auth{
 		"htpasswd": configuration.Parameters{

controllers/suite_test.go

@@ -1190,6 +1190,18 @@ int64
 <p>Size is the number of bytes in the file.</p>
 </td>
 </tr>
+<tr>
+<td>
+<code>metadata</code><br>
+<em>
+map[string]string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>Metadata holds upstream information such as OCI annotations.</p>
+</td>
+</tr>
 </tbody>
 </table>
 </div>