Update artifact tarball when spec.ignore changes

This commit adds a new field `status.ignoreChecksum` which is used to detect changes on `spec.ignore`, that controls the exclusions for the object's `status.artifact`

Signed-off-by: Tiago Angelo <[email protected]>

Author: angelokurtis

api/v1beta2/bucket_types.go

@@ -124,6 +124,12 @@ type BucketStatus struct {
 	// +optional
 	Artifact *Artifact `json:"artifact,omitempty"`
 
+	// IgnoreChecksum is a checksum of .spec.ignore in .status.observedGeneration
+	// version of the object.
+	// It is formatted as `<algo>:<checksum>`. For example: `sha256:<checksum>`.
+	// +optional
+	IgnoreChecksum string `json:"ignoreChecksum,omitempty"`
+
 	meta.ReconcileRequestStatus `json:",inline"`
 }
 

api/v1beta2/gitrepository_types.go

@@ -211,6 +211,12 @@ type GitRepositoryStatus struct {
 	// +optional
 	IncludedArtifacts []*Artifact `json:"includedArtifacts,omitempty"`
 
+	// IgnoreChecksum is a checksum of .spec.ignore in .status.observedGeneration
+	// version of the object.
+	// It is formatted as `<algo>:<checksum>`. For example: `sha256:<checksum>`.
+	// +optional
+	IgnoreChecksum string `json:"ignoreChecksum,omitempty"`
+
 	meta.ReconcileRequestStatus `json:",inline"`
 }
 

config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml

@@ -478,6 +478,11 @@ spec:
                   - type
                   type: object
                 type: array
+              ignoreChecksum:
+                description: 'IgnoreChecksum is a checksum of .spec.ignore in .status.observedGeneration
+                  version of the object. It is formatted as `<algo>:<checksum>`. For
+                  example: `sha256:<checksum>`.'
+                type: string
               lastHandledReconcileAt:
                 description: LastHandledReconcileAt holds the value of the most recent
                   reconcile request value, so a change of the annotation value can

config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml

@@ -653,6 +653,11 @@ spec:
                   - type
                   type: object
                 type: array
+              ignoreChecksum:
+                description: 'IgnoreChecksum is a checksum of .spec.ignore in .status.observedGeneration
+                  version of the object. It is formatted as `<algo>:<checksum>`. For
+                  example: `sha256:<checksum>`.'
+                type: string
               includedArtifacts:
                 description: IncludedArtifacts contains a list of the last successfully
                   included Artifacts as instructed by GitRepositorySpec.Include.

controllers/bucket_controller.go

@@ -28,7 +28,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/fluxcd/source-controller/pkg/azure"
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/sync/semaphore"
 	corev1 "k8s.io/api/core/v1"
@@ -42,6 +41,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/ratelimiter"
 
+	"github.com/fluxcd/source-controller/pkg/azure"
+
 	"github.com/fluxcd/pkg/apis/meta"
 	"github.com/fluxcd/pkg/runtime/conditions"
 	helper "github.com/fluxcd/pkg/runtime/controller"
@@ -562,21 +563,31 @@ func (r *BucketReconciler) reconcileArtifact(ctx context.Context, obj *sourcev1.
 	// Create artifact
 	artifact := r.Storage.NewArtifactFor(obj.Kind, obj, revision, fmt.Sprintf("%s.tar.gz", revision))
 
+	// Calculates checksum of current .spec.ignore
+	var ignoreChecksum string
+	if obj.Spec.Ignore != nil {
+		ignore := []byte(*obj.Spec.Ignore)
+		ignoreChecksum = fmt.Sprintf("sha256:%x", sha256.Sum256(ignore))
+	}
+
 	// Set the ArtifactInStorageCondition if there's no drift.
 	defer func() {
-		if obj.GetArtifact().HasRevision(artifact.Revision) {
+		if obj.GetArtifact().HasRevision(artifact.Revision) && obj.Status.IgnoreChecksum == ignoreChecksum {
 			conditions.Delete(obj, sourcev1.ArtifactOutdatedCondition)
 			conditions.MarkTrue(obj, sourcev1.ArtifactInStorageCondition, meta.SucceededReason,
 				"stored artifact for revision '%s'", artifact.Revision)
 		}
 	}()
 
 	// The artifact is up-to-date
-	if obj.GetArtifact().HasRevision(artifact.Revision) {
+	if obj.GetArtifact().HasRevision(artifact.Revision) && obj.Status.IgnoreChecksum == ignoreChecksum {
 		r.eventLogf(ctx, obj, events.EventTypeTrace, sourcev1.ArtifactUpToDateReason, "artifact up-to-date with remote revision: '%s'", artifact.Revision)
 		return sreconcile.ResultSuccess, nil
 	}
 
+	// Ensure .spec.ignore checksum is up-to-date
+	obj.Status.IgnoreChecksum = ignoreChecksum
+
 	// Ensure target path exists and is a directory
 	if f, err := os.Stat(dir); err != nil {
 		e := &serror.Event{

controllers/bucket_controller_test.go

@@ -37,6 +37,7 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/tools/record"
+	"k8s.io/utils/pointer"
 	kstatus "sigs.k8s.io/cli-utils/pkg/kstatus/status"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
@@ -977,6 +978,34 @@ func TestBucketReconciler_reconcileArtifact(t *testing.T) {
 				*conditions.TrueCondition(sourcev1.StorageOperationFailedCondition, sourcev1.InvalidPathReason, "is not a directory"),
 			},
 		},
+		{
+			name: "Ignore checksum should be set when spec ignore is in use",
+			beforeFunc: func(t *WithT, obj *sourcev1.Bucket, index *etagIndex, dir string) {
+				obj.Spec.Interval = metav1.Duration{Duration: interval}
+				obj.Spec.Ignore = pointer.StringPtr("!**.txt\n")
+			},
+			afterFunc: func(t *WithT, obj *sourcev1.Bucket, dir string) {
+				t.Expect(obj.Status.IgnoreChecksum).To(Equal("sha256:539530fbd29c252e357e36ddc065dc7923265e4ea795f09cada03287288f5e7a"))
+			},
+			want: sreconcile.ResultSuccess,
+			assertConditions: []metav1.Condition{
+				*conditions.TrueCondition(sourcev1.ArtifactInStorageCondition, meta.SucceededReason, "stored artifact for revision 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'"),
+			},
+		},
+		{
+			name: "Ignore checksum should be updated when spec ignore changes",
+			beforeFunc: func(t *WithT, obj *sourcev1.Bucket, index *etagIndex, dir string) {
+				obj.Spec.Interval = metav1.Duration{Duration: interval}
+				obj.Status.IgnoreChecksum = "sha256:539530fbd29c252e357e36ddc065dc7923265e4ea795f09cada03287288f5e7a"
+			},
+			afterFunc: func(t *WithT, obj *sourcev1.Bucket, dir string) {
+				t.Expect(obj.Status.IgnoreChecksum).To(BeEmpty())
+			},
+			want: sreconcile.ResultSuccess,
+			assertConditions: []metav1.Condition{
+				*conditions.TrueCondition(sourcev1.ArtifactInStorageCondition, meta.SucceededReason, "stored artifact for revision 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'"),
+			},
+		},
 	}
 
 	for _, tt := range tests {

controllers/gitrepository_controller.go

@@ -18,6 +18,7 @@ package controllers
 
 import (
 	"context"
+	"crypto/sha256"
 	"errors"
 	"fmt"
 	"os"
@@ -498,21 +499,31 @@ func (r *GitRepositoryReconciler) reconcileArtifact(ctx context.Context,
 	// Create potential new artifact with current available metadata
 	artifact := r.Storage.NewArtifactFor(obj.Kind, obj.GetObjectMeta(), commit.String(), fmt.Sprintf("%s.tar.gz", commit.Hash.String()))
 
+	// Calculates checksum of current .spec.ignore
+	var ignoreChecksum string
+	if obj.Spec.Ignore != nil {
+		ignore := []byte(*obj.Spec.Ignore)
+		ignoreChecksum = fmt.Sprintf("sha256:%x", sha256.Sum256(ignore))
+	}
+
 	// Set the ArtifactInStorageCondition if there's no drift.
 	defer func() {
-		if obj.GetArtifact().HasRevision(artifact.Revision) && !includes.Diff(obj.Status.IncludedArtifacts) {
+		if obj.GetArtifact().HasRevision(artifact.Revision) && !includes.Diff(obj.Status.IncludedArtifacts) && obj.Status.IgnoreChecksum == ignoreChecksum {
 			conditions.Delete(obj, sourcev1.ArtifactOutdatedCondition)
 			conditions.MarkTrue(obj, sourcev1.ArtifactInStorageCondition, meta.SucceededReason,
 				"stored artifact for revision '%s'", artifact.Revision)
 		}
 	}()
 
 	// The artifact is up-to-date
-	if obj.GetArtifact().HasRevision(artifact.Revision) && !includes.Diff(obj.Status.IncludedArtifacts) {
+	if obj.GetArtifact().HasRevision(artifact.Revision) && !includes.Diff(obj.Status.IncludedArtifacts) && obj.Status.IgnoreChecksum == ignoreChecksum {
 		r.eventLogf(ctx, obj, events.EventTypeTrace, sourcev1.ArtifactUpToDateReason, "artifact up-to-date with remote revision: '%s'", artifact.Revision)
 		return sreconcile.ResultSuccess, nil
 	}
 
+	// Ensure .spec.ignore checksum is up-to-date
+	obj.Status.IgnoreChecksum = ignoreChecksum
+
 	// Ensure target path exists and is a directory
 	if f, err := os.Stat(dir); err != nil {
 		e := &serror.Event{

controllers/gitrepository_controller_test.go

@@ -838,6 +838,36 @@ func TestGitRepositoryReconciler_reconcileArtifact(t *testing.T) {
 				*conditions.TrueCondition(sourcev1.StorageOperationFailedCondition, sourcev1.InvalidPathReason, "invalid target path"),
 			},
 		},
+		{
+			name: "Ignore checksum should be set when spec ignore is in use",
+			dir:  "testdata/git/repository",
+			beforeFunc: func(obj *sourcev1.GitRepository) {
+				obj.Spec.Interval = metav1.Duration{Duration: interval}
+				obj.Spec.Ignore = pointer.StringPtr("!**.txt\n")
+			},
+			afterFunc: func(t *WithT, obj *sourcev1.GitRepository) {
+				t.Expect(obj.Status.IgnoreChecksum).To(Equal("sha256:539530fbd29c252e357e36ddc065dc7923265e4ea795f09cada03287288f5e7a"))
+			},
+			want: sreconcile.ResultSuccess,
+			assertConditions: []metav1.Condition{
+				*conditions.TrueCondition(sourcev1.ArtifactInStorageCondition, meta.SucceededReason, "stored artifact for revision 'main/revision'"),
+			},
+		},
+		{
+			name: "Ignore checksum should be updated when spec ignore changes",
+			dir:  "testdata/git/repository",
+			beforeFunc: func(obj *sourcev1.GitRepository) {
+				obj.Spec.Interval = metav1.Duration{Duration: interval}
+				obj.Status.IgnoreChecksum = "sha256:539530fbd29c252e357e36ddc065dc7923265e4ea795f09cada03287288f5e7a"
+			},
+			afterFunc: func(t *WithT, obj *sourcev1.GitRepository) {
+				t.Expect(obj.Status.IgnoreChecksum).To(BeEmpty())
+			},
+			want: sreconcile.ResultSuccess,
+			assertConditions: []metav1.Condition{
+				*conditions.TrueCondition(sourcev1.ArtifactInStorageCondition, meta.SucceededReason, "stored artifact for revision 'main/revision'"),
+			},
+		},
 	}
 	artifactSize := func(g *WithT, artifactURL string) *int64 {
 		if artifactURL == "" {

docs/api/source.md

@@ -1208,6 +1208,20 @@ Artifact
 </tr>
 <tr>
 <td>
+<code>ignoreChecksum</code><br>
+<em>
+string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>IgnoreChecksum is a checksum of .spec.ignore in .status.observedGeneration
+version of the object.
+It is formatted as <code>&lt;algo&gt;:&lt;checksum&gt;</code>. For example: <code>sha256:&lt;checksum&gt;</code>.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>ReconcileRequestStatus</code><br>
 <em>
 <a href="https://godoc.org/github.com/fluxcd/pkg/apis/meta#ReconcileRequestStatus">
@@ -1643,6 +1657,20 @@ Artifacts as instructed by GitRepositorySpec.Include.</p>
 </tr>
 <tr>
 <td>
+<code>ignoreChecksum</code><br>
+<em>
+string
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>IgnoreChecksum is a checksum of .spec.ignore in .status.observedGeneration
+version of the object.
+It is formatted as <code>&lt;algo&gt;:&lt;checksum&gt;</code>. For example: <code>sha256:&lt;checksum&gt;</code>.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>ReconcileRequestStatus</code><br>
 <em>
 <a href="https://godoc.org/github.com/fluxcd/pkg/apis/meta#ReconcileRequestStatus">