Update artifact tarball when spec.ignore changes

This commit adds a new field `status.artifact.excludedPatternsChecksum` which is used to detect changes on `spec.ignore`, that controls the exclusions for the object's `status.artifact`.

Signed-off-by: Tiago Angelo <[email protected]>

Author: angelokurtis

api/v1beta2/artifact_types.go

@@ -17,6 +17,9 @@ limitations under the License.
 package v1beta2
 
 import (
+	"crypto/sha256"
+	"fmt"
+	"io"
 	"path"
 	"strings"
 
@@ -46,6 +49,11 @@ type Artifact struct {
 	// +optional
 	Checksum string `json:"checksum"`
 
+	// ExcludedPatternsChecksum is a SHA256 checksum of the excluded patterns
+	// applied in the last reconciliation.
+	// +optional
+	ExcludedPatternsChecksum string `json:"excludedPatternsChecksum,omitempty"`
+
 	// LastUpdateTime is the timestamp corresponding to the last update of the
 	// Artifact.
 	// +required
@@ -74,6 +82,30 @@ func (in *Artifact) HasChecksum(checksum string) bool {
 	return in.Checksum == checksum
 }
 
+// EqualExcludedPatterns returns if the given excluded patterns match the
+// current Artifact excluded patterns.
+func (in *Artifact) EqualExcludedPatterns(artifact Artifact) bool {
+	var ignored string
+	if in != nil {
+		ignored = in.ExcludedPatternsChecksum
+	}
+	return ignored == artifact.ExcludedPatternsChecksum
+}
+
+// SetExcludedPatterns sets the SHA256 checksum of the given excluded patterns.
+func (in *Artifact) SetExcludedPatterns(ignore *string) {
+	if in == nil {
+		return
+	}
+	if ignore == nil {
+		in.ExcludedPatternsChecksum = ""
+		return
+	}
+	h := sha256.New()
+	_, _ = io.Copy(h, strings.NewReader(*ignore))
+	in.ExcludedPatternsChecksum = fmt.Sprintf("%x", h.Sum(nil))
+}
+
 // ArtifactDir returns the artifact dir path in the form of
 // '<kind>/<namespace>/<name>'.
 func ArtifactDir(kind, namespace, name string) string {

config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml

@@ -379,6 +379,10 @@ spec:
                   checksum:
                     description: Checksum is the SHA256 checksum of the Artifact file.
                     type: string
+                  excludedPatternsChecksum:
+                    description: ExcludedPatternsChecksum is a SHA256 checksum of
+                      the excluded patterns applied in the last reconciliation.
+                    type: string
                   lastUpdateTime:
                     description: LastUpdateTime is the timestamp corresponding to
                       the last update of the Artifact.

config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml

@@ -149,8 +149,8 @@ spec:
               secretRef:
                 description: The secret name containing the Git credentials. For HTTPS
                   repositories the secret must contain username and password fields.
-                  For SSH repositories the secret must contain identity and known_hosts
-                  fields.
+                  For SSH repositories the secret must contain identity, identity.pub
+                  and known_hosts fields.
                 properties:
                   name:
                     description: Name of the referent.
@@ -554,6 +554,10 @@ spec:
                   checksum:
                     description: Checksum is the SHA256 checksum of the Artifact file.
                     type: string
+                  excludedPatternsChecksum:
+                    description: ExcludedPatternsChecksum is a SHA256 checksum of
+                      the excluded patterns applied in the last reconciliation.
+                    type: string
                   lastUpdateTime:
                     description: LastUpdateTime is the timestamp corresponding to
                       the last update of the Artifact.
@@ -663,6 +667,10 @@ spec:
                       description: Checksum is the SHA256 checksum of the Artifact
                         file.
                       type: string
+                    excludedPatternsChecksum:
+                      description: ExcludedPatternsChecksum is a SHA256 checksum of
+                        the excluded patterns applied in the last reconciliation.
+                      type: string
                     lastUpdateTime:
                       description: LastUpdateTime is the timestamp corresponding to
                         the last update of the Artifact.

config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml

@@ -427,6 +427,10 @@ spec:
                   checksum:
                     description: Checksum is the SHA256 checksum of the Artifact file.
                     type: string
+                  excludedPatternsChecksum:
+                    description: ExcludedPatternsChecksum is a SHA256 checksum of
+                      the excluded patterns applied in the last reconciliation.
+                    type: string
                   lastUpdateTime:
                     description: LastUpdateTime is the timestamp corresponding to
                       the last update of the Artifact.

config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml

@@ -350,6 +350,10 @@ spec:
                   checksum:
                     description: Checksum is the SHA256 checksum of the Artifact file.
                     type: string
+                  excludedPatternsChecksum:
+                    description: ExcludedPatternsChecksum is a SHA256 checksum of
+                      the excluded patterns applied in the last reconciliation.
+                    type: string
                   lastUpdateTime:
                     description: LastUpdateTime is the timestamp corresponding to
                       the last update of the Artifact.

controllers/bucket_controller.go

@@ -28,7 +28,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/fluxcd/source-controller/pkg/azure"
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/sync/semaphore"
 	corev1 "k8s.io/api/core/v1"
@@ -42,6 +41,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/ratelimiter"
 
+	"github.com/fluxcd/source-controller/pkg/azure"
+
 	"github.com/fluxcd/pkg/apis/meta"
 	"github.com/fluxcd/pkg/runtime/conditions"
 	helper "github.com/fluxcd/pkg/runtime/controller"
@@ -561,18 +562,19 @@ func (r *BucketReconciler) reconcileArtifact(ctx context.Context, obj *sourcev1.
 
 	// Create artifact
 	artifact := r.Storage.NewArtifactFor(obj.Kind, obj, revision, fmt.Sprintf("%s.tar.gz", revision))
+	artifact.SetExcludedPatterns(obj.Spec.Ignore)
 
 	// Set the ArtifactInStorageCondition if there's no drift.
 	defer func() {
-		if obj.GetArtifact().HasRevision(artifact.Revision) {
+		if obj.GetArtifact().HasRevision(artifact.Revision) && obj.GetArtifact().EqualExcludedPatterns(artifact) {
 			conditions.Delete(obj, sourcev1.ArtifactOutdatedCondition)
 			conditions.MarkTrue(obj, sourcev1.ArtifactInStorageCondition, meta.SucceededReason,
 				"stored artifact for revision '%s'", artifact.Revision)
 		}
 	}()
 
 	// The artifact is up-to-date
-	if obj.GetArtifact().HasRevision(artifact.Revision) {
+	if obj.GetArtifact().HasRevision(artifact.Revision) && obj.GetArtifact().EqualExcludedPatterns(artifact) {
 		r.eventLogf(ctx, obj, events.EventTypeTrace, sourcev1.ArtifactUpToDateReason, "artifact up-to-date with remote revision: '%s'", artifact.Revision)
 		return sreconcile.ResultSuccess, nil
 	}

controllers/gitrepository_controller.go

@@ -497,18 +497,19 @@ func (r *GitRepositoryReconciler) reconcileArtifact(ctx context.Context,
 	obj *sourcev1.GitRepository, commit *git.Commit, includes *artifactSet, dir string) (sreconcile.Result, error) {
 	// Create potential new artifact with current available metadata
 	artifact := r.Storage.NewArtifactFor(obj.Kind, obj.GetObjectMeta(), commit.String(), fmt.Sprintf("%s.tar.gz", commit.Hash.String()))
+	artifact.SetExcludedPatterns(obj.Spec.Ignore)
 
 	// Set the ArtifactInStorageCondition if there's no drift.
 	defer func() {
-		if obj.GetArtifact().HasRevision(artifact.Revision) && !includes.Diff(obj.Status.IncludedArtifacts) {
+		if obj.GetArtifact().HasRevision(artifact.Revision) && !includes.Diff(obj.Status.IncludedArtifacts) && obj.GetArtifact().EqualExcludedPatterns(artifact) {
 			conditions.Delete(obj, sourcev1.ArtifactOutdatedCondition)
 			conditions.MarkTrue(obj, sourcev1.ArtifactInStorageCondition, meta.SucceededReason,
 				"stored artifact for revision '%s'", artifact.Revision)
 		}
 	}()
 
 	// The artifact is up-to-date
-	if obj.GetArtifact().HasRevision(artifact.Revision) && !includes.Diff(obj.Status.IncludedArtifacts) {
+	if obj.GetArtifact().HasRevision(artifact.Revision) && !includes.Diff(obj.Status.IncludedArtifacts) && obj.GetArtifact().EqualExcludedPatterns(artifact) {
 		r.eventLogf(ctx, obj, events.EventTypeTrace, sourcev1.ArtifactUpToDateReason, "artifact up-to-date with remote revision: '%s'", artifact.Revision)
 		return sreconcile.ResultSuccess, nil
 	}