Skip to content

Update artifact tarball when spec.ignore changes #710

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

Update artifact tarball when spec.ignore changes #710

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions api/v1beta2/artifact_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@ limitations under the License.
package v1beta2

import (
"crypto/sha256"
"fmt"
"io"
"path"
"strings"

Expand Down Expand Up @@ -46,6 +49,11 @@ type Artifact struct {
// +optional
Checksum string `json:"checksum"`

// ExcludedPatternsChecksum is a SHA256 checksum of the excluded patterns
// applied in the last reconciliation.
// +optional
ExcludedPatternsChecksum string `json:"excludedPatternsChecksum,omitempty"`
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not every object is making use of exclusions. Given this, I do not think this field should be added to the Artifact, but rather to the Status objects from the Bucket and GitRepository.

In addition, I do not think that the new term ExcludedPatterns should be introduced. I would rather call it IgnoreChecksum or ObservedIgnore.


// LastUpdateTime is the timestamp corresponding to the last update of the
// Artifact.
// +required
Expand Down Expand Up @@ -74,6 +82,30 @@ func (in *Artifact) HasChecksum(checksum string) bool {
return in.Checksum == checksum
}

// EqualExcludedPatterns returns if the given excluded patterns match the
// current Artifact excluded patterns.
func (in *Artifact) EqualExcludedPatterns(artifact Artifact) bool {
var ignored string
if in != nil {
ignored = in.ExcludedPatternsChecksum
}
return ignored == artifact.ExcludedPatternsChecksum
}

// SetExcludedPatterns sets the SHA256 checksum of the given excluded patterns.
func (in *Artifact) SetExcludedPatterns(ignore *string) {
if in == nil {
return
}
if ignore == nil {
in.ExcludedPatternsChecksum = ""
return
}
h := sha256.New()
_, _ = io.Copy(h, strings.NewReader(*ignore))
in.ExcludedPatternsChecksum = fmt.Sprintf("%x", h.Sum(nil))
}

// ArtifactDir returns the artifact dir path in the form of
// '<kind>/<namespace>/<name>'.
func ArtifactDir(kind, namespace, name string) string {
Expand Down
4 changes: 4 additions & 0 deletions config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -379,6 +379,10 @@ spec:
checksum:
description: Checksum is the SHA256 checksum of the Artifact file.
type: string
excludedPatternsChecksum:
description: ExcludedPatternsChecksum is a SHA256 checksum of
the excluded patterns applied in the last reconciliation.
type: string
lastUpdateTime:
description: LastUpdateTime is the timestamp corresponding to
the last update of the Artifact.
Expand Down
12 changes: 10 additions & 2 deletions config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,8 +149,8 @@ spec:
secretRef:
description: The secret name containing the Git credentials. For HTTPS
repositories the secret must contain username and password fields.
For SSH repositories the secret must contain identity and known_hosts
fields.
For SSH repositories the secret must contain identity, identity.pub
and known_hosts fields.
properties:
name:
description: Name of the referent.
Expand Down Expand Up @@ -554,6 +554,10 @@ spec:
checksum:
description: Checksum is the SHA256 checksum of the Artifact file.
type: string
excludedPatternsChecksum:
description: ExcludedPatternsChecksum is a SHA256 checksum of
the excluded patterns applied in the last reconciliation.
type: string
lastUpdateTime:
description: LastUpdateTime is the timestamp corresponding to
the last update of the Artifact.
Expand Down Expand Up @@ -663,6 +667,10 @@ spec:
description: Checksum is the SHA256 checksum of the Artifact
file.
type: string
excludedPatternsChecksum:
description: ExcludedPatternsChecksum is a SHA256 checksum of
the excluded patterns applied in the last reconciliation.
type: string
lastUpdateTime:
description: LastUpdateTime is the timestamp corresponding to
the last update of the Artifact.
Expand Down
4 changes: 4 additions & 0 deletions config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -427,6 +427,10 @@ spec:
checksum:
description: Checksum is the SHA256 checksum of the Artifact file.
type: string
excludedPatternsChecksum:
description: ExcludedPatternsChecksum is a SHA256 checksum of
the excluded patterns applied in the last reconciliation.
type: string
lastUpdateTime:
description: LastUpdateTime is the timestamp corresponding to
the last update of the Artifact.
Expand Down
4 changes: 4 additions & 0 deletions config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,10 @@ spec:
checksum:
description: Checksum is the SHA256 checksum of the Artifact file.
type: string
excludedPatternsChecksum:
description: ExcludedPatternsChecksum is a SHA256 checksum of
the excluded patterns applied in the last reconciliation.
type: string
lastUpdateTime:
description: LastUpdateTime is the timestamp corresponding to
the last update of the Artifact.
Expand Down
8 changes: 5 additions & 3 deletions controllers/bucket_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ import (
"sync"
"time"

"github.com/fluxcd/source-controller/pkg/azure"
"golang.org/x/sync/errgroup"
"golang.org/x/sync/semaphore"
corev1 "k8s.io/api/core/v1"
Expand All @@ -42,6 +41,8 @@ import (
"sigs.k8s.io/controller-runtime/pkg/predicate"
"sigs.k8s.io/controller-runtime/pkg/ratelimiter"

"github.com/fluxcd/source-controller/pkg/azure"

"github.com/fluxcd/pkg/apis/meta"
"github.com/fluxcd/pkg/runtime/conditions"
helper "github.com/fluxcd/pkg/runtime/controller"
Expand Down Expand Up @@ -561,18 +562,19 @@ func (r *BucketReconciler) reconcileArtifact(ctx context.Context, obj *sourcev1.

// Create artifact
artifact := r.Storage.NewArtifactFor(obj.Kind, obj, revision, fmt.Sprintf("%s.tar.gz", revision))
artifact.SetExcludedPatterns(obj.Spec.Ignore)

// Set the ArtifactInStorageCondition if there's no drift.
defer func() {
if obj.GetArtifact().HasRevision(artifact.Revision) {
if obj.GetArtifact().HasRevision(artifact.Revision) && obj.GetArtifact().EqualExcludedPatterns(artifact) {
conditions.Delete(obj, sourcev1.ArtifactOutdatedCondition)
conditions.MarkTrue(obj, sourcev1.ArtifactInStorageCondition, meta.SucceededReason,
"stored artifact for revision '%s'", artifact.Revision)
}
}()

// The artifact is up-to-date
if obj.GetArtifact().HasRevision(artifact.Revision) {
if obj.GetArtifact().HasRevision(artifact.Revision) && obj.GetArtifact().EqualExcludedPatterns(artifact) {
r.eventLogf(ctx, obj, events.EventTypeTrace, sourcev1.ArtifactUpToDateReason, "artifact up-to-date with remote revision: '%s'", artifact.Revision)
return sreconcile.ResultSuccess, nil
}
Expand Down
5 changes: 3 additions & 2 deletions controllers/gitrepository_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -497,18 +497,19 @@ func (r *GitRepositoryReconciler) reconcileArtifact(ctx context.Context,
obj *sourcev1.GitRepository, commit *git.Commit, includes *artifactSet, dir string) (sreconcile.Result, error) {
// Create potential new artifact with current available metadata
artifact := r.Storage.NewArtifactFor(obj.Kind, obj.GetObjectMeta(), commit.String(), fmt.Sprintf("%s.tar.gz", commit.Hash.String()))
artifact.SetExcludedPatterns(obj.Spec.Ignore)

// Set the ArtifactInStorageCondition if there's no drift.
defer func() {
if obj.GetArtifact().HasRevision(artifact.Revision) && !includes.Diff(obj.Status.IncludedArtifacts) {
if obj.GetArtifact().HasRevision(artifact.Revision) && !includes.Diff(obj.Status.IncludedArtifacts) && obj.GetArtifact().EqualExcludedPatterns(artifact) {
conditions.Delete(obj, sourcev1.ArtifactOutdatedCondition)
conditions.MarkTrue(obj, sourcev1.ArtifactInStorageCondition, meta.SucceededReason,
"stored artifact for revision '%s'", artifact.Revision)
}
}()

// The artifact is up-to-date
if obj.GetArtifact().HasRevision(artifact.Revision) && !includes.Diff(obj.Status.IncludedArtifacts) {
if obj.GetArtifact().HasRevision(artifact.Revision) && !includes.Diff(obj.Status.IncludedArtifacts) && obj.GetArtifact().EqualExcludedPatterns(artifact) {
r.eventLogf(ctx, obj, events.EventTypeTrace, sourcev1.ArtifactUpToDateReason, "artifact up-to-date with remote revision: '%s'", artifact.Revision)
return sreconcile.ResultSuccess, nil
}
Expand Down