Merge pull request #738 from somtochiama/sas-key-azure-blob

Add Support for SAS keys in Azure Blob

Author: Paulo Gomes

docs/spec/v1beta2/buckets.md

@@ -295,6 +295,7 @@ sets of `.data` fields:
 - `clientId` for authenticating using a Managed Identity.
 - `accountKey` for authenticating using a
   [Shared Key](https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/storage/azblob#SharedKeyCredential).
+- `sasKey` for authenticating using a [SAS Token](https://docs.microsoft.com/en-us/azure/storage/common/storage-sas-overview)
 
 For any Managed Identity and/or Azure Active Directory authentication method,
 the base URL can be configured using `.data.authorityHost`. If not supplied,
@@ -504,6 +505,41 @@ spec:
   endpoint: https://testfluxsas.blob.core.windows.net
 ```
 
+##### Azure Blob SAS Token example
+
+```yaml
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: Bucket
+metadata:
+  name: azure-sas-token
+  namespace: default
+spec:
+  interval: 5m0s
+  provider: azure
+  bucketName: <bucket-name>
+  endpoint: https://<account-name>.blob.core.windows.net
+  secretRef:
+    name: azure-key
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: azure-key
+  namespace: default
+type: Opaque
+data:
+  sasKey: <base64>
+```
+
+The sasKey only contains the SAS token e.g `?sv=2020-08-0&ss=bfqt&srt=co&sp=rwdlacupitfx&se=2022-05-26T21:55:35Z&st=2022-05...`.
+The leading question mark is optional.
+The query values from the `sasKey` data field in the Secrets gets merged with the ones in the `spec.endpoint` of the `Bucket`.
+If the same key is present in the both of them, the value in the `sasKey` takes precedence.
+
+Note that the Azure SAS Token has an expiry date and it should be updated before it expires so that Flux can
+continue to access Azure Storage.
+
 #### GCP
 
 When a Bucket's `.spec.provider` is set to `gcp`, the source-controller will

go.mod

@@ -37,6 +37,7 @@ require (
 	github.com/fluxcd/pkg/gitutil v0.1.0
 	github.com/fluxcd/pkg/helmtestserver v0.7.4
 	github.com/fluxcd/pkg/lockedfile v0.1.0
+	github.com/fluxcd/pkg/masktoken v0.0.1
 	github.com/fluxcd/pkg/oci v0.3.0
 	github.com/fluxcd/pkg/runtime v0.16.2
 	github.com/fluxcd/pkg/ssh v0.5.0

go.sum

@@ -399,6 +399,8 @@ github.com/fluxcd/pkg/helmtestserver v0.7.4 h1:/Xj2+XLz7wr38MI3uPYvVAsZB9wQOq6rp
 github.com/fluxcd/pkg/helmtestserver v0.7.4/go.mod h1:aL5V4o8wUOMqeHMfjbVHS057E3ejzHMRVMqEbsK9FUQ=
 github.com/fluxcd/pkg/lockedfile v0.1.0 h1:YsYFAkd6wawMCcD74ikadAKXA4s2sukdxrn7w8RB5eo=
 github.com/fluxcd/pkg/lockedfile v0.1.0/go.mod h1:EJLan8t9MiOcgTs8+puDjbE6I/KAfHbdvIy9VUgIjm8=
+github.com/fluxcd/pkg/masktoken v0.0.1 h1:egWR/ibTzf4L3PxE8TauKO1srD1Ye/aalgQRQuKKRdU=
+github.com/fluxcd/pkg/masktoken v0.0.1/go.mod h1:sQmMtX4s5RwdGlByJazzNasWFFgBdmtNcgeZcGBI72Y=
 github.com/fluxcd/pkg/oci v0.3.0 h1:GFn6JZeg5fV2K4vsQ0s5lJFid6qrpA4RybLXL+7qUbQ=
 github.com/fluxcd/pkg/oci v0.3.0/go.mod h1:c1pj9E/G5927gSa6ooACAyZe+HwjgmPk9johL7oXDHw=
 github.com/fluxcd/pkg/runtime v0.16.2 h1:CexfMmJK+r12sHTvKWyAax0pcPomjd6VnaHXcxjUrRY=

pkg/azure/blob.go

@@ -35,6 +35,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 
+	"github.com/fluxcd/pkg/masktoken"
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
 )
 
@@ -53,6 +54,7 @@ const (
 	clientCertificateSendChainField = "clientCertificateSendChain"
 	authorityHostField              = "authorityHost"
 	accountKeyField                 = "accountKey"
+	sasKeyField                     = "sasKey"
 )
 
 // BlobClient is a minimal Azure Blob client for fetching objects.
@@ -105,6 +107,14 @@ func NewClient(obj *sourcev1.Bucket, secret *corev1.Secret) (c *BlobClient, err
 			c.ServiceClient, err = azblob.NewServiceClientWithSharedKey(obj.Spec.Endpoint, cred, &azblob.ClientOptions{})
 			return
 		}
+
+		var fullPath string
+		if fullPath, err = sasTokenFromSecret(obj.Spec.Endpoint, secret); err != nil {
+			return
+		}
+
+		c.ServiceClient, err = azblob.NewServiceClientWithNoCredential(fullPath, &azblob.ClientOptions{})
+		return
 	}
 
 	// Compose token chain based on environment.
@@ -149,6 +159,9 @@ func ValidateSecret(secret *corev1.Secret) error {
 	if _, hasAccountKey := secret.Data[accountKeyField]; hasAccountKey {
 		valid = true
 	}
+	if _, hasSasKey := secret.Data[sasKeyField]; hasSasKey {
+		valid = true
+	}
 	if _, hasAuthorityHost := secret.Data[authorityHostField]; hasAuthorityHost {
 		valid = true
 	}
@@ -355,6 +368,41 @@ func sharedCredentialFromSecret(endpoint string, secret *corev1.Secret) (*azblob
 	return nil, nil
 }
 
+// sasTokenFromSecret retrieves the SAS Token from the `sasKey`. It returns an empty string if the Secret
+// does not contain a valid set of credentials.
+func sasTokenFromSecret(ep string, secret *corev1.Secret) (string, error) {
+	if sasKey, hasSASKey := secret.Data[sasKeyField]; hasSASKey {
+		queryString := strings.TrimPrefix(string(sasKey), "?")
+		values, err := url.ParseQuery(queryString)
+		if err != nil {
+			maskedErrorString, maskErr := masktoken.MaskTokenFromString(err.Error(), string(sasKey))
+			if maskErr != nil {
+				return "", fmt.Errorf("error redacting token from error message: %s", maskErr)
+			}
+			return "", fmt.Errorf("unable to parse SAS token: %s", maskedErrorString)
+		}
+
+		epURL, err := url.Parse(ep)
+		if err != nil {
+			return "", fmt.Errorf("unable to parse endpoint URL: %s", err)
+		}
+
+		//merge the query values in the endpoint with the token
+		epValues := epURL.Query()
+		for key, val := range epValues {
+			if !values.Has(key) {
+				for _, str := range val {
+					values.Add(key, str)
+				}
+			}
+		}
+
+		epURL.RawQuery = values.Encode()
+		return epURL.String(), nil
+	}
+	return "", nil
+}
+
 // chainCredentialWithSecret tries to create a set of tokens, and returns an
 // azidentity.ChainedTokenCredential if at least one of the following tokens was
 // successfully created:

pkg/azure/blob_integration_test.go

@@ -163,6 +163,67 @@ func TestBlobClient_FGetObject(t *testing.T) {
 	g.Expect(f).To(Equal([]byte(testFileData)))
 }
 
+func TestBlobClientSASKey_FGetObject(t *testing.T) {
+	g := NewWithT(t)
+
+	tempDir := t.TempDir()
+
+	// create a client with the shared key
+	client, err := NewClient(testBucket.DeepCopy(), testSecret.DeepCopy())
+	g.Expect(err).ToNot(HaveOccurred())
+	g.Expect(client).ToNot(BeNil())
+
+	g.Expect(client.CanGetAccountSASToken()).To(BeTrue())
+
+	// Generate test container name.
+	testContainer := generateString(testContainerGenerateName)
+
+	// Create test container.
+	ctx, timeout := context.WithTimeout(context.Background(), testTimeout)
+	defer timeout()
+	g.Expect(createContainer(ctx, client, testContainer)).To(Succeed())
+	t.Cleanup(func() {
+		g.Expect(deleteContainer(context.Background(), client, testContainer)).To(Succeed())
+	})
+
+	// Create test blob.
+	ctx, timeout = context.WithTimeout(context.Background(), testTimeout)
+	defer timeout()
+	g.Expect(createBlob(ctx, client, testContainer, testFile, testFileData))
+
+	localPath := filepath.Join(tempDir, testFile)
+
+	// use the shared key client to create a SAS key for the account
+	sasKey, err := client.GetSASToken(azblob.AccountSASResourceTypes{Object: true, Container: true},
+		azblob.AccountSASPermissions{List: true, Read: true},
+		azblob.AccountSASServices{Blob: true},
+		time.Now(),
+		time.Now().Add(48*time.Hour))
+	g.Expect(err).ToNot(HaveOccurred())
+	g.Expect(sasKey).ToNot(BeEmpty())
+
+	// the sdk returns the full SAS url e.g test.blob.core.windows.net/?<actual-sas-token>
+	sasKey = strings.TrimPrefix(sasKey, testBucket.Spec.Endpoint+"/")
+	testSASKeySecret := corev1.Secret{
+		Data: map[string][]byte{
+			sasKeyField: []byte(sasKey),
+		},
+	}
+
+	sasKeyClient, err := NewClient(testBucket.DeepCopy(), testSASKeySecret.DeepCopy())
+	g.Expect(err).ToNot(HaveOccurred())
+
+	// Test if blob exists using sasKey.
+	ctx, timeout = context.WithTimeout(context.Background(), testTimeout)
+	defer timeout()
+	_, err = sasKeyClient.FGetObject(ctx, testContainer, testFile, localPath)
+
+	g.Expect(err).ToNot(HaveOccurred())
+	g.Expect(localPath).To(BeARegularFile())
+	f, _ := os.ReadFile(localPath)
+	g.Expect(f).To(Equal([]byte(testFileData)))
+}
+
 func TestBlobClient_FGetObject_NotFoundErr(t *testing.T) {
 	g := NewWithT(t)
 

pkg/azure/blob_test.go

@@ -25,6 +25,7 @@ import (
 	"errors"
 	"fmt"
 	"math/big"
+	"net/url"
 	"testing"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
@@ -68,6 +69,14 @@ func TestValidateSecret(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "valid SAS Key Secret",
+			secret: &corev1.Secret{
+				Data: map[string][]byte{
+					sasKeyField: []byte("?spr=<some-sas-url"),
+				},
+			},
+		},
 		{
 			name: "valid SharedKey Secret",
 			secret: &corev1.Secret{
@@ -292,6 +301,85 @@ func Test_sharedCredentialFromSecret(t *testing.T) {
 	}
 }
 
+func Test_sasTokenFromSecret(t *testing.T) {
+	tests := []struct {
+		name     string
+		endpoint string
+		secret   *corev1.Secret
+		want     string
+		wantErr  bool
+	}{
+		{
+			name:     "Valid SAS Token",
+			endpoint: "https://accountName.blob.windows.net",
+			secret: &corev1.Secret{
+				Data: map[string][]byte{
+					sasKeyField: []byte("?sv=2020-08-0&ss=bfqt&srt=co&sp=rwdlacupitfx&se=2022-05-26T21:55:35Z&st=2022-05-26T13:55:35Z&spr=https&sig=JlHT"),
+				},
+			},
+			want: "https://accountName.blob.windows.net?sv=2020-08-0&ss=bfqt&srt=co&sp=rwdlacupitfx&se=2022-05-26T21:55:35Z&st=2022-05-26T13:55:35Z&spr=https&sig=JlHT",
+		},
+		{
+			name:     "Valid SAS Token without leading question mark",
+			endpoint: "https://accountName.blob.windows.net",
+			secret: &corev1.Secret{
+				Data: map[string][]byte{
+					sasKeyField: []byte("sv=2020-08-04&ss=bfqt&srt=co&sp=rwdl&se=2022-05-26T21:55:35Z&st=2022-05-26&spr=https&sig=JlHT"),
+				},
+			},
+			want: "https://accountName.blob.windows.net?sv=2020-08-04&ss=bfqt&srt=co&sp=rwdl&se=2022-05-26T21:55:35Z&st=2022-05-26&spr=https&sig=JlHT",
+		},
+		{
+			name:     "endpoint with query values",
+			endpoint: "https://accountName.blob.windows.net?sv=2020-08-04",
+			secret: &corev1.Secret{
+				Data: map[string][]byte{
+					sasKeyField: []byte("ss=bfqt&srt=co&sp=rwdl&se=2022-05-26T21:55:35Z&st=2022-05-26&spr=https&sig=JlHT"),
+				},
+			},
+			want: "https://accountName.blob.windows.net?sv=2020-08-04&ss=bfqt&srt=co&sp=rwdl&se=2022-05-26T21:55:35Z&st=2022-05-26&spr=https&sig=JlHT",
+		},
+		{
+			name:     "conflicting query values in token",
+			endpoint: "https://accountName.blob.windows.net?sv=2020-08-04&ss=abcde",
+			secret: &corev1.Secret{
+				Data: map[string][]byte{
+					sasKeyField: []byte("sv=2019-07-06&ss=bfqt&srt=co&sp=rwdl&se=2022-05-26T21:55:35Z&st=2022-05-26&spr=https&sig=JlHT"),
+				},
+			},
+			want: "https://accountName.blob.windows.net?sv=2019-07-06&ss=bfqt&srt=co&sp=rwdl&se=2022-05-26T21:55:35Z&st=2022-05-26&spr=https&sig=JlHT",
+		},
+		{
+			name: "invalid sas token",
+			secret: &corev1.Secret{
+				Data: map[string][]byte{
+					sasKeyField: []byte("%##sssvecrpt"),
+				},
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			g := NewWithT(t)
+
+			_, err := url.ParseQuery("")
+			got, err := sasTokenFromSecret(tt.endpoint, tt.secret)
+			g.Expect(err != nil).To(Equal(tt.wantErr))
+			if tt.want != "" {
+				ttVaules, err := url.Parse(tt.want)
+				g.Expect(err).To(BeNil())
+
+				gotValues, err := url.Parse(got)
+				g.Expect(err).To(BeNil())
+				g.Expect(gotValues.Query()).To(Equal(ttVaules.Query()))
+				return
+			}
+			g.Expect(got).To(Equal(""))
+		})
+	}
+}
+
 func Test_chainCredentialWithSecret(t *testing.T) {
 	g := NewWithT(t)