migrate fleet integration_policy to framework, fix secret churn

.github/workflows/test.yml

@@ -63,7 +63,6 @@ jobs:
           xpack.security.enabled: true
           xpack.security.authc.api_key.enabled: true
           xpack.security.authc.token.enabled: true
-          xpack.security.http.ssl.enabled: false
           xpack.watcher.enabled: true
           xpack.license.self_generated.type: trial
           repositories.url.allowed_urls: https://example.com/*
@@ -80,10 +79,28 @@ jobs:
           ELASTICSEARCH_USERNAME: ${{ env.KIBANA_SYSTEM_USERNAME }}
           ELASTICSEARCH_PASSWORD: ${{ env.KIBANA_SYSTEM_PASSWORD }}
           XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: a7a6311933d3503b89bc2dbc36572c33a6c10925682e591bffcab6911c06786d
-#          LOGGING_ROOT_LEVEL: debug
+          # LOGGING_ROOT_LEVEL: debug
         ports:
           - 5601:5601
         options: --health-cmd="curl http://localhost:5601/api/status" --health-interval=10s --health-timeout=5s --health-retries=10
+      fleet:
+        image: docker.elastic.co/beats/elastic-agent:${{ matrix.version }}
+        env:
+          SERVER_NAME: fleet
+          FLEET_ENROLL: "1"
+          FLEET_URL: https://fleet:8220
+          FLEET_INSECURE: "true"
+          FLEET_SERVER_ENABLE: "1"
+          FLEET_SERVER_POLICY_ID: fleet-server
+          FLEET_SERVER_ELASTICSEARCH_HOST: http://elasticsearch:9200
+          FLEET_SERVER_ELASTICSEARCH_INSECURE: "true"
+          FLEET_SERVER_INSECURE_HTTP: "true"
+          KIBANA_HOST: http://kibana:5601
+          KIBANA_FLEET_SETUP: "1"
+          KIBANA_FLEET_PASSWORD: ${{ env.ELASTIC_PASSWORD }}
+        ports:
+          - 8220:8220
+        options: --restart="unless-stopped"
 
     timeout-minutes: 15
     strategy:
@@ -123,8 +140,6 @@ jobs:
       - name: Setup Kibana user
         run: make set-kibana-password
         env:
-          ELASTICSEARCH_ENDPOINTS: "http://localhost:9200"
-          ELASTICSEARCH_USERNAME: "elastic"
           ELASTICSEARCH_PASSWORD: ${{ env.ELASTIC_PASSWORD }}
           KIBANA_SYSTEM_USERNAME: ${{ env.KIBANA_SYSTEM_USERNAME }}
           KIBANA_SYSTEM_PASSWORD: ${{ env.KIBANA_SYSTEM_PASSWORD }}
@@ -134,10 +149,17 @@ jobs:
         run: |-
           echo "apikey=$(make create-es-api-key | jq -r .encoded)" >> "$GITHUB_OUTPUT"
         env:
-          ELASTICSEARCH_ENDPOINTS: "http://localhost:9200"
-          ELASTICSEARCH_USERNAME: "elastic"
           ELASTICSEARCH_PASSWORD: ${{ env.ELASTIC_PASSWORD }}
 
+      - id: setup-fleet
+        name: Setup Fleet
+        if: matrix.version == '8.10.3' || matrix.version == '8.11.4' || matrix.version == '8.12.2' || matrix.version == '8.13.4' || matrix.version == '8.14.3' || matrix.version == '8.15.0'
+        run: |-
+          make setup-kibana-fleet
+        env:
+          ELASTICSEARCH_PASSWORD: ${{ env.ELASTIC_PASSWORD }}
+          FLEET_NAME: "fleet"
+
       - id: force-install-synthetics
         name: Force install synthetics
         if: matrix.version == '8.14.3' || matrix.version == '8.15.0'

CHANGELOG.md

@@ -3,6 +3,7 @@
 - Fix handling of `sys_monitoring` in `elasticstack_fleet_agent_policy` ([#792](https://github.com/elastic/terraform-provider-elasticstack/pull/792))
 - Migrate `elasticstack_fleet_agent_policy`, `elasticstack_fleet_integration` (both), and `elasticstack_fleet_server_host` to terraform-plugin-framework ([#785](https://github.com/elastic/terraform-provider-elasticstack/pull/785))
 - Fix for synthetics http/tcp monitor produces inconsistent result after apply ([#801](https://github.com/elastic/terraform-provider-elasticstack/pull/801))
+- Migrate `elasticstack_fleet_integration_policy` to terraform-plugin-framework. Fix drift in integration policy secrets. ([#797](https://github.com/elastic/terraform-provider-elasticstack/pull/797))
 
 ## [0.11.7] - 2024-09-20
 

Makefile

@@ -31,7 +31,11 @@ KIBANA_SYSTEM_USERNAME ?= kibana_system
 KIBANA_SYSTEM_PASSWORD ?= password
 KIBANA_API_KEY_NAME ?= kibana-api-key
 
+FLEET_NAME ?= terraform-elasticstack-fleet
+FLEET_ENDPOINT ?= https://$(FLEET_NAME):8220
+
 SOURCE_LOCATION ?= $(shell pwd)
+, := ,
 
 export GOBIN = $(shell pwd)/bin
 
@@ -72,7 +76,7 @@ retry = until [ $$(if [ -z "$$attempt" ]; then echo -n "0"; else echo -n "$$atte
 # To run specific test (e.g. TestAccResourceActionConnector) execute `make docker-testacc TESTARGS='-run ^TestAccResourceActionConnector$$'`
 # To enable tracing (or debugging), execute `make docker-testacc TF_LOG=TRACE`
 .PHONY: docker-testacc
-docker-testacc: docker-elasticsearch docker-kibana ## Run acceptance tests in the docker container
+docker-testacc: docker-elasticsearch docker-kibana docker-fleet ## Run acceptance tests in the docker container
 	@ docker run --rm \
 		-e ELASTICSEARCH_ENDPOINTS="$(ELASTICSEARCH_ENDPOINTS)" \
 		-e KIBANA_ENDPOINT="$(KIBANA_ENDPOINT)" \
@@ -163,6 +167,30 @@ docker-kibana-with-tls: docker-network docker-elasticsearch set-kibana-password
 		docker.elastic.co/kibana/kibana:$(STACK_VERSION); \
 		fi)
 
+.PHONY: docker-fleet
+docker-fleet: docker-network docker-elasticsearch docker-kibana setup-kibana-fleet ## Start Fleet node in docker container
+	@ docker rm -f $(FLEET_NAME)  &> /dev/null || true
+	@ $(call retry, 5, if ! docker ps --format '{{.Names}}' | grep -w $(FLEET_NAME) > /dev/null 2>&1 ; then \
+		docker run -d \
+		-p 8220:8220 \
+		-e SERVER_NAME=fleet \
+      	-e FLEET_ENROLL=1 \
+      	-e FLEET_URL=$(FLEET_ENDPOINT) \
+      	-e FLEET_INSECURE=true \
+      	-e FLEET_SERVER_ENABLE=1 \
+      	-e FLEET_SERVER_POLICY_ID=fleet-server \
+      	-e FLEET_SERVER_ELASTICSEARCH_HOST=$(ELASTICSEARCH_ENDPOINTS) \
+      	-e FLEET_SERVER_ELASTICSEARCH_INSECURE=true \
+      	-e FLEET_SERVER_INSECURE_HTTP=true \
+      	-e KIBANA_HOST=$(KIBANA_ENDPOINT) \
+      	-e KIBANA_FLEET_SETUP=1 \
+      	-e KIBANA_FLEET_USERNAME=$(ELASTICSEARCH_USERNAME) \
+      	-e KIBANA_FLEET_PASSWORD=$(ELASTICSEARCH_PASSWORD) \
+		--name $(FLEET_NAME) \
+		--network $(ELASTICSEARCH_NETWORK) \
+		docker.elastic.co/beats/elastic-agent:$(STACK_VERSION); \
+		fi)
+
 
 .PHONY: docker-network
 docker-network: ## Create a dedicated network for ES and test runs
@@ -172,19 +200,25 @@ docker-network: ## Create a dedicated network for ES and test runs
 
 .PHONY: set-kibana-password
 set-kibana-password: ## Sets the ES KIBANA_SYSTEM_USERNAME's password to KIBANA_SYSTEM_PASSWORD. This expects Elasticsearch to be available at localhost:9200
-	@ $(call retry, 10, curl -X POST -u $(ELASTICSEARCH_USERNAME):$(ELASTICSEARCH_PASSWORD) -H "Content-Type: application/json" http://localhost:9200/_security/user/$(KIBANA_SYSTEM_USERNAME)/_password -d "{\"password\":\"$(KIBANA_SYSTEM_PASSWORD)\"}" | grep -q "^{}")
+	@ $(call retry, 10, curl -sS -X POST -u $(ELASTICSEARCH_USERNAME):$(ELASTICSEARCH_PASSWORD) -H "Content-Type: application/json" http://localhost:9200/_security/user/$(KIBANA_SYSTEM_USERNAME)/_password -d '{"password":"$(KIBANA_SYSTEM_PASSWORD)"}' | grep -q "^{}")
 
 .PHONY: create-es-api-key
 create-es-api-key: ## Creates and outputs a new API Key. This expects Elasticsearch to be available at localhost:9200
-	@ $(call retry, 10, curl -X POST -u $(ELASTICSEARCH_USERNAME):$(ELASTICSEARCH_PASSWORD) -H "Content-Type: application/json" http://localhost:9200/_security/api_key -d "{\"name\":\"$(KIBANA_API_KEY_NAME)\"}")
+	@ $(call retry, 10, curl -sS -X POST -u $(ELASTICSEARCH_USERNAME):$(ELASTICSEARCH_PASSWORD) -H "Content-Type: application/json" http://localhost:9200/_security/api_key -d '{"name":"$(KIBANA_API_KEY_NAME)"}')
 
 .PHONY: create-es-bearer-token
-create-es-bearer-token:
-	@ $(call retry, 10, curl -X POST -u $(ELASTICSEARCH_USERNAME):$(ELASTICSEARCH_PASSWORD) -H "Content-Type: application/json" http://localhost:9200/_security/oauth2/token -d "{\"grant_type\": \"client_credentials\"}")
+create-es-bearer-token: ## Creates and outputs a new OAuth bearer token. This expects Elasticsearch to be available at localhost:9200
+	@ $(call retry, 10, curl -sS -X POST -u $(ELASTICSEARCH_USERNAME):$(ELASTICSEARCH_PASSWORD) -H "Content-Type: application/json" http://localhost:9200/_security/oauth2/token -d '{"grant_type":"client_credentials"}')
+
+.PHONY: setup-kibana-fleet
+setup-kibana-fleet: ## Creates the agent and integration policies required to run Fleet. This expects Kibana to be available at localhost:5601
+	@ $(call retry, 10, curl -sS --fail-with-body -X POST -u $(ELASTICSEARCH_USERNAME):$(ELASTICSEARCH_PASSWORD) -H "Content-Type: application/json" -H "kbn-xsrf: true" http://localhost:5601/api/fleet/fleet_server_hosts -d '{"name":"default"$(,)"host_urls":["$(FLEET_ENDPOINT)"]$(,)"is_default":true}')
+	@ $(call retry, 10, curl -sS --fail-with-body -X POST -u $(ELASTICSEARCH_USERNAME):$(ELASTICSEARCH_PASSWORD) -H "Content-Type: application/json" -H "kbn-xsrf: true" http://localhost:5601/api/fleet/agent_policies -d '{"id":"fleet-server"$(,)"name":"Fleet Server"$(,)"namespace":"default"$(,)"monitoring_enabled":["logs"$(,)"metrics"]}')
+	@ $(call retry, 10, curl -sS --fail-with-body -X POST -u $(ELASTICSEARCH_USERNAME):$(ELASTICSEARCH_PASSWORD) -H "Content-Type: application/json" -H "kbn-xsrf: true" http://localhost:5601/api/fleet/package_policies -d '{"name":"fleet-server"$(,)"namespace":"default"$(,)"policy_id":"fleet-server"$(,)"enabled":true$(,)"inputs":[{"type":"fleet-server"$(,)"enabled":true$(,)"streams":[]$(,)"vars":{}}]$(,)"package":{"name":"fleet_server"$(,)"version":"1.5.0"}}')
 
 .PHONY: docker-clean
 docker-clean: ## Try to remove provisioned nodes and assigned network
-	@ docker rm -f $(ELASTICSEARCH_NAME) $(KIBANA_NAME) || true
+	@ docker rm -f $(ELASTICSEARCH_NAME) $(KIBANA_NAME) $(FLEET_NAME) || true
 	@ docker network rm $(ELASTICSEARCH_NETWORK) || true
 
 

docs/resources/fleet_integration_policy.md

@@ -93,7 +93,7 @@ resource "elasticstack_fleet_integration_policy" "sample" {
 - `description` (String) The description of the integration policy.
 - `enabled` (Boolean) Enable the integration policy.
 - `force` (Boolean) Force operations, such as creation and deletion, to occur.
-- `input` (Block List) (see [below for nested schema](#nestedblock--input))
+- `input` (Block List) Integration inputs. (see [below for nested schema](#nestedblock--input))
 - `policy_id` (String) Unique identifier of the integration policy.
 - `vars_json` (String, Sensitive) Integration-level variables as JSON.
 

generated/fleet/getschema.go

@@ -71,6 +71,7 @@ var transformers = []TransformFunc{
 	transformSchemasInputsType,
 	transformInlinePackageDefinitions,
 	transformAddPackagePolicyVars,
+	transformAddPackagePolicySecretReferences,
 	transformFixPackageSearchResult,
 }
 
@@ -333,6 +334,30 @@ func transformAddPackagePolicyVars(schema *Schema) {
 	}
 }
 
+// transformAddPackagePolicySecretReferences adds the missing 'secretReferences'
+// field to the PackagePolicy schema struct.
+func transformAddPackagePolicySecretReferences(schema *Schema) {
+	inputs, ok := schema.Components.GetFields("schemas.new_package_policy.properties")
+	if !ok {
+		panic("properties not found")
+	}
+
+	// Only add it if it doesn't exist.
+	if _, ok = inputs.Get("secret_references"); !ok {
+		inputs.Set("secret_references", map[string]any{
+			"type": "array",
+			"items": map[string]any{
+				"type": "object",
+				"properties": map[string]any{
+					"id": map[string]any{
+						"type": "string",
+					},
+				},
+			},
+		})
+	}
+}
+
 // transformFixPackageSearchResult removes unneeded fields from the
 // SearchResult struct. These fields are also causing parsing errors.
 func transformFixPackageSearchResult(schema *Schema) {

internal/clients/fleet/fleet.go

@@ -253,15 +253,15 @@ func DeleteFleetServerHost(ctx context.Context, client *Client, id string) fwdia
 }
 
 // ReadPackagePolicy reads a specific package policy from the API.
-func ReadPackagePolicy(ctx context.Context, client *Client, id string) (*fleetapi.PackagePolicy, diag.Diagnostics) {
+func ReadPackagePolicy(ctx context.Context, client *Client, id string) (*fleetapi.PackagePolicy, fwdiag.Diagnostics) {
 	format := fleetapi.GetPackagePolicyParamsFormatSimplified
 	params := fleetapi.GetPackagePolicyParams{
 		Format: &format,
 	}
 
 	resp, err := client.API.GetPackagePolicyWithResponse(ctx, id, &params)
 	if err != nil {
-		return nil, diag.FromErr(err)
+		return nil, fromErr(err)
 	}
 
 	switch resp.StatusCode() {
@@ -270,56 +270,56 @@ func ReadPackagePolicy(ctx context.Context, client *Client, id string) (*fleetap
 	case http.StatusNotFound:
 		return nil, nil
 	default:
-		return nil, reportUnknownError(resp.StatusCode(), resp.Body)
+		return nil, reportUnknownErrorFw(resp.StatusCode(), resp.Body)
 	}
 }
 
 // CreatePackagePolicy creates a new package policy.
-func CreatePackagePolicy(ctx context.Context, client *Client, req fleetapi.CreatePackagePolicyJSONRequestBody) (*fleetapi.PackagePolicy, diag.Diagnostics) {
+func CreatePackagePolicy(ctx context.Context, client *Client, req fleetapi.CreatePackagePolicyJSONRequestBody) (*fleetapi.PackagePolicy, fwdiag.Diagnostics) {
 	format := fleetapi.CreatePackagePolicyParamsFormatSimplified
 	params := fleetapi.CreatePackagePolicyParams{
 		Format: &format,
 	}
 
 	resp, err := client.API.CreatePackagePolicyWithResponse(ctx, &params, req)
 	if err != nil {
-		return nil, diag.FromErr(err)
+		return nil, fromErr(err)
 	}
 
 	switch resp.StatusCode() {
 	case http.StatusOK:
 		return &resp.JSON200.Item, nil
 	default:
-		return nil, reportUnknownError(resp.StatusCode(), resp.Body)
+		return nil, reportUnknownErrorFw(resp.StatusCode(), resp.Body)
 	}
 }
 
 // UpdatePackagePolicy updates an existing package policy.
-func UpdatePackagePolicy(ctx context.Context, client *Client, id string, req fleetapi.UpdatePackagePolicyJSONRequestBody) (*fleetapi.PackagePolicy, diag.Diagnostics) {
+func UpdatePackagePolicy(ctx context.Context, client *Client, id string, req fleetapi.UpdatePackagePolicyJSONRequestBody) (*fleetapi.PackagePolicy, fwdiag.Diagnostics) {
 	format := fleetapi.UpdatePackagePolicyParamsFormatSimplified
 	params := fleetapi.UpdatePackagePolicyParams{
 		Format: &format,
 	}
 
 	resp, err := client.API.UpdatePackagePolicyWithResponse(ctx, id, &params, req)
 	if err != nil {
-		return nil, diag.FromErr(err)
+		return nil, fromErr(err)
 	}
 
 	switch resp.StatusCode() {
 	case http.StatusOK:
 		return &resp.JSON200.Item, nil
 	default:
-		return nil, reportUnknownError(resp.StatusCode(), resp.Body)
+		return nil, reportUnknownErrorFw(resp.StatusCode(), resp.Body)
 	}
 }
 
 // DeletePackagePolicy deletes an existing package policy.
-func DeletePackagePolicy(ctx context.Context, client *Client, id string, force bool) diag.Diagnostics {
+func DeletePackagePolicy(ctx context.Context, client *Client, id string, force bool) fwdiag.Diagnostics {
 	params := fleetapi.DeletePackagePolicyParams{Force: &force}
 	resp, err := client.API.DeletePackagePolicyWithResponse(ctx, id, &params)
 	if err != nil {
-		return diag.FromErr(err)
+		return fromErr(err)
 	}
 
 	switch resp.StatusCode() {
@@ -328,7 +328,7 @@ func DeletePackagePolicy(ctx context.Context, client *Client, id string, force b
 	case http.StatusNotFound:
 		return nil
 	default:
-		return reportUnknownError(resp.StatusCode(), resp.Body)
+		return reportUnknownErrorFw(resp.StatusCode(), resp.Body)
 	}
 }
 

internal/fleet/agent_policy/resource_test.go

@@ -2,7 +2,6 @@ package agent_policy_test
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"testing"
 
@@ -224,7 +223,7 @@ func checkResourceAgentPolicySkipDestroy(s *terraform.State) error {
 		}
 
 		if diags = fleet.DeleteAgentPolicy(context.Background(), fleetClient, rs.Primary.ID); diags.HasError() {
-			return errors.New(diags.Errors()[0].Summary())
+			return utils.FwDiagsAsError(diags)
 		}
 	}
 	return nil

internal/fleet/enrollment_tokens/models.go

@@ -31,7 +31,7 @@ func (model *enrollmentTokensModel) populateFromAPI(ctx context.Context, data []
 	return
 }
 
-func newEnrollmentTokenModel(data fleetapi.EnrollmentApiKey) enrollmentTokenModel {
+func newEnrollmentTokenModel(data fleetapi.EnrollmentApiKey, meta utils.ListMeta) enrollmentTokenModel {
 	return enrollmentTokenModel{
 		KeyID:     types.StringValue(data.Id),
 		Active:    types.BoolValue(data.Active),