[DataFlow] add additional cmd parameter for quick reuse of cluster fo… (#15878)

* [DataFlow] add additional cmd parameter for quick reuse of cluster for dataflow commands

* added additional details to changelog and constants

* manually tested and set appropriate field when reuse flag is used

* fixed the reuse flag being set and changelog

* fix comments and null check

* update changelog

Co-authored-by: victorlee <[email protected]>
Co-authored-by: Yunchi Wang <[email protected]>

Author: 3 people

src/DataFactory/DataFactoryV2/Changelog.md

@@ -18,6 +18,7 @@
         - Additional information about change #1
 -->
 ## Upcoming Release
+* Added a DataFlowEnableQuickReuse argument for the `Set-AzDataFactoryV2IntegrationRuntime` cmdlet to enable quick reuse of clusters in next pipeline activities.
 * Updated ADF .Net SDK version to 4.25.0
 * Added a VNetInjectionMethod argument for the `Set-AzDataFactoryV2IntegrationRuntime` cmdlet to support the express virtual network injection of Azure-SSIS Integration Runtime.
 
@@ -146,4 +147,4 @@ to enable create Azure-SSIS IR with static public IP addresses.
 
 ## Version 1.0.0
 * General availability of `Az.DataFactory` module
-* Removed -LinkedServiceName parameter from Get-AzDataFactoryV2ActivityRun
+* Removed -LinkedServiceName parameter from Get-AzDataFactoryV2ActivityRun

src/DataFactory/DataFactoryV2/Constants.cs

@@ -152,6 +152,8 @@ internal static class Constants
 
         public const string HelpIntegrationRuntimeDataFlowComputeType = "Compute type of the data flow cluster which will execute data flow job.";
 
+        public const string HelpIntegrationRuntimeDataFlowQuickReuseEnabled = "Enable data flow flow cluster reuse in the next data flow activity run.";
+
         public const string HelpIntegrationRuntimeDataFlowTimeToLive = "Time to live (in minutes) setting of the data flow cluster which will execute data flow job.";
 
         public const string HelpIntegrationRuntimeSetupScriptContainerSasUri = "The SAS URI of the Azure blob container that contains the custom setup script.";

src/DataFactory/DataFactoryV2/IntegrationRuntimes/SetAzureDataFactoryIntegrationRuntimeCommand.cs

@@ -292,6 +292,20 @@ public class SetAzureDataFactoryIntegrationRuntimeCommand : IntegrationRuntimeCm
         [ValidateNotNullOrEmpty]
         public string DataFlowComputeType { get; set; }
 
+        [Parameter(
+            ParameterSetName = ParameterSetNames.ByIntegrationRuntimeName,
+            Mandatory = false,
+            HelpMessage = Constants.HelpIntegrationRuntimeDataFlowQuickReuseEnabled)]
+        [Parameter(
+            ParameterSetName = ParameterSetNames.ByResourceId,
+            Mandatory = false,
+            HelpMessage = Constants.HelpIntegrationRuntimeDataFlowQuickReuseEnabled)]
+        [Parameter(
+            ParameterSetName = ParameterSetNames.ByIntegrationRuntimeObject,
+            Mandatory = false,
+            HelpMessage = Constants.HelpIntegrationRuntimeDataFlowQuickReuseEnabled)]
+        public SwitchParameter DataFlowEnableQuickReuse { get; set; }
+
         [Parameter(
             ParameterSetName = ParameterSetNames.ByIntegrationRuntimeName,
             Mandatory = false,
@@ -759,7 +773,7 @@ private void HandleManagedIntegrationRuntime(ManagedIntegrationRuntime integrati
                 integrationRuntime, VNetInjectionMethod, SubnetId, Subnet, VNetId
                 );
 
-            if (!string.IsNullOrWhiteSpace(DataFlowComputeType) || DataFlowCoreCount != null || DataFlowTimeToLive != null)
+            if (!string.IsNullOrWhiteSpace(DataFlowComputeType) || DataFlowCoreCount != null || DataFlowTimeToLive != null || DataFlowEnableQuickReuse != null)
             {
                 if (integrationRuntime.ComputeProperties == null)
                 {
@@ -773,6 +787,16 @@ private void HandleManagedIntegrationRuntime(ManagedIntegrationRuntime integrati
                 integrationRuntime.ComputeProperties.DataFlowProperties.ComputeType = DataFlowComputeType ?? integrationRuntime.ComputeProperties.DataFlowProperties.ComputeType;
                 integrationRuntime.ComputeProperties.DataFlowProperties.CoreCount = DataFlowCoreCount ?? integrationRuntime.ComputeProperties.DataFlowProperties.CoreCount;
                 integrationRuntime.ComputeProperties.DataFlowProperties.TimeToLive = DataFlowTimeToLive ?? integrationRuntime.ComputeProperties.DataFlowProperties.TimeToLive;
+                if (DataFlowEnableQuickReuse.IsPresent)
+                {
+                    integrationRuntime.ComputeProperties.DataFlowProperties.Cleanup = false;
+                } 
+                else
+                {
+                    // setting it as null as the default value for the cleanup variable is false, and the backend endpoint treats null value as true.
+                    integrationRuntime.ComputeProperties.DataFlowProperties.Cleanup = null;
+                }
+
             }
 
             if (PublicIPs != null)

src/DataFactory/DataFactoryV2/Models/PSManagedIntegrationRuntime.cs

@@ -72,6 +72,8 @@ public PSManagedIntegrationRuntime(
 
         public int? DataFlowTimeToLive => ManagedIntegrationRuntime.ComputeProperties?.DataFlowProperties?.TimeToLive;
 
+        public bool? DataFlowEnableCleanUp => ManagedIntegrationRuntime.ComputeProperties?.DataFlowProperties.Cleanup;
+
         public string State => ManagedIntegrationRuntime.State;
 
         public string LicenseType => ManagedIntegrationRuntime.SsisProperties?.LicenseType;

src/DataFactory/DataFactoryV2/help/Set-AzDataFactoryV2IntegrationRuntime.md

@@ -268,6 +268,21 @@ Accept pipeline input: False
 Accept wildcard characters: False
 ```
 
+### -DataFlowEnableQuickReuse
+To whether enable data flow cluster to be reused in the next dataflow activity.
+
+```yaml
+Type: System.Management.Automation.SwitchParameter
+Parameter Sets: (All)
+Aliases:
+
+Required: False
+Position: Named
+Default value: None
+Accept pipeline input: False
+Accept wildcard characters: False
+```
+
 ### -DataProxyIntegrationRuntimeName
 The Self-Hosted Integration Runtime name which is used as a proxy