[HDInsight] Support HDI Autoscale Feature (#12858)

* HDInsight Support Autoscale:
1. Add a new parameter to existing cmdlet New-AzHDInsightCluster
2. Add 5 cmdlets:
  - New-AzHDInsightClusterAutoscaleScheduleCondition
  - New-AzHDInsightClusterAutoscaleConfiguration
  - Get-AzHDInsightClusterAutoscaleConfiguration
  - Set-AzHDInsightClusterAutoscaleConfiguration
  - Remove-AzHDInsightClusterAutoscaleConfiguration

* update cred scan suppression file

* Add statistic analysis exception by modifing SignatureIssues.csv file

* Change Days to Day

* Update parameter set in Get/Remove-AzHDInsightClusterAutoscaleConfiguration

* Add -Schedule parameter to schedule related parameter sets

Co-authored-by: Zhenyu Zhou <[email protected]>

Author: aim-for-better

src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightAutoscaleTests.cs

@@ -0,0 +1,42 @@
+// ----------------------------------------------------------------------------------
+//
+// Copyright Microsoft Corporation
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ----------------------------------------------------------------------------------
+
+using Microsoft.Azure.ServiceManagement.Common.Models;
+using Microsoft.WindowsAzure.Commands.ScenarioTest;
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Commands.HDInsight.Test.ScenarioTests
+{
+    public class HDInsightAutoscaleTests : TestController
+    {
+        public XunitTracingInterceptor _logger;
+
+        public HDInsightAutoscaleTests(ITestOutputHelper output)
+        {
+            _logger = new XunitTracingInterceptor(output);
+            XunitTracingInterceptor.AddToContext(_logger);
+        }
+
+        [Fact]
+        [Trait(Category.AcceptanceType, Category.CheckIn)]
+        public void TestAutoscaleRelatedCommands()
+        {
+            TestController.NewInstance.RunPowerShellTest(_logger, "Test-AutoscaleRelatedCommands");
+        }
+    }
+}

src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightAutoscaleTests.ps1

@@ -0,0 +1,70 @@
+# ----------------------------------------------------------------------------------
+#
+# Copyright Microsoft Corporation
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ----------------------------------------------------------------------------------
+
+<#
+.SYNOPSIS
+Test Enable, Disable autoscale of HDInsight cluster.
+#>
+function Test-AutoscaleRelatedCommands{
+
+	# Create some resources that will be used throughout test
+	try
+	{
+		# prepare parameter for creating parameter
+		$params= Prepare-ClusterCreateParameterForWASB
+
+		# test create cluster
+		$cluster = New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
+		-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
+		-DefaultStorageAccountName $params.storageAccountName -DefaultStorageAccountKey $params.storageAccountKey `
+		-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
+		-MinSupportedTlsVersion $params.minSupportedTlsVersion
+
+		Assert-NotNull $cluster
+
+		# test Set-AzHDInsightClusterAutoscaleConfiguration: enable Load-based autoscale
+		$loadAutoscaleCluster=Set-AzHDInsightClusterAutoscaleConfiguration -ResourceGroupName $cluster.ResourceGroup `
+		-ClusterName $cluster.Name -MinWorkerNodeCount 4 -MaxWorkerNodeCount 5
+
+		Assert-NotNull $loadAutoscaleCluster
+		Assert-NotNull $loadAutoscaleCluster.ComputeProfile.Roles[1].AutoscaleConfiguration.Capacity
+		Assert-Null $loadAutoscaleCluster.ComputeProfile.Roles[1].AutoscaleConfiguration.Recurrence
+
+		# test Get-AzHDInsightClusterAutoscaleConfiguration
+		$autoscale = Get-AzHDInsightClusterAutoscaleConfiguration -ClusterName $cluster.Name -ResourceGroupName $cluster.ResourceGroup
+		Assert-AreEqual $autoscale.Capacity.MinInstanceCount  4
+		Assert-AreEqual $autoscale.Capacity.MaxInstanceCount  5
+
+		Start-Sleep -s 20
+		# test Remove-AzHDInsightClusterAutoscaleConfiguration
+		Remove-AzHDInsightClusterAutoscaleConfiguration -ClusterName $cluster.Name -ResourceGroupName $cluster.ResourceGroup
+		
+		# test Set-AzHDInsightClusterAutoscaleConfiguration: enable Schedule-based autoscale
+		# test New-AzHDInsightAutoscaleScheduleCondition
+		$condition1=New-AzHDInsightClusterAutoscaleScheduleCondition -Time 09:00 -WorkerNodeCount 5 -Day Monday,Tuesday
+		$condition2=New-AzHDInsightClusterAutoscaleScheduleCondition -Time 08:00 -WorkerNodeCount 4 -Day Friday
+
+		$scheduleAutoscaleCluster=Set-AzHDInsightClusterAutoscaleConfiguration -ResourceGroupName $cluster.ResourceGroup `
+		-ClusterName $cluster.Name -Schedule -TimeZone "Pacific Standard Time" -Condition $condition1,$condition2
+
+		Assert-AreEqual $scheduleAutoscaleCluster.ComputeProfile.Roles[1].AutoscaleConfiguration.Recurrence.TimeZone "Pacific Standard Time"
+		Assert-AreEqual $scheduleAutoscaleCluster.ComputeProfile.Roles[1].AutoscaleConfiguration.Recurrence.Condition[0].WorkerNodeCount 5
+	}
+	finally
+	{
+		# Delete cluster and resource group
+		Remove-AzHDInsightCluster -ClusterName $cluster.Name
+		Remove-AzResourceGroup -ResourceGroupName $cluster.ResourceGroup
+	}
+}

src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightClusterTests.cs

@@ -61,7 +61,21 @@ public void TestCreateClusterWithPrivateLink()
         [Trait(Category.AcceptanceType, Category.CheckIn)]
         public void TestCreateClusterWithEncryptionAtHost()
         {
-            TestController.NewInstance.RunPowerShellTest(_logger, "Test-TestCreateClusterWithEncryptionAtHost");
+            TestController.NewInstance.RunPowerShellTest(_logger, "Test-CreateClusterWithEncryptionAtHost");
+        }
+
+        [Fact]
+        [Trait(Category.AcceptanceType, Category.CheckIn)]
+        public void TestCreateClusterWithLoadBasedAutoscale()
+        {
+            TestController.NewInstance.RunPowerShellTest(_logger, "Test-CreateClusterWithLoadBasedAutoscale");
+        }
+
+        [Fact]
+        [Trait(Category.AcceptanceType, Category.CheckIn)]
+        public void TestCreateClusterWithScheduleBasedAutoscale()
+        {
+            TestController.NewInstance.RunPowerShellTest(_logger, "Test-CreateClusterWithScheduleBasedAutoscale");
         }
     }
 }

src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightClusterTests.ps1

@@ -166,16 +166,16 @@ function Test-CreateClusterWithPrivateLink{
 
 <#
 .SYNOPSIS
-Test Create Azure HDInsight Cluster which Enalbes Encryption At Host
+Test Create Azure HDInsight Cluster which enalbes Encryption At Host
 #>
 
-function Test-TestCreateClusterWithEncryptionAtHost{
+function Test-CreateClusterWithEncryptionAtHost{
 
 	# Create some resources that will be used throughout test
 	try
 	{
 		# prepare parameter for creating parameter
-		$params= Prepare-ClusterCreateParameterForWASB -Location "South Central US"
+		$params= Prepare-ClusterCreateParameterForWASB -location "South Central US"
 		$encryptionAtHost=$true
 		$workerNodeSize="Standard_DS14_v2"
 		$headNodeSize="Standard_DS14_v2"
@@ -199,3 +199,81 @@ function Test-TestCreateClusterWithEncryptionAtHost{
 		Remove-AzResourceGroup -ResourceGroupName $cluster.ResourceGroup
 	}
 }
+
+<#
+.SYNOPSIS
+Test Create Azure HDInsight Cluster with Load-based autoscale
+#>
+
+function Test-CreateClusterWithLoadBasedAutoscale{
+
+	# Create some resources that will be used throughout test
+	try
+	{
+		# prepare parameter for creating parameter
+		$params= Prepare-ClusterCreateParameterForWASB -location "East US"
+
+		# create autoscale cofiguration
+		$autoscaleConfiguration=New-AzHDInsightClusterAutoscaleConfiguration -MinWorkerNodeCount 4 -MaxWorkerNodeCount 5
+
+		# create cluster with load-based autoscale
+		$cluster=New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
+		-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
+		-DefaultStorageAccountName $params.storageAccountName -DefaultStorageAccountKey $params.storageAccountKey `
+		-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
+		-MinSupportedTlsVersion $params.minSupportedTlsVersion -Version 4.0 `
+		-AutoscaleConfiguration $autoscaleConfiguration
+
+		Assert-NotNull $cluster
+		Assert-AreEqual $cluster.ComputeProfile.Roles[1].AutoscaleConfiguration.Capacity.MinInstanceCount 4
+		Assert-AreEqual $cluster.ComputeProfile.Roles[1].AutoscaleConfiguration.Capacity.MaxInstanceCount 5
+	}
+	finally
+	{
+		# Delete cluster and resource group
+		Remove-AzHDInsightCluster -ClusterName $cluster.Name
+		Remove-AzResourceGroup -ResourceGroupName $cluster.ResourceGroup
+	}
+}
+
+<#
+.SYNOPSIS
+Test Create Azure HDInsight Cluster with Schedule-based autoscale
+#>
+
+function Test-CreateClusterWithScheduleBasedAutoscale{
+
+	# Create some resources that will be used throughout test
+	try
+	{
+		# prepare parameter for creating parameter
+		$params= Prepare-ClusterCreateParameterForWASB -location "East US"
+
+		# create autoscale schedule condition
+		$condition1=New-AzHDInsightClusterAutoscaleScheduleCondition -Time "09:00" -WorkerNodeCount 4 -Day Monday,Tuesday
+		$condition2=New-AzHDInsightClusterAutoscaleScheduleCondition -Time "08:00" -WorkerNodeCount 5 -Day Friday
+
+		# create autoscale configuration
+		$autoscaleConfiguration=New-AzHDInsightClusterAutoscaleConfiguration -TimeZone ([System.TimeZoneInfo]::Local).Id `
+		-Condition $condition1,$condition2
+
+		# create cluster with schedule-based autoscale
+		$cluster=New-AzHDInsightCluster -Location $params.location -ResourceGroupName $params.resourceGroupName `
+		-ClusterName $params.clusterName -ClusterSizeInNodes $params.clusterSizeInNodes -ClusterType $params.clusterType `
+		-DefaultStorageAccountName $params.storageAccountName -DefaultStorageAccountKey $params.storageAccountKey `
+		-HttpCredential $params.httpCredential -SshCredential $params.sshCredential `
+		-MinSupportedTlsVersion $params.minSupportedTlsVersion -Version 4.0 `
+		-AutoscaleConfiguration $autoscaleConfiguration
+
+		Assert-NotNull $cluster
+		Assert-NotNull $cluster.ComputeProfile.Roles[1].AutoscaleConfiguration.Recurrence
+		Assert-AreEqual $cluster.ComputeProfile.Roles[1].AutoscaleConfiguration.Recurrence.Condition[0].WorkerNodeCount $condition1.WorkerNodeCount
+		Assert-AreEqual $cluster.ComputeProfile.Roles[1].AutoscaleConfiguration.Recurrence.Condition[1].WorkerNodeCount $condition2.WorkerNodeCount
+	}
+	finally
+	{
+		# Delete cluster and resource group
+		Remove-AzHDInsightCluster -ClusterName $cluster.Name
+		Remove-AzResourceGroup -ResourceGroupName $cluster.ResourceGroup
+	}
+}