Skip to content

Added support for RServer cluster type with edgenode size and RStudio config #3639

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Mar 16, 2017
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/ResourceManager/HDInsight/ChangeLog.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@
- Additional information about change #1
-->
## Current Release
* Added support for RServer cluster type
- Edgenode VM size can be specified for RServer cluster in New-AzureRmHDInsightCluster or New-AzureRmHDInsightClusterConfig
- RServer is now a configuration option in Add-AzureRmHDInsightConfigValues. It allows for RStudio flag to be set to indicate that R Studio installation should be done.

## Version 2.7.0

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
<HintPath>..\..\..\packages\Microsoft.Azure.Management.Authorization.0.18.2-preview\lib\net40\Microsoft.Azure.Management.Authorization.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Azure.Management.HDInsight, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.2.0.1\lib\net40\Microsoft.Azure.Management.HDInsight.dll</HintPath>
<HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.2.0.3\lib\net40\Microsoft.Azure.Management.HDInsight.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Azure.Management.HDInsight.Job">
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,13 @@ namespace Microsoft.Azure.Commands.HDInsight.Test
{
public class HDInsightTestBase : RMTestBase
{
protected const string ClusterType = "Hadoop";
protected const string ClusterName = "hdicluster";
protected const string ResourceGroupName = "hdi-rg1";
protected const string Location = "west us";

protected string ClusterType = "Hadoop";
protected string HdiVersion = "3.1";

protected Mock<AzureHdInsightManagementClient> hdinsightManagementMock;
protected Mock<AzureHdInsightJobManagementClient> hdinsightJobManagementMock;
protected Mock<ICommandRuntime> commandRuntimeMock;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,18 @@ public ConfigurationTests(Xunit.Abstractions.ITestOutputHelper output)
[Fact]
[Trait(Category.AcceptanceType, Category.CheckIn)]
public void CanCreateNewConfig()
{
CreateNewConfig();
}

[Fact]
[Trait(Category.AcceptanceType, Category.CheckIn)]
public void CanCreateNewConfigForRServer()
{
CreateNewConfig(setEdgeNodeVmSize: true);
}

public void CreateNewConfig(bool setEdgeNodeVmSize = false)
{
var newconfigcmdlet = new NewAzureHDInsightClusterConfigCommand
{
Expand All @@ -38,6 +50,8 @@ public void CanCreateNewConfig()
ClusterType = ClusterType
};

if (setEdgeNodeVmSize) newconfigcmdlet.EdgeNodeSize = "edgeNodeVmSizeSetTest";

newconfigcmdlet.ExecuteCmdlet();
commandRuntimeMock.Verify(
f =>
Expand All @@ -52,6 +66,7 @@ public void CanCreateNewConfig()
string.IsNullOrEmpty(c.DefaultStorageAccountName) &&
string.IsNullOrEmpty(c.HeadNodeSize) &&
string.IsNullOrEmpty(c.ZookeeperNodeSize) &&
((!setEdgeNodeVmSize && string.IsNullOrEmpty(c.EdgeNodeSize)) || (setEdgeNodeVmSize && c.EdgeNodeSize == "edgeNodeVmSizeSetTest")) &&
c.HiveMetastore == null &&
c.OozieMetastore == null &&
c.ScriptActions.Count == 0)),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ public void CanCreateNewHDInsightCluster_Linux()
clusterout =>
clusterout.ClusterState == "Running" &&
clusterout.ClusterType == ClusterType &&
clusterout.ClusterVersion == "3.1" &&
clusterout.ClusterVersion == HdiVersion &&
clusterout.CoresUsed == 24 &&
clusterout.Location == Location &&
clusterout.Name == ClusterName &&
Expand All @@ -154,6 +154,16 @@ public void CanCreateNewHDInsightCluster_Linux()

}

[Fact]
[Trait(Category.AcceptanceType, Category.CheckIn)]
public void CanCreateNewHDInsightCluster_RServer_Linux()
{
ClusterType = "RServer";
HdiVersion = "3.5";

CreateNewHDInsightCluster(setEdgeNodeVmSize:true);
}

[Fact]
[Trait(Category.AcceptanceType, Category.CheckIn)]
public void CanCreateNewHDInsightCluster_Secure_Linux()
Expand All @@ -179,7 +189,7 @@ public void CanCreateNewHDInsightCluster_Secure_Linux()
clusterout =>
clusterout.ClusterState == "Running" &&
clusterout.ClusterType == ClusterType &&
clusterout.ClusterVersion == "3.1" &&
clusterout.ClusterVersion == HdiVersion &&
clusterout.CoresUsed == 24 &&
clusterout.Location == Location &&
clusterout.Name == ClusterName &&
Expand All @@ -195,7 +205,7 @@ public void CanCreateNewHDInsightCluster_Secure_Linux()
Times.Once);
}

private void CreateNewHDInsightCluster(bool addSecurityProfileInresponse = false)
private void CreateNewHDInsightCluster(bool addSecurityProfileInresponse = false, bool setEdgeNodeVmSize = false)
{
cmdlet.ClusterName = ClusterName;
cmdlet.ResourceGroupName = ResourceGroupName;
Expand All @@ -207,15 +217,17 @@ private void CreateNewHDInsightCluster(bool addSecurityProfileInresponse = false
cmdlet.ClusterType = ClusterType;
cmdlet.SshCredential = _sshCred;
cmdlet.OSType = OSType.Linux;

if (setEdgeNodeVmSize)
cmdlet.EdgeNodeSize = "edgeNodeVmSizeSetTest";

var cluster = new Cluster
{
Id = "id",
Name = ClusterName,
Location = Location,
Properties = new ClusterGetProperties
{
ClusterVersion = "3.1",
ClusterVersion = HdiVersion,
ClusterState = "Running",
ClusterDefinition = new ClusterDefinition
{
Expand Down Expand Up @@ -286,7 +298,8 @@ private void CreateNewHDInsightCluster(bool addSecurityProfileInresponse = false
parameters.ClusterType == ClusterType &&
parameters.OSType == OSType.Linux &&
parameters.SshUserName == _sshCred.UserName &&
parameters.SshPassword == _sshCred.Password.ConvertToString())))
parameters.SshPassword == _sshCred.Password.ConvertToString() &&
((!setEdgeNodeVmSize && parameters.EdgeNodeSize == null) || (setEdgeNodeVmSize && parameters.EdgeNodeSize == "edgeNodeVmSizeSetTest")))))
.Returns(getresponse)
.Verifiable();

Expand All @@ -297,7 +310,7 @@ private void CreateNewHDInsightCluster(bool addSecurityProfileInresponse = false
clusterout =>
clusterout.ClusterState == "Running" &&
clusterout.ClusterType == ClusterType &&
clusterout.ClusterVersion == "3.1" &&
clusterout.ClusterVersion == HdiVersion &&
clusterout.CoresUsed == 24 &&
clusterout.Location == Location &&
clusterout.Name == ClusterName &&
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
<package id="Microsoft.Azure.Gallery" version="2.6.2-preview" targetFramework="net45" />
<package id="Microsoft.Azure.KeyVault.Core" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.Azure.Management.Authorization" version="0.18.2-preview" targetFramework="net45" />
<package id="Microsoft.Azure.Management.HDInsight" version="2.0.1" targetFramework="net45" />
<package id="Microsoft.Azure.Management.HDInsight" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.Azure.Management.HDInsight.Job" version="2.0.3" targetFramework="net45" />
<package id="Microsoft.Azure.Management.Resources" version="2.20.0-preview" targetFramework="net45" />
<package id="Microsoft.Azure.Management.Storage" version="2.4.0-preview" targetFramework="net45" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@
<HintPath>..\..\..\packages\Microsoft.Azure.Graph.RBAC.3.2.0-preview\lib\net45\Microsoft.Azure.Graph.RBAC.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Azure.Management.HDInsight, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.2.0.1\lib\net40\Microsoft.Azure.Management.HDInsight.dll</HintPath>
<HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.2.0.3\lib\net40\Microsoft.Azure.Management.HDInsight.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Azure.Management.HDInsight.Job">
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ public class AddAzureHDInsightConfigValuesCommand : HDInsightCmdletBase
{
private Dictionary<string, Hashtable> _configurations;

private const string RServerConfigurationKey = "RServer";

#region Input Parameter Definitions

[Parameter(Position = 0,
Expand Down Expand Up @@ -77,6 +79,9 @@ public class AddAzureHDInsightConfigValuesCommand : HDInsightCmdletBase
[Parameter(HelpMessage = "Gets the Hdfs Site configurations of this HDInsight cluster.")]
public Hashtable Hdfs { get; set; }

[Parameter(HelpMessage = "Gets the RServer configurations.")]
public Hashtable RServer { get; set; }

#endregion

public AddAzureHDInsightConfigValuesCommand()
Expand All @@ -94,6 +99,7 @@ public AddAzureHDInsightConfigValuesCommand()
MapRed = new Hashtable();
Tez = new Hashtable();
Hdfs = new Hashtable();
RServer = new Hashtable();
}

public override void ExecuteCmdlet()
Expand All @@ -113,6 +119,7 @@ public override void ExecuteCmdlet()
AddConfigToConfigurations(MapRed, ConfigurationKey.MapRedSite);
AddConfigToConfigurations(Tez, ConfigurationKey.TezSite);
AddConfigToConfigurations(Hdfs, ConfigurationKey.HdfsSite);
AddConfigToConfigurations(RServer, RServerConfigurationKey);

WriteObject(Config);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,7 @@ public AzureHDInsightConfig Config
DefaultStorageAccountKey = _defaultStorageAccountKey,
WorkerNodeSize = parameters.WorkerNodeSize,
HeadNodeSize = parameters.HeadNodeSize,
EdgeNodeSize = parameters.EdgeNodeSize,
ZookeeperNodeSize = parameters.ZookeeperNodeSize,
HiveMetastore = HiveMetastore,
OozieMetastore = OozieMetastore,
Expand Down Expand Up @@ -176,6 +177,7 @@ var storageAccount in
}
parameters.WorkerNodeSize = value.WorkerNodeSize;
parameters.HeadNodeSize = value.HeadNodeSize;
parameters.EdgeNodeSize = value.EdgeNodeSize;
parameters.ZookeeperNodeSize = value.ZookeeperNodeSize;
HiveMetastore = value.HiveMetastore;
OozieMetastore = value.OozieMetastore;
Expand Down Expand Up @@ -254,6 +256,13 @@ public string WorkerNodeSize
set { parameters.WorkerNodeSize = value; }
}

[Parameter(HelpMessage = "Gets or sets the size of the Edge Node if available for the cluster type.")]
public string EdgeNodeSize
{
get { return parameters.EdgeNodeSize; }
set { parameters.EdgeNodeSize = value; }
}

[Parameter(HelpMessage = "Gets or sets the size of the Zookeeper Node.")]
public string ZookeeperNodeSize
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,13 @@ public string WorkerNodeSize
set { _config.WorkerNodeSize = value; }
}

[Parameter(HelpMessage = "Gets or sets the size of the Edge Node if available for the cluster type.")]
public string EdgeNodeSize
{
get { return _config.EdgeNodeSize; }
set { _config.EdgeNodeSize = value; }
}

[Parameter(HelpMessage = "Gets or sets the size of the Zookeeper Node.")]
public string ZookeeperNodeSize
{
Expand Down
Loading