Skip to content

Commit af39f74

Browse files
ogailogail
committed
Merge pull request #713 from shefaliv/hdinsight
HDInsight bug fixes
2 parents 50aeea9 + 9ef8d54 commit af39f74

File tree

7 files changed

+81
-41
lines changed

7 files changed

+81
-41
lines changed

‎setup/azurecmdfiles.wxi

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1267,6 +1267,9 @@
12671267
<Component Id="cmpF417A81D5F1E4FA1121F5CF7B9DC8439" Guid="*">
12681268
<File Id="fil43DE88B90B4EAEB16296A84082C1DCF1" KeyPath="yes" Source="$(var.sourceDir)\ResourceManager\AzureResourceManager\HDInsight\Microsoft.Azure.Commands.HDInsight.dll" />
12691269
</Component>
1270+
<Component Id="cmp278E6AC149924003FD07F271A0D39B73" Guid="*">
1271+
<File Id="filB6023ABB5409C60A6392D145BB79604E" KeyPath="yes" Source="$(var.sourceDir)\ResourceManager\AzureResourceManager\HDInsight\Microsoft.Azure.Commands.HDInsight.dll-help.psd1" />
1272+
</Component>
12701273
<Component Id="cmp0CEB7817889A2764028EDD1A3E0955C8" Guid="*">
12711274
<File Id="fil843773A2EA2CC25507CE5DFA567E6722" KeyPath="yes" Source="$(var.sourceDir)\ResourceManager\AzureResourceManager\HDInsight\Microsoft.Azure.Common.Authentication.dll" />
12721275
</Component>
@@ -5349,6 +5352,7 @@
53495352
<ComponentRef Id="cmpAB177D1334F111EB52C17D3C8B63D74F" />
53505353
<ComponentRef Id="cmp1516DA5C976539D7953783DBE0C35FC0" />
53515354
<ComponentRef Id="cmpF417A81D5F1E4FA1121F5CF7B9DC8439" />
5355+
<ComponentRef Id="cmp278E6AC149924003FD07F271A0D39B73" />
53525356
<ComponentRef Id="cmp0CEB7817889A2764028EDD1A3E0955C8" />
53535357
<ComponentRef Id="cmp12254838A1AB62028019D3E828B5EDBC" />
53545358
<ComponentRef Id="cmpCEC1013D8449BCE69C91DCA01F609D2E" />

‎src/ResourceManager/HDInsight/Commands.HDInsight/Commands.HDInsight.csproj

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,9 @@
180180
<Name>Commands.Common</Name>
181181
</ProjectReference>
182182
</ItemGroup>
183+
<ItemGroup>
184+
<Content Include="Microsoft.Azure.Commands.HDInsight.dll-help.Help.xml" />
185+
</ItemGroup>
183186
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
184187
<Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" />
185188
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">

‎src/ResourceManager/HDInsight/Commands.HDInsight/Constants.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ public static class CommandNames
3030
public const string AzureHDInsightJob = "AzureHDInsightJob";
3131
public const string AzureHDInsightJobOutput = "AzureHDInsightJobOutput";
3232
public const string AzureHDInsightDefaultStorage = "AzureHDInsightDefaultStorage";
33-
public const string Hive = "Hive";
33+
public const string AzureHDInsightHiveJob = "AzureHDInsightHiveJob";
3434
}
3535

3636
public static class JobDefinitions

‎src/ResourceManager/HDInsight/Commands.HDInsight/JobCommands/InvokeHiveCommand.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
namespace Microsoft.Azure.Commands.HDInsight
2424
{
2525
[Cmdlet(VerbsLifecycle.Invoke,
26-
Constants.CommandNames.Hive),
26+
Constants.CommandNames.AzureHDInsightHiveJob),
2727
OutputType(typeof(string))]
2828
public class InvokeHiveCommand : HDInsightCmdletBase
2929
{

‎src/ResourceManager/HDInsight/Commands.HDInsight/ManagementCommands/AddAzureHDInsightConfigValuesCommand.cs

Lines changed: 42 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
// limitations under the License.
1313
// ----------------------------------------------------------------------------------
1414

15+
using System.Collections;
1516
using System.Collections.Generic;
1617
using System.Management.Automation;
1718
using Microsoft.Azure.Commands.HDInsight.Commands;
@@ -27,7 +28,7 @@ namespace Microsoft.Azure.Commands.HDInsight
2728
typeof(AzureHDInsightConfig))]
2829
public class AddAzureHDInsightConfigValuesCommand : HDInsightCmdletBase
2930
{
30-
private Dictionary<string, Dictionary<string, string>> _configurations;
31+
private Dictionary<string, Hashtable> _configurations;
3132

3233
#region Input Parameter Definitions
3334

@@ -38,66 +39,66 @@ public class AddAzureHDInsightConfigValuesCommand : HDInsightCmdletBase
3839
public AzureHDInsightConfig Config { get; set; }
3940

4041
[Parameter(HelpMessage = "Gets the Core Site configurations of this HDInsight cluster.")]
41-
public Dictionary<string, string> Core { get; set; }
42+
public Hashtable Core { get; set; }
4243

4344
[Parameter(HelpMessage = "Gets the Hive Site configurations of this HDInsight cluster.")]
44-
public Dictionary<string, string> HiveSite { get; set; }
45+
public Hashtable HiveSite { get; set; }
4546

4647
[Parameter(HelpMessage = "Gets the Hive Env configurations of this HDInsight cluster.")]
47-
public Dictionary<string, string> HiveEnv { get; set; }
48+
public Hashtable HiveEnv { get; set; }
4849

4950
[Parameter(HelpMessage = "Gets the Oozie Site configurations of this HDInsight cluster.")]
50-
public Dictionary<string, string> OozieSite { get; set; }
51+
public Hashtable OozieSite { get; set; }
5152

5253
[Parameter(HelpMessage = "Gets the Oozie Env configurations of this HDInsight cluster.")]
53-
public Dictionary<string, string> OozieEnv { get; set; }
54+
public Hashtable OozieEnv { get; set; }
5455

5556
[Parameter(HelpMessage = "Gets the WebHCat Site configurations of this HDInsight cluster.")]
56-
public Dictionary<string, string> WebHCat { get; set; }
57+
public Hashtable WebHCat { get; set; }
5758

5859
[Parameter(HelpMessage = "Gets the HBase Site configurations of this HDInsight cluster.")]
59-
public Dictionary<string, string> HBaseSite { get; set; }
60+
public Hashtable HBaseSite { get; set; }
6061

6162
[Parameter(HelpMessage = "Gets the HBase Env configurations of this HDInsight cluster.")]
62-
public Dictionary<string, string> HBaseEnv { get; set; }
63+
public Hashtable HBaseEnv { get; set; }
6364

6465
[Parameter(HelpMessage = "Gets the Storm Site configurations of this HDInsight cluster.")]
65-
public Dictionary<string, string> Storm { get; set; }
66+
public Hashtable Storm { get; set; }
6667

6768
[Parameter(HelpMessage = "Gets the Yarn Site configurations of this HDInsight cluster.")]
68-
public Dictionary<string, string> Yarn { get; set; }
69+
public Hashtable Yarn { get; set; }
6970

7071
[Parameter(HelpMessage = "Gets the MapRed Site configurations of this HDInsight cluster.")]
71-
public Dictionary<string, string> MapRed { get; set; }
72+
public Hashtable MapRed { get; set; }
7273

7374
[Parameter(HelpMessage = "Gets the Tez Site configurations of this HDInsight cluster.")]
74-
public Dictionary<string, string> Tez { get; set; }
75+
public Hashtable Tez { get; set; }
7576

7677
[Parameter(HelpMessage = "Gets the Hdfs Site configurations of this HDInsight cluster.")]
77-
public Dictionary<string, string> Hdfs { get; set; }
78+
public Hashtable Hdfs { get; set; }
7879

7980
#endregion
8081

8182
public AddAzureHDInsightConfigValuesCommand()
8283
{
83-
Core = new Dictionary<string, string>();
84-
HiveSite = new Dictionary<string, string>();
85-
HiveEnv = new Dictionary<string, string>();
86-
OozieSite = new Dictionary<string, string>();
87-
OozieEnv = new Dictionary<string, string>();
88-
WebHCat = new Dictionary<string, string>();
89-
HBaseSite = new Dictionary<string, string>();
90-
HBaseEnv = new Dictionary<string, string>();
91-
Storm = new Dictionary<string, string>();
92-
Yarn = new Dictionary<string, string>();
93-
MapRed = new Dictionary<string, string>();
94-
Tez = new Dictionary<string, string>();
95-
Hdfs = new Dictionary<string, string>();
84+
Core = new Hashtable();
85+
HiveSite = new Hashtable();
86+
HiveEnv = new Hashtable();
87+
OozieSite = new Hashtable();
88+
OozieEnv = new Hashtable();
89+
WebHCat = new Hashtable();
90+
HBaseSite = new Hashtable();
91+
HBaseEnv = new Hashtable();
92+
Storm = new Hashtable();
93+
Yarn = new Hashtable();
94+
MapRed = new Hashtable();
95+
Tez = new Hashtable();
96+
Hdfs = new Hashtable();
9697
}
9798

9899
public override void ExecuteCmdlet()
99100
{
100-
_configurations = Config.Configurations ?? new Dictionary<string, Dictionary<string, string>>();
101+
_configurations = Config.Configurations ?? new Dictionary<string, Hashtable>();
101102

102103
AddConfigToConfigurations(Core, ConfigurationKey.CoreSite);
103104
AddConfigToConfigurations(HiveSite, ConfigurationKey.HiveSite);
@@ -116,15 +117,16 @@ public override void ExecuteCmdlet()
116117
WriteObject(Config);
117118
}
118119

119-
private void AddConfigToConfigurations(Dictionary<string, string> userConfigs, string configKey)
120+
private void AddConfigToConfigurations(Hashtable userConfigs, string configKey)
120121
{
122+
//var userConfigs = HashtableToDictionary(configs);
121123
//if no configs of this type provided, do nothing
122124
if (userConfigs == null || userConfigs.Count == 0)
123125
{
124126
return;
125127
}
126128

127-
Dictionary<string, string> config;
129+
Hashtable config;
128130

129131
//if configs provided and key does not already exist, add the key with provided dictionary
130132
if (!_configurations.TryGetValue(configKey, out config))
@@ -134,11 +136,18 @@ private void AddConfigToConfigurations(Dictionary<string, string> userConfigs, s
134136
}
135137

136138
//if configs provided and key already exists, add the provided values to the dictionary for the key
137-
foreach (var conf in userConfigs)
139+
var updatedConfig = ConcatHashtables(config, userConfigs);
140+
141+
_configurations[configKey] = updatedConfig;
142+
}
143+
144+
private static Hashtable ConcatHashtables(Hashtable first, Hashtable second)
145+
{
146+
foreach (DictionaryEntry item in second)
138147
{
139-
config.Add(conf.Key, conf.Value);
148+
first[item.Key] = item.Value;
140149
}
141-
_configurations[configKey] = config;
150+
return first;
142151
}
143152
}
144153
}

‎src/ResourceManager/HDInsight/Commands.HDInsight/ManagementCommands/NewAzureHDInsightClusterCommand.cs

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
// ----------------------------------------------------------------------------------
1414

1515
using System;
16+
using System.Collections;
1617
using System.Collections.Generic;
1718
using System.Linq;
1819
using System.Management.Automation;
@@ -208,7 +209,7 @@ var storageAccount in
208209
}
209210
foreach (var val in parameters.Configurations.Where(val => !result.Configurations.ContainsKey(val.Key)))
210211
{
211-
result.Configurations.Add(val.Key, val.Value);
212+
result.Configurations.Add(val.Key, DictionaryToHashtable(val.Value));
212213
}
213214
return result;
214215
}
@@ -231,13 +232,14 @@ var storageAccount in
231232
}
232233
foreach (var val in value.Configurations.Where(val => !parameters.Configurations.ContainsKey(val.Key)))
233234
{
234-
parameters.Configurations.Add(val.Key, val.Value);
235+
parameters.Configurations.Add(val.Key, HashtableToDictionary(val.Value));
235236
}
236237
}
237238
}
238239

239240
#endregion
240241

242+
241243
public NewAzureHDInsightClusterCommand()
242244
{
243245
parameters = new ClusterCreateParameters();
@@ -256,11 +258,20 @@ public override void ExecuteCmdlet()
256258
parameters.RdpUsername = RdpCredential.UserName;
257259
parameters.RdpPassword = RdpCredential.Password.ConvertToString();
258260
}
259-
if (SshCredential != null)
261+
262+
if (OSType == OSType.Linux)
260263
{
261264
parameters.SshUserName = SshCredential.UserName;
262-
parameters.SshPassword = SshCredential.Password.ConvertToString();
265+
if (!string.IsNullOrEmpty(SshCredential.Password.ConvertToString()))
266+
{
267+
parameters.SshPassword = SshCredential.Password.ConvertToString();
268+
}
269+
if (!string.IsNullOrEmpty(SshPublicKey))
270+
{
271+
parameters.SshPublicKey = SshPublicKey;
272+
}
263273
}
274+
264275
foreach (
265276
var storageAccount in
266277
AdditionalStorageAccounts.Where(
@@ -294,5 +305,17 @@ var storageAccount in
294305
WriteObject(new AzureHDInsightCluster(cluster.Cluster));
295306
}
296307
}
308+
309+
private static Hashtable DictionaryToHashtable(Dictionary<string, string> dictionary)
310+
{
311+
return new Hashtable(dictionary);
312+
}
313+
314+
private static Dictionary<string, string> HashtableToDictionary(Hashtable table)
315+
{
316+
return table
317+
.Cast<DictionaryEntry>()
318+
.ToDictionary(kvp => (string)kvp.Key, kvp => (string)kvp.Value);
319+
}
297320
}
298321
}

‎src/ResourceManager/HDInsight/Commands.HDInsight/Models/Management/AzureHDInsightConfig.cs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
// limitations under the License.
1313
// ----------------------------------------------------------------------------------
1414

15+
using System.Collections;
1516
using System.Collections.Generic;
1617
using Microsoft.Azure.Management.HDInsight.Models;
1718

@@ -67,7 +68,7 @@ public class AzureHDInsightConfig
6768
/// <summary>
6869
/// Gets the configurations of this HDInsight cluster.
6970
/// </summary>
70-
public Dictionary<string, Dictionary<string, string>> Configurations { get; private set; }
71+
public Dictionary<string, Hashtable> Configurations { get; private set; }
7172

7273
/// <summary>
7374
/// Gets config actions for the cluster.
@@ -78,7 +79,7 @@ public AzureHDInsightConfig()
7879
{
7980
ClusterType = HDInsightClusterType.Hadoop;
8081
AdditionalStorageAccounts = new Dictionary<string, string>();
81-
Configurations = new Dictionary<string, Dictionary<string, string>>();
82+
Configurations = new Dictionary<string, Hashtable>();
8283
ScriptActions = new Dictionary<ClusterNodeType, List<ScriptAction>>();
8384
}
8485
}

0 commit comments

Comments
 (0)