Skip to content

Commit bc1b9ba

Browse files
stankovskistankovski
committed
Merge pull request #1021 from shefaliv/dev
HDInsight Bug Fixes
2 parents 5476ccf + 3f07f40 commit bc1b9ba

File tree

55 files changed

+509
-168
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

55 files changed

+509
-168
lines changed

‎src/ResourceManager/HDInsight/Commands.HDInsight.Test/UnitTests/JobTests.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -203,7 +203,7 @@ public void StartJob()
203203
{
204204
CommandRuntime = commandRuntimeMock.Object,
205205
HDInsightJobClient = hdinsightJobManagementMock.Object,
206-
ClusterCredential = new PSCredential("httpuser", string.Format("Password1!").ConvertToSecureString()),
206+
HttpCredential = new PSCredential("httpuser", string.Format("Password1!").ConvertToSecureString()),
207207
ClusterName = ClusterName
208208
};
209209

@@ -245,7 +245,7 @@ public void StartJob()
245245
JobDetail = new JobDetailRootJsonObject
246246
{
247247
Completed = "false",
248-
User = cmdlet.ClusterCredential.UserName,
248+
User = cmdlet.HttpCredential.UserName,
249249
Id = jobid
250250
}
251251
};

‎src/ResourceManager/HDInsight/Commands.HDInsight/Commands.HDInsight.csproj

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
</PropertyGroup>
4040
<ItemGroup>
4141
<Compile Include="JobCommands\GetAzureHDInsightJobCommand.cs" />
42+
<Compile Include="JobCommands\NewAzureHDInsightSqoopJobDefinitionCommand.cs" />
4243
<Compile Include="JobCommands\WaitAzureHDInsightJobCommand.cs" />
4344
<Compile Include="JobCommands\NewAzureHDInsightStreamingMapReduceJobDefinitionCommand.cs" />
4445
<Compile Include="JobCommands\NewAzureHDInsightMapReduceJobDefinitionCommand.cs" />
@@ -65,12 +66,14 @@
6566
<Compile Include="HDInsightCmdletBase.cs" />
6667
<Compile Include="ManagementCommands\GetAzureHDInsightClusterCommand.cs" />
6768
<Compile Include="Constants.cs" />
69+
<Compile Include="Models\Job\AzureHDInsightSqoopJobDefinition.cs" />
6870
<Compile Include="Models\Job\AzureHDInsightJob.cs" />
6971
<Compile Include="Models\Job\AzureHDInsightMapReduceJobDefinition.cs" />
7072
<Compile Include="Models\Job\AzureHDInsightStreamingMapReduceJobDefinition.cs" />
7173
<Compile Include="Models\Job\AzureHDInsightPigJobDefinition.cs" />
7274
<Compile Include="Models\Job\AzureHDInsightHiveJobDefinition.cs" />
7375
<Compile Include="Models\Job\AzureHDInsightJobDefinition.cs" />
76+
<Compile Include="Models\Job\JobDisplayOutputType.cs" />
7477
<Compile Include="Models\Management\AzureHDInsightCluster.cs" />
7578
<Compile Include="Models\Management\AzureHDInsightConfig.cs" />
7679
<Compile Include="Models\Job\AzureHdInsightJobManagementClient.cs" />
@@ -80,6 +83,7 @@
8083
<Compile Include="Models\Management\AzureHDInsightMetastore.cs" />
8184
<Compile Include="ManagementCommands\NewAzureHDInsightClusterCommand.cs" />
8285
<Compile Include="ClusterConfigurationUtils.cs" />
86+
<Compile Include="Models\Management\AzureHDInsightScriptAction.cs" />
8387
<Compile Include="Properties\AssemblyInfo.cs" />
8488
</ItemGroup>
8589
<ItemGroup>
@@ -111,11 +115,11 @@
111115
</Reference>
112116
<Reference Include="Microsoft.Azure.Management.HDInsight, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
113117
<SpecificVersion>False</SpecificVersion>
114-
<HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.1.0.5-preview\lib\net40\Microsoft.Azure.Management.HDInsight.dll</HintPath>
118+
<HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.1.0.6-preview\lib\net40\Microsoft.Azure.Management.HDInsight.dll</HintPath>
115119
</Reference>
116120
<Reference Include="Microsoft.Azure.Management.HDInsight.Job, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
117121
<SpecificVersion>False</SpecificVersion>
118-
<HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.Job.1.0.5-preview\lib\net40\Microsoft.Azure.Management.HDInsight.Job.dll</HintPath>
122+
<HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.Job.1.0.6-preview\lib\net40\Microsoft.Azure.Management.HDInsight.Job.dll</HintPath>
119123
</Reference>
120124
<Reference Include="Microsoft.Azure.ResourceManager, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
121125
<SpecificVersion>False</SpecificVersion>

‎src/ResourceManager/HDInsight/Commands.HDInsight/Constants.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ public static class JobDefinitions
3939
public const string AzureHDInsightPigJobDefinition = "AzureRmHDInsightPigJobDefinition";
4040
public const string AzureHDInsightMapReduceJobDefinition = "AzureRmHDInsightMapReduceJobDefinition";
4141
public const string AzureHDInsightStreamingMapReduceJobDefinition = "AzureRmHDInsightStreamingMapReduceJobDefinition";
42+
public const string AzureHDInsightSqoopJobDefinition = "AzureRmHDInsightSqoopJobDefinition";
4243
}
4344

4445
public static class ClusterConfiguration

‎src/ResourceManager/HDInsight/Commands.HDInsight/HDInsightCmdletBase.cs

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,5 +83,21 @@ protected string GetClusterConnection(string resourceGroupName, string clusterNa
8383
}
8484
return httpEndpoint;
8585
}
86+
87+
protected string GetResourceGroupByAccountName(string clusterName)
88+
{
89+
try
90+
{
91+
var clusterId = HDInsightManagementClient.ListClusters().First(x => x.Name.Equals(clusterName, StringComparison.InvariantCultureIgnoreCase)).Id;
92+
var rgStart = clusterId.IndexOf("resourceGroups/", StringComparison.InvariantCultureIgnoreCase) + ("resourceGroups/".Length);
93+
var rgLength = (clusterId.IndexOf("/providers/", StringComparison.InvariantCultureIgnoreCase)) - rgStart;
94+
return clusterId.Substring(rgStart, rgLength);
95+
}
96+
catch
97+
{
98+
throw new CloudException(string.Format("Could not find resource group for cluster {0}.", clusterName));
99+
}
100+
}
101+
86102
}
87103
}

‎src/ResourceManager/HDInsight/Commands.HDInsight/JobCommands/GetAzureHDInsightJobCommand.cs

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -26,14 +26,9 @@ namespace Microsoft.Azure.Commands.HDInsight
2626
OutputType(typeof(AzureHDInsightJob))]
2727
public class GetAzureHDInsightJobCommand : HDInsightCmdletBase
2828
{
29-
[Parameter(
30-
Position = 0,
31-
Mandatory = true,
32-
HelpMessage = "Gets or sets the name of the resource group.")]
33-
public string ResourceGroupName { get; set; }
34-
29+
#region Input Parameter Definitions
3530
[Parameter(Mandatory = true,
36-
Position = 1,
31+
Position = 0,
3732
HelpMessage = "The name of the cluster.")]
3833
public string ClusterName
3934
{
@@ -42,9 +37,10 @@ public string ClusterName
4237
}
4338

4439
[Parameter(Mandatory = true,
45-
Position = 2,
40+
Position = 1,
4641
HelpMessage = "The credentials with which to connect to the cluster.")]
47-
public PSCredential ClusterCredential
42+
[Alias("ClusterCredential")]
43+
public PSCredential HttpCredential
4844
{
4945
get
5046
{
@@ -62,12 +58,22 @@ public PSCredential ClusterCredential
6258
}
6359
}
6460

65-
[Parameter(Position = 3,
61+
[Parameter(Position = 2,
6662
HelpMessage = "The JobID of the jobDetails to stop.")]
6763
public string JobId { get; set; }
6864

65+
[Parameter(HelpMessage = "Gets or sets the name of the resource group.")]
66+
public string ResourceGroupName { get; set; }
67+
68+
#endregion
69+
70+
6971
protected override void ProcessRecord()
7072
{
73+
if (ResourceGroupName == null)
74+
{
75+
ResourceGroupName = GetResourceGroupByAccountName(ClusterName);
76+
}
7177
_clusterName = GetClusterConnection(ResourceGroupName, ClusterName);
7278
if (JobId != null)
7379
{

‎src/ResourceManager/HDInsight/Commands.HDInsight/JobCommands/GetAzureHDInsightJobOutputCommand.cs

Lines changed: 74 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
using System.Management.Automation;
1717
using Hyak.Common;
1818
using Microsoft.Azure.Commands.HDInsight.Commands;
19+
using Microsoft.Azure.Commands.HDInsight.Models.Job;
1920
using Microsoft.WindowsAzure.Commands.Common;
2021

2122
namespace Microsoft.Azure.Commands.HDInsight
@@ -25,45 +26,42 @@ namespace Microsoft.Azure.Commands.HDInsight
2526
OutputType(typeof(string))]
2627
public class GetAzureHDInsightJobOutputCommand : HDInsightCmdletBase
2728
{
28-
[Parameter(
29-
Position = 0,
30-
Mandatory = true,
31-
HelpMessage = "Gets or sets the name of the resource group.")]
32-
public string ResourceGroupName { get; set; }
29+
#region Input Parameter Definitions
3330

3431
[Parameter(Mandatory = true,
35-
Position = 1,
32+
Position = 0,
3633
HelpMessage = "The name of the cluster.")]
3734
public string ClusterName
3835
{
3936
get { return _clusterName; }
4037
set { _clusterName = value; }
4138
}
4239

43-
[Parameter(Position = 2,
40+
[Parameter(Position = 1,
4441
Mandatory = true,
4542
HelpMessage = "The JobID of the jobDetails to stop.")]
4643
public string JobId { get; set; }
4744

48-
[Parameter(Position = 3,
45+
[Parameter(Position = 2,
4946
Mandatory = true,
5047
HelpMessage = "The default container name.")]
5148
public string DefaultContainer { get; set; }
5249

53-
[Parameter(Position = 4,
50+
[Parameter(Position = 3,
5451
Mandatory = true,
5552
HelpMessage = "The default storage account name.")]
5653
public string DefaultStorageAccountName { get; set; }
5754

58-
[Parameter(Position = 5,
55+
[Parameter(Position = 4,
5956
Mandatory = true,
6057
HelpMessage = "The default storage account key.")]
6158
public string DefaultStorageAccountKey { get; set; }
6259

6360
[Parameter(Mandatory = true,
64-
Position = 6,
61+
Position = 5,
6562
HelpMessage = "The credentials with which to connect to the cluster.")]
66-
public PSCredential ClusterCredential
63+
[Alias("ClusterCredential")]
64+
public PSCredential HttpCredential
6765
{
6866
get
6967
{
@@ -79,20 +77,80 @@ public PSCredential ClusterCredential
7977
}
8078
}
8179

80+
[Parameter(HelpMessage = "Gets or sets the name of the resource group.")]
81+
public string ResourceGroupName { get; set; }
82+
83+
[Parameter(HelpMessage = "The type of job output.", ParameterSetName = "Display")]
84+
public JobDisplayOutputType DisplayOutputType { get; set; }
85+
86+
[Parameter(Mandatory = true, HelpMessage = "The type of output to download.", ParameterSetName = "Download")]
87+
public JobDownloadOutputType DownloadOutputType { get; set; }
88+
89+
[Parameter(Mandatory = true, HelpMessage = "The folder to save the output to.", ParameterSetName = "Download")]
90+
public string Folder { get; set; }
91+
92+
#endregion
93+
8294
protected override void ProcessRecord()
8395
{
96+
if (ResourceGroupName == null)
97+
{
98+
ResourceGroupName = GetResourceGroupByAccountName(ClusterName);
99+
}
84100
_clusterName = GetClusterConnection(ResourceGroupName, ClusterName);
85-
var output = GetJobOutput();
86-
WriteObject(output);
87-
}
88101

89-
public string GetJobOutput()
102+
if (ParameterSetName == "Display")
103+
{
104+
string output;
105+
switch (DisplayOutputType)
106+
{
107+
case JobDisplayOutputType.StandardError:
108+
output = GetJobError();
109+
break;
110+
case JobDisplayOutputType.TaskSummary:
111+
output = GetJobTaskLogSummary();
112+
break;
113+
default:
114+
output = GetJobOutput();
115+
break;
116+
}
117+
WriteObject(output);
118+
}
119+
else
120+
{
121+
DownloadJobTaskLogs();
122+
}
123+
}
124+
125+
internal string GetJobOutput()
90126
{
91127
var output = HDInsightJobClient.GetJobOutput(JobId, DefaultStorageAccountName, DefaultStorageAccountKey, DefaultContainer);
92128
var outputStr = Convert(output);
93129
return outputStr;
94130
}
95131

132+
private string GetJobError()
133+
{
134+
var output = HDInsightJobClient.GetJobError(JobId, DefaultStorageAccountName, DefaultStorageAccountKey,
135+
DefaultContainer);
136+
var outputStr = Convert(output);
137+
return outputStr;
138+
}
139+
140+
private string GetJobTaskLogSummary()
141+
{
142+
var output = HDInsightJobClient.GetJobTaskLogSummary(JobId, DefaultStorageAccountName, DefaultStorageAccountKey,
143+
DefaultContainer);
144+
var outputStr = Convert(output);
145+
return outputStr;
146+
}
147+
148+
private void DownloadJobTaskLogs()
149+
{
150+
HDInsightJobClient.GetJobTaskLogSummary(JobId, DefaultStorageAccountName, DefaultStorageAccountKey,
151+
DefaultContainer);
152+
}
153+
96154
private static string Convert(Stream stream)
97155
{
98156
var reader = new StreamReader(stream);

‎src/ResourceManager/HDInsight/Commands.HDInsight/JobCommands/InvokeHiveCommand.cs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ protected override void ProcessRecord()
109109
//get variables from session
110110
var clusterConnection = SessionState.PSVariable.Get(UseAzureHDInsightClusterCommand.ClusterEndpoint).Value.ToString();
111111
var clusterCred =
112-
(PSCredential) SessionState.PSVariable.Get(UseAzureHDInsightClusterCommand.ClusterCred).Value;
112+
(PSCredential)SessionState.PSVariable.Get(UseAzureHDInsightClusterCommand.ClusterCred).Value;
113113
var resourceGroup =
114114
SessionState.PSVariable.Get(UseAzureHDInsightClusterCommand.CurrentResourceGroup).Value.ToString();
115115

@@ -137,7 +137,7 @@ protected override void ProcessRecord()
137137
ClusterName = clusterConnection,
138138
ResourceGroupName = resourceGroup,
139139
JobDefinition = hivejob,
140-
ClusterCredential = clusterCred
140+
HttpCredential = clusterCred
141141
};
142142

143143
var jobCreationResult = startJobCommand.SubmitJob();
@@ -148,7 +148,7 @@ protected override void ProcessRecord()
148148
WriteProgress(new ProgressRecord(0, "Waiting for job to complete", "In Progress"));
149149
var waitJobCommand = new WaitAzureHDInsightJobCommand
150150
{
151-
ClusterCredential = clusterCred,
151+
HttpCredential = clusterCred,
152152
ResourceGroupName = resourceGroup,
153153
ClusterName = clusterConnection,
154154
JobId = jobid
@@ -161,7 +161,7 @@ protected override void ProcessRecord()
161161
//get job output
162162
var getOutputCommand = new GetAzureHDInsightJobOutputCommand
163163
{
164-
ClusterCredential = clusterCred,
164+
HttpCredential = clusterCred,
165165
ResourceGroupName = resourceGroup,
166166
ClusterName = clusterConnection,
167167
DefaultContainer = DefaultContainer,

0 commit comments

Comments
 (0)