Skip to content

Commit b8b8725

Browse files
pattipakapattipaka
committed
Fix Invoke Hive command issues and job pagination feature
1 parent 2d48f7d commit b8b8725

File tree

5 files changed

+86
-69
lines changed

5 files changed

+86
-69
lines changed

‎src/ResourceManager/HDInsight/Commands.HDInsight/JobCommands/GetAzureHDInsightJobCommand.cs

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,20 +62,29 @@ public PSCredential HttpCredential
6262
HelpMessage = "The JobID of the jobDetails to stop.")]
6363
public string JobId { get; set; }
6464

65+
[Parameter(HelpMessage = "Gets or sets the number of jobs to retrieve.")]
66+
public int NumOfJobs { get; set; }
67+
6568
[Parameter(HelpMessage = "Gets or sets the name of the resource group.")]
6669
public string ResourceGroupName { get; set; }
6770

6871
#endregion
6972

70-
7173
public override void ExecuteCmdlet()
7274
{
7375
if (ResourceGroupName == null)
7476
{
7577
ResourceGroupName = GetResourceGroupByAccountName(ClusterName);
7678
}
79+
7780
_clusterName = GetClusterConnection(ResourceGroupName, ClusterName);
78-
if (JobId != null)
81+
82+
if (NumOfJobs > 0)
83+
{
84+
var jobs = HDInsightJobClient.ListJobsAfterJobId(JobId, NumOfJobs).Select(job => new AzureHDInsightJob(job.Detail, HDInsightJobClient.ClusterName));
85+
WriteObject(jobs, true);
86+
}
87+
else if (JobId != null)
7988
{
8089
var job = HDInsightJobClient.GetJob(JobId);
8190
var jobDetails = new AzureHDInsightJob(job.JobDetail, HDInsightJobClient.ClusterName);

‎src/ResourceManager/HDInsight/Commands.HDInsight/JobCommands/GetAzureHDInsightJobOutputCommand.cs

Lines changed: 45 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
using Microsoft.WindowsAzure.Commands.Common;
2222
using Microsoft.Azure.Commands.HDInsight.Models;
2323
using Microsoft.Azure.Management.HDInsight.Job.Models;
24+
using Microsoft.Azure.Management.HDInsight;
2425

2526
namespace Microsoft.Azure.Commands.HDInsight
2627
{
@@ -94,65 +95,33 @@ public override void ExecuteCmdlet()
9495
ResourceGroupName = GetResourceGroupByAccountName(ClusterName);
9596
}
9697

97-
if (DefaultContainer == null || DefaultStorageAccountName == null || DefaultStorageAccountKey == null)
98-
{
99-
var result = HDInsightManagementClient.GetCluster(ResourceGroupName, _clusterName);
100-
101-
if (result == null || result.Count == 0)
102-
{
103-
throw new CloudException(string.Format("Couldn't find cluster {0}", _clusterName));
104-
}
105-
106-
var cluster = result.FirstOrDefault();
107-
string resourceGroupName = ClusterConfigurationUtils.GetResourceGroupFromClusterId(cluster.Id);
108-
var configuration = HDInsightManagementClient.GetClusterConfigurations(resourceGroupName, cluster.Name, "core-site");
109-
110-
if (configuration == null)
111-
{
112-
throw new CloudException(string.Format("Couldn't find storage information for cluster {0}", _clusterName));
113-
}
114-
115-
var DefaultStorageAccount = ClusterConfigurationUtils.GetDefaultStorageAccountDetails(
116-
configuration,
117-
cluster.Properties.ClusterVersion);
118-
119-
if (DefaultStorageAccount == null)
120-
{
121-
throw new CloudException(string.Format("Couldn't find storage information for cluster {0}", _clusterName));
122-
}
123-
124-
DefaultContainer = DefaultStorageAccount.StorageContainerName;
125-
DefaultStorageAccountName = DefaultStorageAccount.StorageAccountName;
126-
DefaultStorageAccountKey = DefaultStorageAccount.StorageAccountKey;
127-
}
128-
129-
storageAccess = new AzureStorageAccess(DefaultStorageAccountName, DefaultStorageAccountKey, DefaultContainer);
98+
storageAccess = GetDefaultStorageAccess(ResourceGroupName, _clusterName);
13099

131100
_clusterName = GetClusterConnection(ResourceGroupName, ClusterName);
132101

133102
string output;
134103
switch (DisplayOutputType)
135104
{
136105
case JobDisplayOutputType.StandardError:
137-
output = GetJobError();
106+
output = GetJobError(this.storageAccess);
138107
break;
139108
default:
140-
output = GetJobOutput();
109+
output = GetJobOutput(this.storageAccess);
141110
break;
142111
}
143112
WriteObject(output);
144113
}
145114

146-
internal string GetJobOutput()
115+
internal string GetJobOutput(IStorageAccess storageAccess)
147116
{
148-
var output = HDInsightJobClient.GetJobOutput(JobId, this.storageAccess);
117+
var output = HDInsightJobClient.GetJobOutput(JobId, storageAccess);
149118
var outputStr = Convert(output);
150119
return outputStr;
151120
}
152121

153-
private string GetJobError()
122+
internal string GetJobError(IStorageAccess storageAccess)
154123
{
155-
var output = HDInsightJobClient.GetJobError(JobId, this.storageAccess);
124+
var output = HDInsightJobClient.GetJobError(JobId, storageAccess);
156125
var outputStr = Convert(output);
157126
return outputStr;
158127
}
@@ -163,5 +132,42 @@ private static string Convert(Stream stream)
163132
var text = reader.ReadToEnd();
164133
return text;
165134
}
135+
136+
internal IStorageAccess GetDefaultStorageAccess(string ResourceGroupName, string clusterName)
137+
{
138+
if (DefaultContainer == null && DefaultStorageAccountName == null && DefaultStorageAccountKey == null)
139+
{
140+
var result = HDInsightManagementClient.GetCluster(ResourceGroupName, clusterName);
141+
142+
if (result == null || result.Count == 0)
143+
{
144+
throw new CloudException(string.Format("Couldn't find cluster {0}", clusterName));
145+
}
146+
147+
var cluster = result.FirstOrDefault();
148+
string resourceGroupName = ClusterConfigurationUtils.GetResourceGroupFromClusterId(cluster.Id);
149+
var configuration = HDInsightManagementClient.GetClusterConfigurations(resourceGroupName, cluster.Name, "core-site");
150+
151+
if (configuration == null)
152+
{
153+
throw new CloudException(string.Format("Couldn't find storage information for cluster {0}", clusterName));
154+
}
155+
156+
var DefaultStorageAccount = ClusterConfigurationUtils.GetDefaultStorageAccountDetails(
157+
configuration,
158+
cluster.Properties.ClusterVersion);
159+
160+
if (DefaultStorageAccount == null)
161+
{
162+
throw new CloudException(string.Format("Couldn't find storage information for cluster {0}", clusterName));
163+
}
164+
165+
DefaultContainer = DefaultStorageAccount.StorageContainerName;
166+
DefaultStorageAccountName = DefaultStorageAccount.StorageAccountName;
167+
DefaultStorageAccountKey = DefaultStorageAccount.StorageAccountKey;
168+
}
169+
170+
return new AzureStorageAccess(DefaultStorageAccountName, DefaultStorageAccountKey, DefaultContainer);
171+
}
166172
}
167173
}

‎src/ResourceManager/HDInsight/Commands.HDInsight/JobCommands/InvokeHiveCommand.cs

Lines changed: 24 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,13 @@
1515
using System;
1616
using System.Collections;
1717
using System.IO;
18+
using System.Linq;
1819
using System.Management.Automation;
1920
using Hyak.Common;
2021
using Microsoft.Azure.Commands.HDInsight.Commands;
2122
using Microsoft.WindowsAzure.Commands.Common;
2223
using Microsoft.Azure.Management.HDInsight.Job.Models;
24+
using Microsoft.Azure.Commands.HDInsight.Models;
2325

2426
namespace Microsoft.Azure.Commands.HDInsight
2527
{
@@ -46,8 +48,7 @@ public string[] Files
4648
set { hiveJobDefinitionCommand.Files = value; }
4749
}
4850

49-
[Parameter(Mandatory = true,
50-
HelpMessage = "The output location to use for the job.")]
51+
[Parameter(HelpMessage = "The output location to use for the job.")]
5152
public string StatusFolder
5253
{
5354
get { return hiveJobDefinitionCommand.StatusFolder; }
@@ -89,13 +90,13 @@ public SwitchParameter RunAsFileJob
8990
set { hiveJobDefinitionCommand.RunAsFileJob = value; }
9091
}
9192

92-
[Parameter(Mandatory = true, HelpMessage = "The default container name.")]
93+
[Parameter(HelpMessage = "The default container name.")]
9394
public string DefaultContainer { get; set; }
9495

95-
[Parameter(Mandatory = true, HelpMessage = "The default storage account name.")]
96+
[Parameter(HelpMessage = "The default storage account name.")]
9697
public string DefaultStorageAccountName { get; set; }
9798

98-
[Parameter(Mandatory = true, HelpMessage = "The default storage account key.")]
99+
[Parameter(HelpMessage = "The default storage account key.")]
99100
public string DefaultStorageAccountKey { get; set; }
100101

101102
#endregion
@@ -114,7 +115,6 @@ public override void ExecuteCmdlet()
114115
var resourceGroup =
115116
SessionState.PSVariable.Get(UseAzureHDInsightClusterCommand.CurrentResourceGroup).Value.ToString();
116117

117-
_clusterName = clusterConnection;
118118
_credential = new BasicAuthenticationCloudCredentials
119119
{
120120
Username = clusterCred.UserName,
@@ -156,38 +156,35 @@ public override void ExecuteCmdlet()
156156
};
157157

158158
var job = waitJobCommand.WaitJob();
159+
160+
_clusterName = clusterConnection.Substring(0, clusterConnection.IndexOf('.'));
161+
162+
var getOutputCommand = new GetAzureHDInsightJobOutputCommand
163+
{
164+
HttpCredential = clusterCred,
165+
ResourceGroupName = resourceGroup,
166+
ClusterName = clusterConnection,
167+
DefaultContainer = DefaultContainer,
168+
DefaultStorageAccountName = DefaultStorageAccountName,
169+
DefaultStorageAccountKey = DefaultStorageAccountKey,
170+
JobId = jobid
171+
};
172+
173+
var storageAccess = getOutputCommand.GetDefaultStorageAccess(resourceGroup, _clusterName);
174+
159175
string output;
160176
if (job.ExitValue == 0)
161177
{
162178
//get job output
163-
var getOutputCommand = new GetAzureHDInsightJobOutputCommand
164-
{
165-
HttpCredential = clusterCred,
166-
ResourceGroupName = resourceGroup,
167-
ClusterName = clusterConnection,
168-
DefaultContainer = DefaultContainer,
169-
DefaultStorageAccountName = DefaultStorageAccountName,
170-
DefaultStorageAccountKey = DefaultStorageAccountKey,
171-
JobId = jobid
172-
};
173-
174-
output = getOutputCommand.GetJobOutput();
179+
output = getOutputCommand.GetJobOutput(storageAccess);
175180
}
176181
else
177182
{
178183
//get job error
179-
IStorageAccess storageAccess = new AzureStorageAccess(DefaultStorageAccountName, DefaultStorageAccountKey, DefaultContainer);
180-
output = Convert(HDInsightJobClient.GetJobError(jobid, storageAccess));
184+
output = getOutputCommand.GetJobError(storageAccess);
181185
}
182186

183187
WriteObject(output);
184188
}
185-
186-
private static string Convert(Stream stream)
187-
{
188-
var reader = new StreamReader(stream);
189-
var text = reader.ReadToEnd();
190-
return text;
191-
}
192189
}
193190
}

‎src/ResourceManager/HDInsight/Commands.HDInsight/Models/Job/AzureHDInsightJob.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ public AzureHDInsightJob(JobDetailRootJsonObject jobDetails, string cluster)
3838
User = jobDetails.User;
3939
Callback = jobDetails.Callback;
4040
Completed = jobDetails.Completed;
41-
StatusFolder = jobDetails.Userargs.Statusdir.ToString();
41+
StatusFolder = jobDetails.Userargs.Statusdir != null ? jobDetails.Userargs.Statusdir.ToString() : string.Empty;
4242
}
4343

4444
/// <summary>

‎src/ResourceManager/HDInsight/Commands.HDInsight/Models/Job/AzureHdInsightJobManagementClient.cs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,11 @@ public virtual JobListResponse ListJobs()
126126
return HdInsightJobManagementClient.JobManagement.ListJobs();
127127
}
128128

129+
public virtual JobListResponse ListJobsAfterJobId(string jobId, int numberOfJobs)
130+
{
131+
return HdInsightJobManagementClient.JobManagement.ListJobsAfterJobId(jobId, numberOfJobs);
132+
}
133+
129134
public void StopJob(string jobId)
130135
{
131136
HdInsightJobManagementClient.JobManagement.KillJob(jobId);

0 commit comments

Comments
 (0)