Skip to content

Commit 2d48f7d

Browse files
pattipakapattipaka
committed
Move to Job SDK 2.0.0-preview
1 parent b3fe985 commit 2d48f7d

13 files changed

+100
-116
lines changed

‎src/ResourceManager/HDInsight/Commands.HDInsight.Test/Commands.HDInsight.Test.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@
5757
</Reference>
5858
<Reference Include="Microsoft.Azure.Management.HDInsight.Job">
5959
<SpecificVersion>False</SpecificVersion>
60-
<HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.Job.1.1.0-preview\lib\net40\Microsoft.Azure.Management.HDInsight.Job.dll</HintPath>
60+
<HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.Job.2.0.0-preview\lib\net40\Microsoft.Azure.Management.HDInsight.Job.dll</HintPath>
6161
</Reference>
6262
<Reference Include="Microsoft.Azure.Management.Storage">
6363
<HintPath>..\..\..\packages\Microsoft.Azure.Management.Storage.2.4.0-preview\lib\net40\Microsoft.Azure.Management.Storage.dll</HintPath>

‎src/ResourceManager/HDInsight/Commands.HDInsight.Test/packages.config

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
<package id="Microsoft.Azure.Gallery" version="2.6.2-preview" targetFramework="net45" />
88
<package id="Microsoft.Azure.Management.Authorization" version="0.18.2-preview" targetFramework="net45" />
99
<package id="Microsoft.Azure.Management.HDInsight" version="1.0.9-preview" targetFramework="net45" />
10-
<package id="Microsoft.Azure.Management.HDInsight.Job" version="1.1.0-preview" targetFramework="net45" />
10+
<package id="Microsoft.Azure.Management.HDInsight.Job" version="2.0.0-preview" targetFramework="net45" />
1111
<package id="Microsoft.Azure.Management.Resources" version="2.18.11-preview" targetFramework="net45" />
1212
<package id="Microsoft.Azure.Management.Storage" version="2.4.0-preview" targetFramework="net45" />
1313
<package id="Microsoft.Azure.Test.Framework" version="1.0.5799.28345-prerelease" targetFramework="net45" />

‎src/ResourceManager/HDInsight/Commands.HDInsight/Commands.HDInsight.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@
123123
</Reference>
124124
<Reference Include="Microsoft.Azure.Management.HDInsight.Job">
125125
<SpecificVersion>False</SpecificVersion>
126-
<HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.Job.1.1.0-preview\lib\net40\Microsoft.Azure.Management.HDInsight.Job.dll</HintPath>
126+
<HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.Job.2.0.0-preview\lib\net40\Microsoft.Azure.Management.HDInsight.Job.dll</HintPath>
127127
</Reference>
128128
<Reference Include="Microsoft.Azure.ResourceManager, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
129129
<SpecificVersion>False</SpecificVersion>

‎src/ResourceManager/HDInsight/Commands.HDInsight/JobCommands/GetAzureHDInsightJobOutputCommand.cs

Lines changed: 49 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,14 @@
1313
// ----------------------------------------------------------------------------------
1414

1515
using System.IO;
16+
using System.Linq;
1617
using System.Management.Automation;
1718
using Hyak.Common;
1819
using Microsoft.Azure.Commands.HDInsight.Commands;
1920
using Microsoft.Azure.Commands.HDInsight.Models.Job;
2021
using Microsoft.WindowsAzure.Commands.Common;
22+
using Microsoft.Azure.Commands.HDInsight.Models;
23+
using Microsoft.Azure.Management.HDInsight.Job.Models;
2124

2225
namespace Microsoft.Azure.Commands.HDInsight
2326
{
@@ -43,17 +46,14 @@ public string ClusterName
4346
public string JobId { get; set; }
4447

4548
[Parameter(Position = 2,
46-
Mandatory = true,
4749
HelpMessage = "The default container name.")]
4850
public string DefaultContainer { get; set; }
4951

5052
[Parameter(Position = 3,
51-
Mandatory = true,
5253
HelpMessage = "The default storage account name.")]
5354
public string DefaultStorageAccountName { get; set; }
5455

5556
[Parameter(Position = 4,
56-
Mandatory = true,
5757
HelpMessage = "The default storage account key.")]
5858
public string DefaultStorageAccountKey { get; set; }
5959

@@ -80,77 +80,83 @@ public PSCredential HttpCredential
8080
[Parameter(HelpMessage = "Gets or sets the name of the resource group.")]
8181
public string ResourceGroupName { get; set; }
8282

83-
[Parameter(HelpMessage = "The type of job output.", ParameterSetName = "Display")]
83+
[Parameter(HelpMessage = "The type of job output.")]
8484
public JobDisplayOutputType DisplayOutputType { get; set; }
8585

86-
[Parameter(Mandatory = true, HelpMessage = "The type of output to download.", ParameterSetName = "Download")]
87-
public JobDownloadOutputType DownloadOutputType { get; set; }
88-
89-
[Parameter(Mandatory = true, HelpMessage = "The folder to save the output to.", ParameterSetName = "Download")]
90-
public string Folder { get; set; }
91-
9286
#endregion
9387

88+
private IStorageAccess storageAccess = null;
89+
9490
public override void ExecuteCmdlet()
9591
{
9692
if (ResourceGroupName == null)
9793
{
9894
ResourceGroupName = GetResourceGroupByAccountName(ClusterName);
9995
}
100-
_clusterName = GetClusterConnection(ResourceGroupName, ClusterName);
10196

102-
if (ParameterSetName == "Display")
97+
if (DefaultContainer == null || DefaultStorageAccountName == null || DefaultStorageAccountKey == null)
10398
{
104-
string output;
105-
switch (DisplayOutputType)
99+
var result = HDInsightManagementClient.GetCluster(ResourceGroupName, _clusterName);
100+
101+
if (result == null || result.Count == 0)
102+
{
103+
throw new CloudException(string.Format("Couldn't find cluster {0}", _clusterName));
104+
}
105+
106+
var cluster = result.FirstOrDefault();
107+
string resourceGroupName = ClusterConfigurationUtils.GetResourceGroupFromClusterId(cluster.Id);
108+
var configuration = HDInsightManagementClient.GetClusterConfigurations(resourceGroupName, cluster.Name, "core-site");
109+
110+
if (configuration == null)
111+
{
112+
throw new CloudException(string.Format("Couldn't find storage information for cluster {0}", _clusterName));
113+
}
114+
115+
var DefaultStorageAccount = ClusterConfigurationUtils.GetDefaultStorageAccountDetails(
116+
configuration,
117+
cluster.Properties.ClusterVersion);
118+
119+
if (DefaultStorageAccount == null)
106120
{
107-
case JobDisplayOutputType.StandardError:
108-
output = GetJobError();
109-
break;
110-
case JobDisplayOutputType.TaskSummary:
111-
output = GetJobTaskLogSummary();
112-
break;
113-
default:
114-
output = GetJobOutput();
115-
break;
121+
throw new CloudException(string.Format("Couldn't find storage information for cluster {0}", _clusterName));
116122
}
117-
WriteObject(output);
123+
124+
DefaultContainer = DefaultStorageAccount.StorageContainerName;
125+
DefaultStorageAccountName = DefaultStorageAccount.StorageAccountName;
126+
DefaultStorageAccountKey = DefaultStorageAccount.StorageAccountKey;
118127
}
119-
else
128+
129+
storageAccess = new AzureStorageAccess(DefaultStorageAccountName, DefaultStorageAccountKey, DefaultContainer);
130+
131+
_clusterName = GetClusterConnection(ResourceGroupName, ClusterName);
132+
133+
string output;
134+
switch (DisplayOutputType)
120135
{
121-
DownloadJobTaskLogs();
136+
case JobDisplayOutputType.StandardError:
137+
output = GetJobError();
138+
break;
139+
default:
140+
output = GetJobOutput();
141+
break;
122142
}
143+
WriteObject(output);
123144
}
124145

125146
internal string GetJobOutput()
126147
{
127-
var output = HDInsightJobClient.GetJobOutput(JobId, DefaultStorageAccountName, DefaultStorageAccountKey, DefaultContainer);
148+
var output = HDInsightJobClient.GetJobOutput(JobId, this.storageAccess);
128149
var outputStr = Convert(output);
129150
return outputStr;
130151
}
131152

132153
private string GetJobError()
133154
{
134-
var output = HDInsightJobClient.GetJobError(JobId, DefaultStorageAccountName, DefaultStorageAccountKey,
135-
DefaultContainer);
155+
var output = HDInsightJobClient.GetJobError(JobId, this.storageAccess);
136156
var outputStr = Convert(output);
137157
return outputStr;
138158
}
139159

140-
private string GetJobTaskLogSummary()
141-
{
142-
var output = HDInsightJobClient.GetJobTaskLogSummary(JobId, DefaultStorageAccountName, DefaultStorageAccountKey,
143-
DefaultContainer);
144-
var outputStr = Convert(output);
145-
return outputStr;
146-
}
147-
148-
private void DownloadJobTaskLogs()
149-
{
150-
HDInsightJobClient.GetJobTaskLogSummary(JobId, DefaultStorageAccountName, DefaultStorageAccountKey,
151-
DefaultContainer);
152-
}
153-
154160
private static string Convert(Stream stream)
155161
{
156162
var reader = new StreamReader(stream);

‎src/ResourceManager/HDInsight/Commands.HDInsight/JobCommands/InvokeHiveCommand.cs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
using Hyak.Common;
2020
using Microsoft.Azure.Commands.HDInsight.Commands;
2121
using Microsoft.WindowsAzure.Commands.Common;
22+
using Microsoft.Azure.Management.HDInsight.Job.Models;
2223

2324
namespace Microsoft.Azure.Commands.HDInsight
2425
{
@@ -175,9 +176,10 @@ public override void ExecuteCmdlet()
175176
else
176177
{
177178
//get job error
178-
output = Convert(HDInsightJobClient.GetJobError(jobid, DefaultStorageAccountName, DefaultStorageAccountKey,
179-
DefaultContainer));
179+
IStorageAccess storageAccess = new AzureStorageAccess(DefaultStorageAccountName, DefaultStorageAccountKey, DefaultContainer);
180+
output = Convert(HDInsightJobClient.GetJobError(jobid, storageAccess));
180181
}
182+
181183
WriteObject(output);
182184
}
183185

‎src/ResourceManager/HDInsight/Commands.HDInsight/JobCommands/NewAzureHDInsightSqoopJobDefinitionCommand.cs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,13 @@ public string Command
5555
set { job.Command = value; }
5656
}
5757

58+
[Parameter(HelpMessage = "The output location to use for the job.")]
59+
public string LibDir
60+
{
61+
get { return job.LibDir; }
62+
set { job.LibDir = value; }
63+
}
64+
5865
#endregion
5966

6067
public NewAzureHDInsightSqoopJobDefinitionCommand()

‎src/ResourceManager/HDInsight/Commands.HDInsight/JobCommands/NewAzureHDInsightStreamingMapReduceJobDefinitionCommand.cs

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ public string StatusFolder
4949
}
5050

5151
[Parameter(HelpMessage = "The command line environment for the mappers or the reducers.")]
52-
public string[] CommandEnvironment { get; set; }
52+
public Hashtable CommandEnvironment { get; set; }
5353

5454
[Parameter(HelpMessage = "The parameters for the jobDetails.")]
5555
public Hashtable Defines { get; set; }
@@ -88,7 +88,7 @@ public string Reducer
8888
public NewAzureHDInsightStreamingMapReduceJobDefinitionCommand()
8989
{
9090
Arguments = new string[] {};
91-
CommandEnvironment = new string[] {};
91+
CommandEnvironment = new Hashtable();
9292
Defines = new Hashtable();
9393
job = new AzureHDInsightStreamingMapReduceJobDefinition();
9494
}
@@ -100,9 +100,10 @@ public override void ExecuteCmdlet()
100100
job.Arguments.Add(arg);
101101
}
102102

103-
foreach (var cmdenv in CommandEnvironment)
103+
var cmdEnvDic = CommandEnvironment.ToDictionary(false);
104+
foreach (var cmdEnv in cmdEnvDic)
104105
{
105-
job.CommandEnvironment.Add(cmdenv);
106+
job.CommandEnvironment.Add(cmdEnv.Key, cmdEnv.Value.ToString());
106107
}
107108

108109
var defineDic = Defines.ToDictionary(false);

‎src/ResourceManager/HDInsight/Commands.HDInsight/JobCommands/StartAzureHDInsightJobCommand.cs

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -95,11 +95,6 @@ public AzureHDInsightJob Execute()
9595

9696
public JobSubmissionResponse SubmitJob()
9797
{
98-
if (string.IsNullOrEmpty(JobDefinition.StatusFolder))
99-
{
100-
JobDefinition.StatusFolder = Guid.NewGuid().ToString();
101-
}
102-
10398
JobSubmissionResponse jobCreationResults;
10499

105100
var azureMapReduceJobDefinition = JobDefinition as AzureHDInsightMapReduceJobDefinition;

‎src/ResourceManager/HDInsight/Commands.HDInsight/Models/Job/AzureHDInsightSqoopJobDefinition.cs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,5 +35,10 @@ public class AzureHDInsightSqoopJobDefinition : AzureHDInsightJobDefinition
3535
/// Gets or sets the name of the jobDetails.
3636
/// </summary>
3737
public string JobName { get; set; }
38+
39+
/// <summary>
40+
/// Gets or sets the name of the directory for libraries.
41+
/// </summary>
42+
public string LibDir { get; set; }
3843
}
3944
}

‎src/ResourceManager/HDInsight/Commands.HDInsight/Models/Job/AzureHDInsightStreamingMapReduceJobDefinition.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,14 +54,14 @@ public class AzureHDInsightStreamingMapReduceJobDefinition : AzureHDInsightJobDe
5454
/// <summary>
5555
/// Gets the command line environment for the mappers or the reducers.
5656
/// </summary>
57-
public IList<string> CommandEnvironment { get; private set; }
57+
public IDictionary<string, string> CommandEnvironment { get; private set; }
5858

5959
/// <summary>
6060
/// Initializes a new instance of the AzureHDInsightStreamingMapReduceJobDefinition class.
6161
/// </summary>
6262
public AzureHDInsightStreamingMapReduceJobDefinition()
6363
{
64-
CommandEnvironment = new List<string>();
64+
CommandEnvironment = new Dictionary<string, string>();
6565
Defines = new Dictionary<string, string>();
6666
}
6767
}

0 commit comments

Comments
 (0)