Skip to content

Commit 2fc3a68

Browse files
markcowlmarkcowl
committed
Merge pull request Azure#1964 from pattipaka/dev
Moving to HDInsight Job SDK 2.0.0 preview and couple of bug fixes.
2 parents 13a278f + 774217b commit 2fc3a68

21 files changed

+392
-282
lines changed

‎src/ResourceManager/HDInsight/Commands.HDInsight.Test/Commands.HDInsight.Test.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@
5656
</Reference>
5757
<Reference Include="Microsoft.Azure.Management.HDInsight.Job">
5858
<SpecificVersion>False</SpecificVersion>
59-
<HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.Job.1.1.0-preview\lib\net40\Microsoft.Azure.Management.HDInsight.Job.dll</HintPath>
59+
<HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.Job.2.0.0-preview\lib\net40\Microsoft.Azure.Management.HDInsight.Job.dll</HintPath>
6060
</Reference>
6161
<Reference Include="Microsoft.Azure.Management.Storage">
6262
<HintPath>..\..\..\packages\Microsoft.Azure.Management.Storage.2.4.0-preview\lib\net40\Microsoft.Azure.Management.Storage.dll</HintPath>

‎src/ResourceManager/HDInsight/Commands.HDInsight.Test/HDInsightTestBase.cs

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,12 @@
1212
// limitations under the License.
1313
// ----------------------------------------------------------------------------------
1414

15+
using System;
16+
using System.Collections.Generic;
1517
using System.Management.Automation;
1618
using Hyak.Common;
1719
using Microsoft.Azure.Commands.HDInsight.Models;
1820
using Microsoft.Azure.Management.HDInsight.Models;
19-
using Microsoft.WindowsAzure.Commands.Common;
2021
using Microsoft.WindowsAzure.Commands.Test.Utilities.Common;
2122
using Moq;
2223

@@ -41,9 +42,51 @@ public virtual void SetupTestsForManagement()
4142

4243
public virtual void SetupTestsForData()
4344
{
45+
hdinsightManagementMock = new Mock<AzureHdInsightManagementClient>();
4446
var cred = new BasicAuthenticationCloudCredentials {Username = "username", Password = "Password1!"};
4547
hdinsightJobManagementMock = new Mock<AzureHdInsightJobManagementClient>(ClusterName, cred);
4648
commandRuntimeMock = new Mock<ICommandRuntime>();
4749
}
50+
51+
public virtual void SetupManagementClientForJobTests()
52+
{
53+
// Update HDInsight Management properties for Job.
54+
var cluster1 = new Cluster
55+
{
56+
Id = "/subscriptions/" + Guid.NewGuid() + "/resourceGroups/" + ResourceGroupName + "/providers/Microsoft.HDInsight/clusters/" + ClusterName,
57+
Name = ClusterName,
58+
Location = Location,
59+
Properties = new ClusterGetProperties
60+
{
61+
ClusterVersion = "3.2",
62+
ClusterState = "Running",
63+
ClusterDefinition = new ClusterDefinition
64+
{
65+
ClusterType = ClusterType
66+
},
67+
QuotaInfo = new QuotaInfo
68+
{
69+
CoresUsed = 24
70+
},
71+
OperatingSystemType = OSType.Windows,
72+
ConnectivityEndpoints = new List<ConnectivityEndpoint> { new ConnectivityEndpoint { Location = ClusterName, Name = "HTTPS" } }
73+
}
74+
};
75+
76+
var listresponse = new ClusterListResponse { Clusters = new[] { cluster1 } };
77+
hdinsightManagementMock.Setup(c => c.ListClusters())
78+
.Returns(listresponse)
79+
.Verifiable();
80+
81+
hdinsightManagementMock.Setup(c => c.GetCluster(It.IsAny<string>(), It.IsAny<string>()))
82+
.Returns(new List<Cluster> { cluster1 })
83+
.Verifiable();
84+
85+
var configurationResponse = new Dictionary<string, string>();
86+
87+
hdinsightManagementMock.Setup(c => c.GetClusterConfigurations(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>()))
88+
.Returns(configurationResponse)
89+
.Verifiable();
90+
}
4891
}
4992
}

‎src/ResourceManager/HDInsight/Commands.HDInsight.Test/UnitTests/JobTests.cs

Lines changed: 122 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,14 @@
1515
using System.Collections.Generic;
1616
using System.Management.Automation;
1717
using System.Net;
18+
using System.Linq;
1819
using Microsoft.Azure.Commands.HDInsight.Models;
1920
using Microsoft.Azure.Management.HDInsight.Job.Models;
2021
using Microsoft.WindowsAzure.Commands.Common;
2122
using Microsoft.WindowsAzure.Commands.ScenarioTest;
2223
using Moq;
2324
using Xunit;
25+
using Microsoft.Azure.Management.HDInsight.Models;
2426

2527
namespace Microsoft.Azure.Commands.HDInsight.Test
2628
{
@@ -195,14 +197,93 @@ public void CreateStreamingJob()
195197
job.Reducer == reducer && job.Defines.Count == defines.Count)));
196198
}
197199

198-
[Fact(Skip = "Test requires setting env variable, TODO remove that constraint")]
200+
[Fact]
201+
[Trait(Category.AcceptanceType, Category.CheckIn)]
202+
public void GetJobWithIdProvided()
203+
{
204+
// Update HDInsight Management properties for Job.
205+
SetupManagementClientForJobTests();
206+
207+
var jobId = "jobid_1984120_001";
208+
var cmdlet = GetJobCommandDefinition();
209+
cmdlet.JobId = jobId;
210+
211+
// Setup Job Management mocks
212+
var jobResponse = new JobGetResponse
213+
{
214+
JobDetail = new JobDetailRootJsonObject { Id = jobId, Status = new Status(), Userargs = new Userargs() }
215+
};
216+
217+
hdinsightJobManagementMock.Setup(c => c.GetJob(It.IsAny<string>()))
218+
.Returns(jobResponse)
219+
.Verifiable();
220+
221+
cmdlet.ExecuteCmdlet();
222+
commandRuntimeMock.VerifyAll();
223+
commandRuntimeMock.Verify(
224+
f =>
225+
f.WriteObject(It.Is<AzureHDInsightJob>(job => job.JobId.Equals(jobId))));
226+
}
227+
228+
[Fact]
229+
[Trait(Category.AcceptanceType, Category.CheckIn)]
230+
public void ListJobs()
231+
{
232+
// Update HDInsight Management properties for Job.
233+
SetupManagementClientForJobTests();
234+
235+
var cmdlet = GetJobCommandDefinition();
236+
237+
// Setup Job Management mocks
238+
var jobListResponse = GetJobListResponse();
239+
240+
hdinsightJobManagementMock.Setup(c => c.ListJobs())
241+
.Returns(jobListResponse)
242+
.Verifiable();
243+
244+
cmdlet.ExecuteCmdlet();
245+
commandRuntimeMock.VerifyAll();
246+
commandRuntimeMock.Verify(
247+
f =>
248+
f.WriteObject(It.Is<IEnumerable<string>>(job => job.ElementAt(0).Equals(jobListResponse.ElementAt(0).Detail.Id) && job.ElementAt(1).Equals(jobListResponse.ElementAt(1).Detail.Id)), true));
249+
}
250+
251+
[Fact]
252+
[Trait(Category.AcceptanceType, Category.CheckIn)]
253+
public void ListJobsAfterJobId()
254+
{
255+
// Update HDInsight Management properties for Job.
256+
SetupManagementClientForJobTests();
257+
258+
var cmdlet = GetJobCommandDefinition();
259+
cmdlet.NumOfJobs = 2;
260+
261+
// Setup Job Management mocks
262+
var jobListResponse = GetJobListResponse();
263+
264+
hdinsightJobManagementMock.Setup(c => c.ListJobsAfterJobId(It.IsAny<string>(), It.IsAny<int>()))
265+
.Returns(jobListResponse)
266+
.Verifiable();
267+
268+
cmdlet.ExecuteCmdlet();
269+
commandRuntimeMock.VerifyAll();
270+
commandRuntimeMock.Verify(
271+
f =>
272+
f.WriteObject(It.Is<IEnumerable<AzureHDInsightJob>>(job => job.ElementAt(0).JobId.Equals(jobListResponse.ElementAt(0).Detail.Id) && job.ElementAt(1).JobId.Equals(jobListResponse.ElementAt(1).Detail.Id)), true));
273+
}
274+
275+
[Fact]
199276
[Trait(Category.AcceptanceType, Category.CheckIn)]
200277
public void StartJob()
201278
{
279+
// Update HDInsight Management properties for Job.
280+
SetupManagementClientForJobTests();
281+
202282
var cmdlet = new StartAzureHDInsightJobCommand
203283
{
204284
CommandRuntime = commandRuntimeMock.Object,
205285
HDInsightJobClient = hdinsightJobManagementMock.Object,
286+
HDInsightManagementClient = hdinsightManagementMock.Object,
206287
HttpCredential = new PSCredential("httpuser", string.Format("Password1!").ConvertToSecureString()),
207288
ClusterName = ClusterName
208289
};
@@ -246,7 +327,9 @@ public void StartJob()
246327
{
247328
Completed = "false",
248329
User = cmdlet.HttpCredential.UserName,
249-
Id = jobid
330+
Id = jobid,
331+
Status = new Status(),
332+
Userargs = new Userargs()
250333
}
251334
};
252335
hdinsightJobManagementMock.Setup(c => c.GetJob(jobsub.JobSubmissionJsonResponse.Id)).Returns(getresponse).Verifiable();
@@ -259,5 +342,42 @@ public void StartJob()
259342
It.Is<AzureHDInsightJob>(
260343
job => job.Cluster == ClusterName && job.JobId == jobid && job.Completed == "false")));
261344
}
345+
346+
public JobListResponse GetJobListResponse()
347+
{
348+
var jobListobject1 = new JobListJsonObject
349+
{
350+
Detail = new JobDetailRootJsonObject { Id = "jobid_1984120_001", Status = new Status(), Userargs = new Userargs() },
351+
Id = "jobid_1984120_001"
352+
};
353+
354+
var jobListobject2 = new JobListJsonObject
355+
{
356+
Detail = new JobDetailRootJsonObject { Id = "jobid_1984120_002", Status = new Status(), Userargs = new Userargs() },
357+
Id = "jobid_1984120_002"
358+
};
359+
360+
var jobListResponse = new JobListResponse
361+
{
362+
JobList = new List<JobListJsonObject> { jobListobject1, jobListobject2 },
363+
StatusCode = HttpStatusCode.OK
364+
};
365+
366+
return jobListResponse;
367+
}
368+
369+
public GetAzureHDInsightJobCommand GetJobCommandDefinition()
370+
{
371+
var cmdlet = new GetAzureHDInsightJobCommand
372+
{
373+
CommandRuntime = commandRuntimeMock.Object,
374+
HDInsightJobClient = hdinsightJobManagementMock.Object,
375+
HDInsightManagementClient = hdinsightManagementMock.Object,
376+
HttpCredential = new PSCredential("httpuser", string.Format("Password1!").ConvertToSecureString()),
377+
ClusterName = ClusterName
378+
};
379+
380+
return cmdlet;
381+
}
262382
}
263383
}

‎src/ResourceManager/HDInsight/Commands.HDInsight.Test/packages.config

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
<package id="Microsoft.Azure.KeyVault.Core" version="1.0.0" targetFramework="net45" />
88
<package id="Microsoft.Azure.Management.Authorization" version="0.18.2-preview" targetFramework="net45" />
99
<package id="Microsoft.Azure.Management.HDInsight" version="1.0.10-preview" targetFramework="net45" />
10-
<package id="Microsoft.Azure.Management.HDInsight.Job" version="1.1.0-preview" targetFramework="net45" />
10+
<package id="Microsoft.Azure.Management.HDInsight.Job" version="2.0.0-preview" targetFramework="net45" />
1111
<package id="Microsoft.Azure.Management.Resources" version="2.19.0-preview" targetFramework="net45" />
1212
<package id="Microsoft.Azure.Management.Storage" version="2.4.0-preview" targetFramework="net45" />
1313
<package id="Microsoft.Azure.Test.Framework" version="1.0.5896.19355-prerelease" targetFramework="net45" />

‎src/ResourceManager/HDInsight/Commands.HDInsight/Commands.HDInsight.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@
127127
</Reference>
128128
<Reference Include="Microsoft.Azure.Management.HDInsight.Job">
129129
<SpecificVersion>False</SpecificVersion>
130-
<HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.Job.1.1.0-preview\lib\net40\Microsoft.Azure.Management.HDInsight.Job.dll</HintPath>
130+
<HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.Job.2.0.0-preview\lib\net40\Microsoft.Azure.Management.HDInsight.Job.dll</HintPath>
131131
</Reference>
132132
<Reference Include="Microsoft.Azure.ResourceManager, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
133133
<SpecificVersion>False</SpecificVersion>

‎src/ResourceManager/HDInsight/Commands.HDInsight/HDInsightCmdletBase.cs

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
using Microsoft.Azure.Commands.HDInsight.Models;
1919
using Microsoft.Azure.Commands.ResourceManager.Common;
2020
using Microsoft.WindowsAzure.Commands.Utilities.Common;
21+
using Microsoft.Azure.Management.HDInsight;
2122

2223
namespace Microsoft.Azure.Commands.HDInsight.Commands
2324
{
@@ -99,5 +100,28 @@ protected string GetResourceGroupByAccountName(string clusterName)
99100
}
100101
}
101102

103+
protected AzureHDInsightDefaultStorageAccount GetDefaultStorageAccount(string resourceGroupName, string clusterName)
104+
{
105+
var result = HDInsightManagementClient.GetCluster(resourceGroupName, clusterName);
106+
107+
if (result == null || result.Count == 0)
108+
{
109+
throw new CloudException(string.Format("Couldn't find cluster {0}", clusterName));
110+
}
111+
112+
var cluster = result.FirstOrDefault();
113+
var configuration = HDInsightManagementClient.GetClusterConfigurations(resourceGroupName, cluster.Name, ConfigurationKey.CoreSite);
114+
115+
var DefaultStorageAccount = ClusterConfigurationUtils.GetDefaultStorageAccountDetails(
116+
configuration,
117+
cluster.Properties.ClusterVersion);
118+
119+
if (DefaultStorageAccount == null)
120+
{
121+
throw new CloudException(string.Format("Couldn't find storage information for cluster {0}", clusterName));
122+
}
123+
124+
return DefaultStorageAccount;
125+
}
102126
}
103127
}

‎src/ResourceManager/HDInsight/Commands.HDInsight/JobCommands/GetAzureHDInsightJobCommand.cs

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,20 +62,29 @@ public PSCredential HttpCredential
6262
HelpMessage = "The JobID of the jobDetails to stop.")]
6363
public string JobId { get; set; }
6464

65+
[Parameter(HelpMessage = "Gets or sets the number of jobs to retrieve.")]
66+
public int NumOfJobs { get; set; }
67+
6568
[Parameter(HelpMessage = "Gets or sets the name of the resource group.")]
6669
public string ResourceGroupName { get; set; }
6770

6871
#endregion
6972

70-
7173
public override void ExecuteCmdlet()
7274
{
7375
if (ResourceGroupName == null)
7476
{
7577
ResourceGroupName = GetResourceGroupByAccountName(ClusterName);
7678
}
79+
7780
_clusterName = GetClusterConnection(ResourceGroupName, ClusterName);
78-
if (JobId != null)
81+
82+
if (NumOfJobs > 0)
83+
{
84+
var jobs = HDInsightJobClient.ListJobsAfterJobId(JobId, NumOfJobs).Select(job => new AzureHDInsightJob(job.Detail, HDInsightJobClient.ClusterName));
85+
WriteObject(jobs, true);
86+
}
87+
else if (JobId != null)
7988
{
8089
var job = HDInsightJobClient.GetJob(JobId);
8190
var jobDetails = new AzureHDInsightJob(job.JobDetail, HDInsightJobClient.ClusterName);

0 commit comments

Comments
 (0)