Merge branch 'dev' of https://github.com/Azure/azure-powershell into dev

Author: hyonholee

README.md

@@ -3,8 +3,8 @@
 
 This repository contains a set of PowerShell cmdlets for developers and administrators to develop, deploy and manage Microsoft Azure applications.
 
-* For documentation on how to build and deploy applications to Microsoft Azure please see the [Microsoft Azure Documentation Center](http://azure.microsoft.com/en-us/documentation/).
-* For comprehensive documentation on the developer cmdlets see [How to install and configure Azure PowerShell](http://azure.microsoft.com/en-us/documentation/articles/install-configure-powershell/).
+* For documentation on how to build and deploy applications to Microsoft Azure please see the [Microsoft Azure Documentation Center](https://azure.microsoft.com/en-us/documentation/).
+* For comprehensive documentation on the developer cmdlets see [How to install and configure Azure PowerShell](https://azure.microsoft.com/en-us/documentation/articles/install-configure-powershell/).
 * For comprehensive documentation on the full set of Microsoft Azure cmdlets see [Microsoft Azure Management Center](http://go.microsoft.com/fwlink/?linkID=254459&clcid=0x409).
 
 ## Features
@@ -55,17 +55,17 @@ For detail descriptions and examples of the cmdlets, type
 
 ## Supported Environments
 
-* [Microsoft Azure](http://www.azure.microsoft.com)
+* [Microsoft Azure](https://azure.microsoft.com)
 * [Azure Stack](https://azure.microsoft.com/en-us/overview/azure-stack/)
-* [Windows Azure Pack](http://www.microsoft.com/en-us/server-cloud/windows-azure-pack.aspx)
-* [Microsoft Azure China](http://www.windowsazure.cn/)
+* [Windows Azure Pack](https://www.microsoft.com/en-us/server-cloud/windows-azure-pack.aspx)
+* [Microsoft Azure China](https://www.azure.cn/)
 * [USGovernment](https://azure.microsoft.com/en-us/features/gov/)
 
 ## Installation
 
 ### Microsoft Web Platform Installer
 
-1. Install [Microsoft Web Platform Installer](http://www.microsoft.com/web/downloads/platform.aspx).
+1. Install [Microsoft Web Platform Installer](https://www.microsoft.com/web/downloads/platform.aspx).
 2. Open Microsoft Web Platform Installer and search for __Microsoft Azure PowerShell__.
 3. Install.
 
@@ -84,15 +84,15 @@ You can also find the standalone installers for all the versions at [Downloads](
 
 ### Supported PowerShell Versions
 
-* [Windows Management Framework 3] (http://www.microsoft.com/en-us/download/details.aspx?id=34595)
+* [Windows Management Framework 3] (https://www.microsoft.com/en-us/download/details.aspx?id=34595)
 * [Windows Management Framework 4] (https://www.microsoft.com/en-us/download/details.aspx?id=40855)
 * [Windows Management Framework 5] (https://www.microsoft.com/en-us/download/details.aspx?id=50395)
 
 ## Get Started
 
 In general, following are the steps to start using Microsoft Azure PowerShell
 
-* Get yourself authenticated with Microsoft Azure. For details, please check out [this article](http://azure.microsoft.com/en-us/documentation/articles/install-configure-powershell/).
+* Get yourself authenticated with Microsoft Azure. For details, please check out [this article](https://azure.microsoft.com/en-us/documentation/articles/install-configure-powershell/).
   * Option 1: Login with your Microsoft account or Organizational account directly from PowerShell. Microsoft Azure Active Directory authentication is used in this case. No management certificate is needed.
       * Starting from 1.0.0, you can use ```Add-AzureRmAccount -Credential``` to avoid the browser pop up for Organizational account.
       * To use RDFE cmdlets, use ```Add-AzureAccount```
@@ -120,7 +120,7 @@ New-AzureRmResourceGroup -Name myresourceGroup -Location "West US"
 Add-AzureRmAccount -EnvironmentName AzureChinaCloud
 
 # use the cmdlets to manage your services/applications
-New-AzureRmResourceGroup -Name myresourceGroup -Location "Chine East"
+New-AzureRmResourceGroup -Name myresourceGroup -Location "China East"
 ```
 
 ### Windows Azure Pack
@@ -181,10 +181,10 @@ Be sure to check out the [Microsoft Azure Developer Forums on Stack Overflow](ht
 
 ## Contribute Code or Provide Feedback
 
-If you would like to become an active contributor to this project please follow the instructions provided in [Microsoft Azure Projects Contribution Guidelines](http://windowsazure.github.com/guidelines.html).
+If you would like to become an active contributor to this project please follow the instructions provided in [Microsoft Azure Projects Contribution Guidelines](https://azure.github.io/guidelines/).
 
 If you encounter any bugs with the library please file an issue in the [Issues](https://github.com/Azure/azure-powershell/issues) section of the project.
 
 # Learn More
 
-* [Microsoft Azure Script Center](http://www.azure.microsoft.com/en-us/documentation/scripts/)
+* [Microsoft Azure Script Center](https://azure.microsoft.com/en-us/documentation/scripts/)

src/ResourceManager/AzureBackup/Commands.AzureBackup.Test/ScenarioTests/AzureBackupTestBase.cs

@@ -24,6 +24,7 @@
 using System.Net.Security;
 using System.Reflection;
 using Microsoft.WindowsAzure.Commands.Test.Utilities.Common;
+using System.Collections.Generic;
 
 namespace Microsoft.Azure.Commands.AzureBackup.Test.ScenarioTests
 {
@@ -58,6 +59,14 @@ protected void SetupManagementClients()
 
         protected void RunPowerShellTest(params string[] scripts)
         {
+            Dictionary<string, string> d = new Dictionary<string, string>();
+            d.Add("Microsoft.Resources", null);
+            d.Add("Microsoft.Features", null);
+            d.Add("Microsoft.Authorization", null);
+            d.Add("Microsoft.Compute", null);
+            var providersToIgnore = new Dictionary<string, string>();
+            providersToIgnore.Add("Microsoft.Azure.Management.Resources.ResourceManagementClient", "2016-02-01");
+            HttpMockServer.Matcher = new PermissiveRecordMatcherWithApiExclusion(true, d, providersToIgnore);
             using (UndoContext context = UndoContext.Current)
             {
                 context.Start(TestUtilities.GetCallingClass(2), TestUtilities.GetCurrentMethodName(2));

src/ResourceManager/HDInsight/Commands.HDInsight.Test/Commands.HDInsight.Test.csproj

@@ -56,7 +56,7 @@
     </Reference>
     <Reference Include="Microsoft.Azure.Management.HDInsight.Job">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.Job.1.1.0-preview\lib\net40\Microsoft.Azure.Management.HDInsight.Job.dll</HintPath>
+      <HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.Job.2.0.0-preview\lib\net40\Microsoft.Azure.Management.HDInsight.Job.dll</HintPath>
     </Reference>
     <Reference Include="Microsoft.Azure.Management.Storage">
       <HintPath>..\..\..\packages\Microsoft.Azure.Management.Storage.2.4.0-preview\lib\net40\Microsoft.Azure.Management.Storage.dll</HintPath>

src/ResourceManager/HDInsight/Commands.HDInsight.Test/HDInsightTestBase.cs

@@ -12,11 +12,12 @@
 // limitations under the License.
 // ----------------------------------------------------------------------------------
 
+using System;
+using System.Collections.Generic;
 using System.Management.Automation;
 using Hyak.Common;
 using Microsoft.Azure.Commands.HDInsight.Models;
 using Microsoft.Azure.Management.HDInsight.Models;
-using Microsoft.WindowsAzure.Commands.Common;
 using Microsoft.WindowsAzure.Commands.Test.Utilities.Common;
 using Moq;
 
@@ -41,9 +42,51 @@ public virtual void SetupTestsForManagement()
 
         public virtual void SetupTestsForData()
         {
+            hdinsightManagementMock = new Mock<AzureHdInsightManagementClient>();
             var cred = new BasicAuthenticationCloudCredentials {Username = "username", Password = "Password1!"};
             hdinsightJobManagementMock = new Mock<AzureHdInsightJobManagementClient>(ClusterName, cred);
             commandRuntimeMock = new Mock<ICommandRuntime>();
         }
+
+        public virtual void SetupManagementClientForJobTests()
+        {
+            // Update HDInsight Management properties for Job.
+            var cluster1 = new Cluster
+            {
+                Id = "/subscriptions/" + Guid.NewGuid() + "/resourceGroups/" + ResourceGroupName + "/providers/Microsoft.HDInsight/clusters/" + ClusterName,
+                Name = ClusterName,
+                Location = Location,
+                Properties = new ClusterGetProperties
+                {
+                    ClusterVersion = "3.2",
+                    ClusterState = "Running",
+                    ClusterDefinition = new ClusterDefinition
+                    {
+                        ClusterType = ClusterType
+                    },
+                    QuotaInfo = new QuotaInfo
+                    {
+                        CoresUsed = 24
+                    },
+                    OperatingSystemType = OSType.Windows,
+                    ConnectivityEndpoints = new List<ConnectivityEndpoint> { new ConnectivityEndpoint { Location = ClusterName, Name = "HTTPS" } }
+                }
+            };
+
+            var listresponse = new ClusterListResponse { Clusters = new[] { cluster1 } };
+            hdinsightManagementMock.Setup(c => c.ListClusters())
+                .Returns(listresponse)
+                .Verifiable();
+
+            hdinsightManagementMock.Setup(c => c.GetCluster(It.IsAny<string>(), It.IsAny<string>()))
+                .Returns(new List<Cluster> { cluster1 })
+                .Verifiable();
+
+            var configurationResponse = new Dictionary<string, string>();
+
+            hdinsightManagementMock.Setup(c => c.GetClusterConfigurations(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>()))
+                .Returns(configurationResponse)
+                .Verifiable();
+        }
     }
 }

src/ResourceManager/HDInsight/Commands.HDInsight.Test/UnitTests/JobTests.cs

@@ -15,12 +15,14 @@
 using System.Collections.Generic;
 using System.Management.Automation;
 using System.Net;
+using System.Linq;
 using Microsoft.Azure.Commands.HDInsight.Models;
 using Microsoft.Azure.Management.HDInsight.Job.Models;
 using Microsoft.WindowsAzure.Commands.Common;
 using Microsoft.WindowsAzure.Commands.ScenarioTest;
 using Moq;
 using Xunit;
+using Microsoft.Azure.Management.HDInsight.Models;
 
 namespace Microsoft.Azure.Commands.HDInsight.Test
 {
@@ -195,14 +197,93 @@ public void CreateStreamingJob()
                                 job.Reducer == reducer && job.Defines.Count == defines.Count)));
         }
 
-        [Fact(Skip = "Test requires setting env variable, TODO remove that constraint")]
+        [Fact]
+        [Trait(Category.AcceptanceType, Category.CheckIn)]
+        public void GetJobWithIdProvided()
+        {
+            // Update HDInsight Management properties for Job.
+            SetupManagementClientForJobTests();
+
+            var jobId = "jobid_1984120_001";
+            var cmdlet = GetJobCommandDefinition();
+            cmdlet.JobId = jobId;
+
+            // Setup Job Management mocks
+            var jobResponse = new JobGetResponse
+            {
+                JobDetail = new JobDetailRootJsonObject { Id = jobId, Status = new Status(), Userargs = new Userargs() }
+            };
+
+            hdinsightJobManagementMock.Setup(c => c.GetJob(It.IsAny<string>()))
+                .Returns(jobResponse)
+                .Verifiable();
+
+            cmdlet.ExecuteCmdlet();
+            commandRuntimeMock.VerifyAll();
+            commandRuntimeMock.Verify(
+                f =>
+                    f.WriteObject(It.Is<AzureHDInsightJob>(job => job.JobId.Equals(jobId))));
+        }
+
+        [Fact]
+        [Trait(Category.AcceptanceType, Category.CheckIn)]
+        public void ListJobs()
+        {
+            // Update HDInsight Management properties for Job.
+            SetupManagementClientForJobTests();
+
+            var cmdlet = GetJobCommandDefinition();
+
+            // Setup Job Management mocks
+            var jobListResponse = GetJobListResponse();
+
+            hdinsightJobManagementMock.Setup(c => c.ListJobs())
+                .Returns(jobListResponse)
+                .Verifiable();
+
+            cmdlet.ExecuteCmdlet();
+            commandRuntimeMock.VerifyAll();
+            commandRuntimeMock.Verify(
+                f =>
+                    f.WriteObject(It.Is<IEnumerable<string>>(job => job.ElementAt(0).Equals(jobListResponse.ElementAt(0).Detail.Id) && job.ElementAt(1).Equals(jobListResponse.ElementAt(1).Detail.Id)), true));
+        }
+
+        [Fact]
+        [Trait(Category.AcceptanceType, Category.CheckIn)]
+        public void ListJobsAfterJobId()
+        {
+            // Update HDInsight Management properties for Job.
+            SetupManagementClientForJobTests();
+
+            var cmdlet = GetJobCommandDefinition();
+            cmdlet.NumOfJobs = 2;
+
+            // Setup Job Management mocks
+            var jobListResponse = GetJobListResponse();
+
+            hdinsightJobManagementMock.Setup(c => c.ListJobsAfterJobId(It.IsAny<string>(), It.IsAny<int>()))
+                .Returns(jobListResponse)
+                .Verifiable();
+
+            cmdlet.ExecuteCmdlet();
+            commandRuntimeMock.VerifyAll();
+            commandRuntimeMock.Verify(
+                f =>
+                    f.WriteObject(It.Is<IEnumerable<AzureHDInsightJob>>(job => job.ElementAt(0).JobId.Equals(jobListResponse.ElementAt(0).Detail.Id) && job.ElementAt(1).JobId.Equals(jobListResponse.ElementAt(1).Detail.Id)), true));
+        }
+
+        [Fact]
         [Trait(Category.AcceptanceType, Category.CheckIn)]
         public void StartJob()
         {
+            // Update HDInsight Management properties for Job.
+            SetupManagementClientForJobTests();
+
             var cmdlet = new StartAzureHDInsightJobCommand
             {
                 CommandRuntime = commandRuntimeMock.Object,
                 HDInsightJobClient = hdinsightJobManagementMock.Object,
+                HDInsightManagementClient = hdinsightManagementMock.Object,
                 HttpCredential = new PSCredential("httpuser", string.Format("Password1!").ConvertToSecureString()),
                 ClusterName = ClusterName
             };
@@ -246,7 +327,9 @@ public void StartJob()
                 {
                     Completed = "false",
                     User = cmdlet.HttpCredential.UserName,
-                    Id = jobid
+                    Id = jobid,
+                    Status = new Status(),
+                    Userargs = new Userargs()
                 }
             };
             hdinsightJobManagementMock.Setup(c => c.GetJob(jobsub.JobSubmissionJsonResponse.Id)).Returns(getresponse).Verifiable();
@@ -259,5 +342,42 @@ public void StartJob()
                         It.Is<AzureHDInsightJob>(
                             job => job.Cluster == ClusterName && job.JobId == jobid && job.Completed == "false")));
         }
+
+        public JobListResponse GetJobListResponse()
+        {
+            var jobListobject1 = new JobListJsonObject
+            {
+                Detail = new JobDetailRootJsonObject { Id = "jobid_1984120_001", Status = new Status(), Userargs = new Userargs() },
+                Id = "jobid_1984120_001"
+            };
+
+            var jobListobject2 = new JobListJsonObject
+            {
+                Detail = new JobDetailRootJsonObject { Id = "jobid_1984120_002", Status = new Status(), Userargs = new Userargs() },
+                Id = "jobid_1984120_002"
+            };
+
+            var jobListResponse = new JobListResponse
+            {
+                JobList = new List<JobListJsonObject> { jobListobject1, jobListobject2 },
+                StatusCode = HttpStatusCode.OK
+            };
+
+            return jobListResponse;
+        }
+
+        public GetAzureHDInsightJobCommand GetJobCommandDefinition()
+        {
+            var cmdlet = new GetAzureHDInsightJobCommand
+            {
+                CommandRuntime = commandRuntimeMock.Object,
+                HDInsightJobClient = hdinsightJobManagementMock.Object,
+                HDInsightManagementClient = hdinsightManagementMock.Object,
+                HttpCredential = new PSCredential("httpuser", string.Format("Password1!").ConvertToSecureString()),
+                ClusterName = ClusterName
+            };
+
+            return cmdlet;
+        }
     }
 }

src/ResourceManager/HDInsight/Commands.HDInsight.Test/packages.config

@@ -7,7 +7,7 @@
   <package id="Microsoft.Azure.KeyVault.Core" version="1.0.0" targetFramework="net45" />
   <package id="Microsoft.Azure.Management.Authorization" version="0.18.2-preview" targetFramework="net45" />
   <package id="Microsoft.Azure.Management.HDInsight" version="1.0.10-preview" targetFramework="net45" />
-  <package id="Microsoft.Azure.Management.HDInsight.Job" version="1.1.0-preview" targetFramework="net45" />
+  <package id="Microsoft.Azure.Management.HDInsight.Job" version="2.0.0-preview" targetFramework="net45" />
   <package id="Microsoft.Azure.Management.Resources" version="2.19.0-preview" targetFramework="net45" />
   <package id="Microsoft.Azure.Management.Storage" version="2.4.0-preview" targetFramework="net45" />
   <package id="Microsoft.Azure.Test.Framework" version="1.0.5896.19355-prerelease" targetFramework="net45" />

src/ResourceManager/HDInsight/Commands.HDInsight/Commands.HDInsight.csproj

@@ -127,7 +127,7 @@
     </Reference>
     <Reference Include="Microsoft.Azure.Management.HDInsight.Job">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.Job.1.1.0-preview\lib\net40\Microsoft.Azure.Management.HDInsight.Job.dll</HintPath>
+      <HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.Job.2.0.0-preview\lib\net40\Microsoft.Azure.Management.HDInsight.Job.dll</HintPath>
     </Reference>
     <Reference Include="Microsoft.Azure.ResourceManager, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>

src/ResourceManager/HDInsight/Commands.HDInsight/HDInsightCmdletBase.cs

@@ -18,6 +18,7 @@
 using Microsoft.Azure.Commands.HDInsight.Models;
 using Microsoft.Azure.Commands.ResourceManager.Common;
 using Microsoft.WindowsAzure.Commands.Utilities.Common;
+using Microsoft.Azure.Management.HDInsight;
 
 namespace Microsoft.Azure.Commands.HDInsight.Commands
 {
@@ -99,5 +100,28 @@ protected string GetResourceGroupByAccountName(string clusterName)
             }
         }
 
+        protected AzureHDInsightDefaultStorageAccount GetDefaultStorageAccount(string resourceGroupName, string clusterName)
+        {
+            var result = HDInsightManagementClient.GetCluster(resourceGroupName, clusterName);
+
+            if (result == null || result.Count == 0)
+            {
+                throw new CloudException(string.Format("Couldn't find cluster {0}", clusterName));
+            }
+
+            var cluster = result.FirstOrDefault();
+            var configuration = HDInsightManagementClient.GetClusterConfigurations(resourceGroupName, cluster.Name, ConfigurationKey.CoreSite);
+
+            var DefaultStorageAccount = ClusterConfigurationUtils.GetDefaultStorageAccountDetails(
+                                    configuration,
+                                    cluster.Properties.ClusterVersion);
+
+            if (DefaultStorageAccount == null)
+            {
+                throw new CloudException(string.Format("Couldn't find storage information for cluster {0}", clusterName));
+            }
+
+            return DefaultStorageAccount;
+        }
     }
 }