HDInsight Bug Fixes

src/ResourceManager/HDInsight/Commands.HDInsight.Test/UnitTests/JobTests.cs

@@ -203,7 +203,7 @@ public void StartJob()
             {
                 CommandRuntime = commandRuntimeMock.Object,
                 HDInsightJobClient = hdinsightJobManagementMock.Object,
-                ClusterCredential = new PSCredential("httpuser", string.Format("Password1!").ConvertToSecureString()),
+                HttpCredential = new PSCredential("httpuser", string.Format("Password1!").ConvertToSecureString()),
                 ClusterName = ClusterName
             };
 
@@ -245,7 +245,7 @@ public void StartJob()
                 JobDetail = new JobDetailRootJsonObject
                 {
                     Completed = "false",
-                    User = cmdlet.ClusterCredential.UserName,
+                    User = cmdlet.HttpCredential.UserName,
                     Id = jobid
                 }
             };

src/ResourceManager/HDInsight/Commands.HDInsight/Commands.HDInsight.csproj

@@ -39,6 +39,7 @@
   </PropertyGroup>
   <ItemGroup>
     <Compile Include="JobCommands\GetAzureHDInsightJobCommand.cs" />
+    <Compile Include="JobCommands\NewAzureHDInsightSqoopJobDefinitionCommand.cs" />
     <Compile Include="JobCommands\WaitAzureHDInsightJobCommand.cs" />
     <Compile Include="JobCommands\NewAzureHDInsightStreamingMapReduceJobDefinitionCommand.cs" />
     <Compile Include="JobCommands\NewAzureHDInsightMapReduceJobDefinitionCommand.cs" />
@@ -65,12 +66,14 @@
     <Compile Include="HDInsightCmdletBase.cs" />
     <Compile Include="ManagementCommands\GetAzureHDInsightClusterCommand.cs" />
     <Compile Include="Constants.cs" />
+    <Compile Include="Models\Job\AzureHDInsightSqoopJobDefinition.cs" />
     <Compile Include="Models\Job\AzureHDInsightJob.cs" />
     <Compile Include="Models\Job\AzureHDInsightMapReduceJobDefinition.cs" />
     <Compile Include="Models\Job\AzureHDInsightStreamingMapReduceJobDefinition.cs" />
     <Compile Include="Models\Job\AzureHDInsightPigJobDefinition.cs" />
     <Compile Include="Models\Job\AzureHDInsightHiveJobDefinition.cs" />
     <Compile Include="Models\Job\AzureHDInsightJobDefinition.cs" />
+    <Compile Include="Models\Job\JobDisplayOutputType.cs" />
     <Compile Include="Models\Management\AzureHDInsightCluster.cs" />
     <Compile Include="Models\Management\AzureHDInsightConfig.cs" />
     <Compile Include="Models\Job\AzureHdInsightJobManagementClient.cs" />
@@ -80,6 +83,7 @@
     <Compile Include="Models\Management\AzureHDInsightMetastore.cs" />
     <Compile Include="ManagementCommands\NewAzureHDInsightClusterCommand.cs" />
     <Compile Include="ClusterConfigurationUtils.cs" />
+    <Compile Include="Models\Management\AzureHDInsightScriptAction.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
   </ItemGroup>
   <ItemGroup>
@@ -111,11 +115,11 @@
     </Reference>
     <Reference Include="Microsoft.Azure.Management.HDInsight, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.1.0.5-preview\lib\net40\Microsoft.Azure.Management.HDInsight.dll</HintPath>
+      <HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.1.0.6-preview\lib\net40\Microsoft.Azure.Management.HDInsight.dll</HintPath>
     </Reference>
     <Reference Include="Microsoft.Azure.Management.HDInsight.Job, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.Job.1.0.5-preview\lib\net40\Microsoft.Azure.Management.HDInsight.Job.dll</HintPath>
+      <HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.Job.1.0.6-preview\lib\net40\Microsoft.Azure.Management.HDInsight.Job.dll</HintPath>
     </Reference>
     <Reference Include="Microsoft.Azure.ResourceManager, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>

src/ResourceManager/HDInsight/Commands.HDInsight/Constants.cs

@@ -39,6 +39,7 @@ public static class JobDefinitions
             public const string AzureHDInsightPigJobDefinition = "AzureRmHDInsightPigJobDefinition";
             public const string AzureHDInsightMapReduceJobDefinition = "AzureRmHDInsightMapReduceJobDefinition";
             public const string AzureHDInsightStreamingMapReduceJobDefinition = "AzureRmHDInsightStreamingMapReduceJobDefinition";
+            public const string AzureHDInsightSqoopJobDefinition = "AzureRmHDInsightSqoopJobDefinition";
         }
 
         public static class ClusterConfiguration

src/ResourceManager/HDInsight/Commands.HDInsight/HDInsightCmdletBase.cs

@@ -83,5 +83,21 @@ protected string GetClusterConnection(string resourceGroupName, string clusterNa
             }
             return httpEndpoint;
         }
+
+        protected string GetResourceGroupByAccountName(string clusterName)
+        {
+            try
+            {
+                var clusterId = HDInsightManagementClient.ListClusters().First(x => x.Name.Equals(clusterName, StringComparison.InvariantCultureIgnoreCase)).Id;
+                var rgStart = clusterId.IndexOf("resourceGroups/", StringComparison.InvariantCultureIgnoreCase) + ("resourceGroups/".Length);
+                var rgLength = (clusterId.IndexOf("/providers/", StringComparison.InvariantCultureIgnoreCase)) - rgStart;
+                return clusterId.Substring(rgStart, rgLength);
+            }
+            catch
+            {
+                throw new CloudException(string.Format("Could not find resource group for cluster {0}.", clusterName));
+            }
+        }
+
     }
 }

src/ResourceManager/HDInsight/Commands.HDInsight/JobCommands/GetAzureHDInsightJobCommand.cs

@@ -26,14 +26,9 @@ namespace Microsoft.Azure.Commands.HDInsight
     OutputType(typeof(AzureHDInsightJob))]
     public class GetAzureHDInsightJobCommand : HDInsightCmdletBase
     {
-        [Parameter(
-            Position = 0,
-            Mandatory = true,
-            HelpMessage = "Gets or sets the name of the resource group.")]
-        public string ResourceGroupName { get; set; }
-
+        #region Input Parameter Definitions
         [Parameter(Mandatory = true,
-            Position = 1,
+            Position = 0,
             HelpMessage = "The name of the cluster.")]
         public string ClusterName
         {
@@ -42,9 +37,10 @@ public string ClusterName
         }
 
         [Parameter(Mandatory = true,
-            Position = 2,
+            Position = 1,
             HelpMessage = "The credentials with which to connect to the cluster.")]
-        public PSCredential ClusterCredential
+        [Alias("ClusterCredential")]
+        public PSCredential HttpCredential
         {
             get
             {
@@ -62,12 +58,22 @@ public PSCredential ClusterCredential
             }
         }
 
-        [Parameter(Position = 3,
+        [Parameter(Position = 2,
             HelpMessage = "The JobID of the jobDetails to stop.")]
         public string JobId { get; set; }
 
+        [Parameter(HelpMessage = "Gets or sets the name of the resource group.")]
+        public string ResourceGroupName { get; set; }
+
+        #endregion
+
+
         protected override void ProcessRecord()
         {
+            if (ResourceGroupName == null)
+            {
+                ResourceGroupName = GetResourceGroupByAccountName(ClusterName);
+            }
             _clusterName = GetClusterConnection(ResourceGroupName, ClusterName);
             if (JobId != null)
             {

src/ResourceManager/HDInsight/Commands.HDInsight/JobCommands/GetAzureHDInsightJobOutputCommand.cs

@@ -16,6 +16,7 @@
 using System.Management.Automation;
 using Hyak.Common;
 using Microsoft.Azure.Commands.HDInsight.Commands;
+using Microsoft.Azure.Commands.HDInsight.Models.Job;
 using Microsoft.WindowsAzure.Commands.Common;
 
 namespace Microsoft.Azure.Commands.HDInsight
@@ -25,45 +26,42 @@ namespace Microsoft.Azure.Commands.HDInsight
     OutputType(typeof(string))]
     public class GetAzureHDInsightJobOutputCommand : HDInsightCmdletBase
     {
-        [Parameter(
-            Position = 0,
-            Mandatory = true,
-            HelpMessage = "Gets or sets the name of the resource group.")]
-        public string ResourceGroupName { get; set; }
+        #region Input Parameter Definitions
 
         [Parameter(Mandatory = true,
-            Position = 1,
+            Position = 0,
             HelpMessage = "The name of the cluster.")]
         public string ClusterName
         {
             get { return _clusterName; }
             set { _clusterName = value; }
         }
 
-        [Parameter(Position = 2,
+        [Parameter(Position = 1,
             Mandatory = true,
             HelpMessage = "The JobID of the jobDetails to stop.")]
         public string JobId { get; set; }
 
-        [Parameter(Position = 3,
+        [Parameter(Position = 2,
             Mandatory = true,
             HelpMessage = "The default container name.")]
         public string DefaultContainer { get; set; }
 
-        [Parameter(Position = 4,
+        [Parameter(Position = 3,
             Mandatory = true, 
             HelpMessage = "The default storage account name.")]
         public string DefaultStorageAccountName { get; set; }
 
-        [Parameter(Position = 5,
+        [Parameter(Position = 4,
             Mandatory = true, 
             HelpMessage = "The default storage account key.")]
         public string DefaultStorageAccountKey { get; set; }
 
         [Parameter(Mandatory = true,
-            Position = 6,
+            Position = 5,
             HelpMessage = "The credentials with which to connect to the cluster.")]
-        public PSCredential ClusterCredential
+        [Alias("ClusterCredential")]
+        public PSCredential HttpCredential
         {
             get
             {
@@ -79,20 +77,80 @@ public PSCredential ClusterCredential
             }
         }
 
+        [Parameter(HelpMessage = "Gets or sets the name of the resource group.")]
+        public string ResourceGroupName { get; set; }
+
+        [Parameter(HelpMessage = "The type of job output.", ParameterSetName = "Display")]
+        public JobDisplayOutputType DisplayOutputType { get; set; }
+
+        [Parameter(Mandatory = true, HelpMessage = "The type of output to download.", ParameterSetName = "Download")]
+        public JobDownloadOutputType DownloadOutputType { get; set; }
+
+        [Parameter(Mandatory = true, HelpMessage = "The folder to save the output to.", ParameterSetName = "Download")]
+        public string Folder { get; set; }
+
+        #endregion
+
         protected override void ProcessRecord()
         {
+            if (ResourceGroupName == null)
+            {
+                ResourceGroupName = GetResourceGroupByAccountName(ClusterName);
+            }
             _clusterName = GetClusterConnection(ResourceGroupName, ClusterName);
-            var output = GetJobOutput();
-            WriteObject(output);
-        }
 
-        public string GetJobOutput()
+            if (ParameterSetName == "Display")
+            {
+                string output;
+                switch (DisplayOutputType)
+                {
+                    case JobDisplayOutputType.StandardError:
+                        output = GetJobError();
+                        break;
+                    case JobDisplayOutputType.TaskSummary:
+                        output = GetJobTaskLogSummary();
+                        break;
+                    default:
+                        output = GetJobOutput();
+                        break;
+                }
+                WriteObject(output);
+            }
+            else
+            {
+                DownloadJobTaskLogs();
+            }
+        }
+
+        internal string GetJobOutput()
         {
             var output = HDInsightJobClient.GetJobOutput(JobId, DefaultStorageAccountName, DefaultStorageAccountKey, DefaultContainer);
             var outputStr = Convert(output);
             return outputStr;
         }
 
+        private string GetJobError()
+        {
+            var output = HDInsightJobClient.GetJobError(JobId, DefaultStorageAccountName, DefaultStorageAccountKey,
+                DefaultContainer);
+            var outputStr = Convert(output);
+            return outputStr;
+        }
+
+        private string GetJobTaskLogSummary()
+        {
+            var output = HDInsightJobClient.GetJobTaskLogSummary(JobId, DefaultStorageAccountName, DefaultStorageAccountKey,
+                DefaultContainer);
+            var outputStr = Convert(output);
+            return outputStr;
+        }
+
+        private void DownloadJobTaskLogs()
+        {
+            HDInsightJobClient.GetJobTaskLogSummary(JobId, DefaultStorageAccountName, DefaultStorageAccountKey,
+                DefaultContainer);
+        }
+
         private static string Convert(Stream stream)
         {
             var reader = new StreamReader(stream);

src/ResourceManager/HDInsight/Commands.HDInsight/JobCommands/InvokeHiveCommand.cs

@@ -109,7 +109,7 @@ protected override void ProcessRecord()
             //get variables from session
             var clusterConnection = SessionState.PSVariable.Get(UseAzureHDInsightClusterCommand.ClusterEndpoint).Value.ToString();
             var clusterCred =
-                (PSCredential) SessionState.PSVariable.Get(UseAzureHDInsightClusterCommand.ClusterCred).Value;
+                (PSCredential)SessionState.PSVariable.Get(UseAzureHDInsightClusterCommand.ClusterCred).Value;
             var resourceGroup =
                 SessionState.PSVariable.Get(UseAzureHDInsightClusterCommand.CurrentResourceGroup).Value.ToString();
 
@@ -137,7 +137,7 @@ protected override void ProcessRecord()
                 ClusterName = clusterConnection,
                 ResourceGroupName = resourceGroup,
                 JobDefinition = hivejob,
-                ClusterCredential = clusterCred
+                HttpCredential = clusterCred
             };
 
             var jobCreationResult = startJobCommand.SubmitJob();
@@ -148,7 +148,7 @@ protected override void ProcessRecord()
             WriteProgress(new ProgressRecord(0, "Waiting for job to complete", "In Progress"));
             var waitJobCommand = new WaitAzureHDInsightJobCommand
             {
-                ClusterCredential = clusterCred,
+                HttpCredential = clusterCred,
                 ResourceGroupName = resourceGroup,
                 ClusterName = clusterConnection,
                 JobId = jobid
@@ -161,7 +161,7 @@ protected override void ProcessRecord()
                 //get job output
                 var getOutputCommand = new GetAzureHDInsightJobOutputCommand
                 {
-                    ClusterCredential = clusterCred,
+                    HttpCredential = clusterCred,
                     ResourceGroupName = resourceGroup,
                     ClusterName = clusterConnection,
                     DefaultContainer = DefaultContainer,