ADL PowerShell Updates

AzurePowershell.Test.targets

@@ -91,10 +91,10 @@
       <XUnitTests Include=".\src\ResourceManager\CognitiveServices\CognitiveServices.Test\bin\Debug\Microsoft.Azure.Commands.Management.CognitiveServices.Test.dll"/>
       <XUnitTests Include=".\src\ResourceManager\Automation\Commands.Automation.Test\bin\Debug\Microsoft.Azure.Commands.ResourceManager.Automation.Test.dll"/>
       <XUnitTests Include=".\src\ResourceManager\PowerBIEmbedded\Commands.PowerBIEmbedded.Test\bin\Debug\Microsoft.Azure.Commands.PowerBIEmbedded.Test.dll"/>
-      <!-- <XUnitTests Include=".\src\ResourceManager\AzureStackAdmin\Commands.AzureStackAdmin.Test\bin\Debug\Microsoft.AzureStack.Commands.Admin.Test.dll"/>
-      <XUnitTests Include=".\src\ResourceManager\AzureStackStorage\Commands.AzureStackStorage.Tests\bin\Debug\Microsoft.AzureStack.Commands.StorageAdmin.Test.dll"/>
       <XUnitTests Include=".\src\ResourceManager\DataLakeAnalytics\Commands.DataLakeAnalytics.Test\bin\Debug\Microsoft.Azure.Commands.DataLakeAnalytics.Test.dll"/>
       <XUnitTests Include=".\src\ResourceManager\DataLakeStore\Commands.DataLakeStore.Test\bin\Debug\Microsoft.Azure.Commands.DataLake.Test.dll"/>
+      <!-- <XUnitTests Include=".\src\ResourceManager\AzureStackAdmin\Commands.AzureStackAdmin.Test\bin\Debug\Microsoft.AzureStack.Commands.Admin.Test.dll"/>
+      <XUnitTests Include=".\src\ResourceManager\AzureStackStorage\Commands.AzureStackStorage.Tests\bin\Debug\Microsoft.AzureStack.Commands.StorageAdmin.Test.dll"/>
       <XUnitTests Include=".\src\ResourceManager\DevTestLabs\Commands.DevTestLabs.Test\bin\Debug\Microsoft.Azure.Commands.DevTestLabs.Test.dll"/> 
       <XUnitTests Include=".\src\ResourceManager\HDInsight\Commands.HDInsight.Test\bin\Debug\Commands.HDInsight.Test.dll"/>
       <XUnitTests Include=".\src\ResourceManager\Insights\Commands.Insights.Test\bin\Debug\Microsoft.Azure.Commands.Insights.Test.dll"/> 

ChangeLog.md

@@ -7,6 +7,20 @@
 	* Set-AzureRmSqlDatabaseBackupLongTermRetentionPolicy
   * Restore-AzureRmSqlDatabase now supports point-in-time restore of a deleted database
   * Restore-AzureRmSqlDatabase now supports restoring from a Long Term Retention backup
+* Azure Data Lake Store
+	* Drastically improve performance of file and folder upload and download.
+		* This includes a slight change to the parameter names for download and inclusion of two new parameters for upload:
+			* NumThreads -> PerFileThreadCount, used to indicate the number of threads to use in a single file
+			* ConcurrentFileCount, used to indicate the number of files to upload/download in parallel for folder upload/download.
+		* Default threading values are now designed to give a better all around throughput for most file sizes. If performance is not as desired, the values above can be modified to meet requirements.
+* Azure Data Lake Analytics
+	* Get-AzureRMDataLakeAnalyticsDataSource now returns all data sources when called with no arguments.
+		* This change also removes the data source type parameter from the cmdlet.
+		* This change results in a new object being returned for the list operation with the following properties:
+			* Type, the type of data source
+			* Name, the name of the data source
+			* IsDefault, set to true if this is the default data source for the account
+	* Get-AzureRMDataLakeAnalyticsJob fixed for list for certain date time offset values when filtering on submittedBefore and submittedAfter.
 ##2016.06.23 version 1.5.1
 * Azure Resource Manager
   - Fix a bug in New-AzureRmResourceGroupDeployment. In some deployments the cmdlet throws an exception with "Deployment 'deploymentName' could not be found." and causes the cmdlet to fail. The fix makes sure the deployment is created before getting operations.

src/ResourceManager/Common/Commands.ScenarioTests.ResourceManager.Common/PermissiveRecordMatcherWithApiExclusion.cs

@@ -113,7 +113,8 @@ private bool ContainsIgnoredProvider(string requestUri, out string version)
                 !requestUri.StartsWith("/jobs", StringComparison.InvariantCultureIgnoreCase) &&
                 !requestUri.StartsWith("/jobschedules", StringComparison.InvariantCultureIgnoreCase) &&
                 !requestUri.Contains("/applications?") &&
-                !requestUri.Contains("/servicePrincipals?"))
+                !requestUri.Contains("/servicePrincipals?") &&
+                !requestUri.StartsWith("/webhdfs/v1/?aclspec", StringComparison.InvariantCultureIgnoreCase))
             {
                 version = String.Empty;
                 return true;

src/ResourceManager/DataLakeAnalytics/Commands.DataLakeAnalytics.Test/Commands.DataLakeAnalytics.Test.csproj

@@ -64,12 +64,12 @@
     </Reference>
     <Reference Include="Microsoft.Azure.Management.DataLake.Analytics, Version=0.11.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\..\..\packages\Microsoft.Azure.Management.DataLake.Analytics.0.11.3-preview\lib\net45\Microsoft.Azure.Management.DataLake.Analytics.dll</HintPath>
+      <HintPath>..\..\..\packages\Microsoft.Azure.Management.DataLake.Analytics.0.11.5-preview\lib\net45\Microsoft.Azure.Management.DataLake.Analytics.dll</HintPath>
       <Private>True</Private>
     </Reference>
     <Reference Include="Microsoft.Azure.Management.DataLake.Store, Version=0.12.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\..\..\packages\Microsoft.Azure.Management.DataLake.Store.0.12.0-preview\lib\net45\Microsoft.Azure.Management.DataLake.Store.dll</HintPath>
+      <HintPath>..\..\..\packages\Microsoft.Azure.Management.DataLake.Store.0.12.2-preview\lib\net45\Microsoft.Azure.Management.DataLake.Store.dll</HintPath>
       <Private>True</Private>
     </Reference>
     <Reference Include="Microsoft.Azure.Management.ResourceManager, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
@@ -209,6 +209,9 @@
     <None Include="packages.config">
       <SubType>Designer</SubType>
     </None>
+    <None Include="..\..\Common\Commands.ScenarioTests.ResourceManager.Common\AzureRM.Resources.ps1">
+      <Link>ScenarioTests\AzureRM.Resources.ps1</Link>
+    </None>
     <None Include="ScenarioTests\AdlaAliasTests.ps1">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
@@ -219,19 +222,19 @@
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
     <None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaAliasTests\TestAdlaAccount.json">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
     <None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaAliasTests\TestAdlaCatalog.json">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
     <None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaAliasTests\TestAdlaJob.json">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
     <None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaAliasTests\TestNegativeAdlaAccount.json">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
     <None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaAliasTests\TestNegativeAdlaJob.json">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
     <None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaTests\TestAdlaAccount.json">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>

src/ResourceManager/DataLakeAnalytics/Commands.DataLakeAnalytics.Test/ScenarioTests/AdlaAliasTests.ps1

@@ -186,7 +186,7 @@ function Test-DataLakeAnalyticsJob
 		New-AzureRmResourceGroup -Name $resourceGroupName -Location $location
 		New-AdlStore -ResourceGroupName $resourceGroupName -Name $dataLakeAccountName -Location $location
 		$accountCreated = New-AdlAnalyticsAccount -ResourceGroupName $resourceGroupName -Name $accountName -Location $location -DefaultDataLakeStore $dataLakeAccountName
-    
+		$nowTime = $accountCreated.Properties.CreationTime
 		Assert-AreEqual $accountName $accountCreated.Name
 		Assert-AreEqual $location $accountCreated.Location
 		Assert-AreEqual "Microsoft.DataLakeAnalytics/accounts" $accountCreated.Type
@@ -214,6 +214,9 @@ function Test-DataLakeAnalyticsJob
 		# Wait for two minutes and 30 seconds prior to attempting to submit the job in the freshly created account.
 		[Microsoft.Rest.ClientRuntime.Azure.TestFramework.TestUtilities]::Wait(150000)
 		# submit a job
+		$guidForJob = [Microsoft.Rest.ClientRuntime.Azure.TestFramework.TestUtilities]::GenerateGuid("jobTest02")
+		[Microsoft.Azure.Commands.DataLakeAnalytics.Models.DataLakeAnalyticsClient]::JobIdQueue.Enqueue($guidForJob)
+
 		$jobInfo = Submit-AdlJob -AccountName $accountName -Name "TestJob" -Script "DROP DATABASE IF EXISTS foo; CREATE DATABASE foo;"
 		Assert-NotNull {$jobInfo}
 
@@ -229,11 +232,12 @@ function Test-DataLakeAnalyticsJob
 
 		Assert-NotNull {Get-AdlJob -AccountName $accountName}
 
-		$jobsWithDateOffset = Get-AdlJob -AccountName $accountName -SubmittedAfter $(([DateTime]::Now).AddMinutes(-5))
+		$jobsWithDateOffset = Get-AdlJob -AccountName $accountName -SubmittedAfter $([DateTimeOffset]($nowTime).AddMinutes(-5))
 
 		Assert-True {$jobsWithDateOffset.Count -gt 0} "Failed to retrieve jobs submitted after five miuntes ago"
-
-		$jobsWithDateOffset = Get-AdlJob -AccountName $accountName -SubmittedBefore $(([DateTime]::Now).AddMinutes(0))
+
+		# we add five minutes to ensure that the timing is right, since we are using the account creation time, and not truly "now"
+		$jobsWithDateOffset = Get-AdlJob -AccountName $accountName -SubmittedBefore $([DateTimeOffset]($nowTime).AddMinutes(5))
 
 		Assert-True {$jobsWithDateOffset.Count -gt 0} "Failed to retrieve jobs submitted before right now"
 
@@ -274,7 +278,7 @@ function Test-NegativeDataLakeAnalyticsAccount
 		New-AzureRmResourceGroup -Name $resourceGroupName -Location $location
 		New-AdlStore -ResourceGroupName $resourceGroupName -Name $dataLakeAccountName -Location $location
 		$accountCreated = New-AdlAnalyticsAccount -ResourceGroupName $resourceGroupName -Name $accountName -Location $location -DefaultDataLakeStore $dataLakeAccountName
-    
+		
 		Assert-AreEqual $accountName $accountCreated.Name
 		Assert-AreEqual $location $accountCreated.Location
 		Assert-AreEqual "Microsoft.DataLakeAnalytics/accounts" $accountCreated.Type
@@ -347,7 +351,7 @@ function Test-NegativeDataLakeAnalyticsJob
 		New-AzureRmResourceGroup -Name $resourceGroupName -Location $location
 		New-AdlStore -ResourceGroupName $resourceGroupName -Name $dataLakeAccountName -Location $location
 		$accountCreated = New-AdlAnalyticsAccount -ResourceGroupName $resourceGroupName -Name $accountName -Location $location -DefaultDataLakeStore $dataLakeAccountName
-    
+		$nowTime = $accountCreated.Properties.CreationTime
 		Assert-AreEqual $accountName $accountCreated.Name
 		Assert-AreEqual $location $accountCreated.Location
 		Assert-AreEqual "Microsoft.DataLakeAnalytics/accounts" $accountCreated.Type
@@ -380,7 +384,7 @@ function Test-NegativeDataLakeAnalyticsJob
 		# Attempt to Get debug data for a non-existent job
 		Assert-Throws {Get-AdlJobDebugInfo -AccountName $accountName -JobIdentity [Guid]::Empty}
 
-		$jobsWithDateOffset = Get-AdlJob -AccountName $accountName -SubmittedAfter $([DateTime]::Now)
+		$jobsWithDateOffset = Get-AdlJob -AccountName $accountName -SubmittedAfter $([DateTimeOffset]$nowTime)
 
 		Assert-True {$jobsWithDateOffset.Count -eq 0} "Retrieval of jobs submitted after right now returned results and should not have"
 
@@ -549,6 +553,8 @@ function Test-DataLakeAnalyticsCatalog
 "@
 		# run the script
 		$scriptToRun = [string]::Format($scriptTemplate, $databaseName, $tableName, $tvfName, $viewName, $procName)
+		$guidForJob = [Microsoft.Rest.ClientRuntime.Azure.TestFramework.TestUtilities]::GenerateGuid("catalogCreationJob01")
+		[Microsoft.Azure.Commands.DataLakeAnalytics.Models.DataLakeAnalyticsClient]::JobIdQueue.Enqueue($guidForJob)
 		$jobInfo = Submit-AdlJob -AccountName $accountName -Name "TestJob" -Script $scriptToRun
 		$result = Wait-AdlJob -AccountName $accountName -JobId $jobInfo.JobId
 		Assert-AreEqual "Succeeded" $result.Result
@@ -706,6 +712,8 @@ function Test-DataLakeAnalyticsCatalog
 "@
 		$credentialJob = [string]::Format($credentialJobTemplate, $databaseName, $credentialName, $secretName)
 
+		$guidForJob = [Microsoft.Rest.ClientRuntime.Azure.TestFramework.TestUtilities]::GenerateGuid("credentialCreateJob01")
+		[Microsoft.Azure.Commands.DataLakeAnalytics.Models.DataLakeAnalyticsClient]::JobIdQueue.Enqueue($guidForJob)
 		$jobInfo = Submit-AdlJob -AccountName $accountName -Name "TestJobCredential" -Script $credentialJob
 		$result = Wait-AdlJob -AccountName $accountName -JobId $jobInfo.JobId
 		Assert-AreEqual "Succeeded" $result.Result
@@ -737,7 +745,8 @@ function Test-DataLakeAnalyticsCatalog
 	DROP CREDENTIAL {1};
 "@
 		$credentialJob = [string]::Format($credentialJobTemplate, $databaseName, $credentialName)
-
+		$guidForJob = [Microsoft.Rest.ClientRuntime.Azure.TestFramework.TestUtilities]::GenerateGuid("credentialDropJob01")
+		[Microsoft.Azure.Commands.DataLakeAnalytics.Models.DataLakeAnalyticsClient]::JobIdQueue.Enqueue($guidForJob)
 		$jobInfo = Submit-AdlJob -AccountName $accountName -Name "TestJobCredential" -Script $credentialJob
 		$result = Wait-AdlJob -AccountName $accountName -JobId $jobInfo.JobId
 		Assert-AreEqual "Succeeded" $result.Result

src/ResourceManager/DataLakeAnalytics/Commands.DataLakeAnalytics.Test/ScenarioTests/AdlaTests.ps1

@@ -186,7 +186,8 @@ function Test-DataLakeAnalyticsJob
 		New-AzureRmResourceGroup -Name $resourceGroupName -Location $location
 		New-AzureRmDataLakeStoreAccount -ResourceGroupName $resourceGroupName -Name $dataLakeAccountName -Location $location
 		$accountCreated = New-AzureRmDataLakeAnalyticsAccount -ResourceGroupName $resourceGroupName -Name $accountName -Location $location -DefaultDataLakeStore $dataLakeAccountName
-
+		$nowTime = $accountCreated.Properties.CreationTime
+
 		Assert-AreEqual $accountName $accountCreated.Name
 		Assert-AreEqual $location $accountCreated.Location
 		Assert-AreEqual "Microsoft.DataLakeAnalytics/accounts" $accountCreated.Type
@@ -214,6 +215,8 @@ function Test-DataLakeAnalyticsJob
 		# Wait for two minutes and 30 seconds prior to attempting to submit the job in the freshly created account.
 		[Microsoft.Rest.ClientRuntime.Azure.TestFramework.TestUtilities]::Wait(150000)
 		# submit a job
+		$guidForJob = [Microsoft.Rest.ClientRuntime.Azure.TestFramework.TestUtilities]::GenerateGuid("jobTest01")
+		[Microsoft.Azure.Commands.DataLakeAnalytics.Models.DataLakeAnalyticsClient]::JobIdQueue.Enqueue($guidForJob)
 		$jobInfo = Submit-AzureRmDataLakeAnalyticsJob -AccountName $accountName -Name "TestJob" -Script "DROP DATABASE IF EXISTS foo; CREATE DATABASE foo;"
 		Assert-NotNull {$jobInfo}
 
@@ -229,11 +232,11 @@ function Test-DataLakeAnalyticsJob
 
 		Assert-NotNull {Get-AzureRmDataLakeAnalyticsJob -AccountName $accountName}
 
-		$jobsWithDateOffset = Get-AzureRmDataLakeAnalyticsJob -AccountName $accountName -SubmittedAfter $(([DateTime]::Now).AddMinutes(-5))
+		$jobsWithDateOffset = Get-AzureRmDataLakeAnalyticsJob -AccountName $accountName -SubmittedAfter $([DateTimeOffset]($nowTime).AddMinutes(-5))
 
 		Assert-True {$jobsWithDateOffset.Count -gt 0} "Failed to retrieve jobs submitted after five miuntes ago"
-
-		$jobsWithDateOffset = Get-AzureRmDataLakeAnalyticsJob -AccountName $accountName -SubmittedBefore $(([DateTime]::Now).AddMinutes(0))
+		# we add five minutes to ensure that the timing is right, since we are using the account creation time, and not truly "now"
+		$jobsWithDateOffset = Get-AzureRmDataLakeAnalyticsJob -AccountName $accountName -SubmittedBefore $([DateTimeOffset]($nowTime).AddMinutes(5)) 
 
 		Assert-True {$jobsWithDateOffset.Count -gt 0} "Failed to retrieve jobs submitted before right now"
 
@@ -347,7 +350,7 @@ function Test-NegativeDataLakeAnalyticsJob
 		New-AzureRmResourceGroup -Name $resourceGroupName -Location $location
 		New-AzureRmDataLakeStoreAccount -ResourceGroupName $resourceGroupName -Name $dataLakeAccountName -Location $location
 		$accountCreated = New-AzureRmDataLakeAnalyticsAccount -ResourceGroupName $resourceGroupName -Name $accountName -Location $location -DefaultDataLakeStore $dataLakeAccountName
-
+        $nowTime = $accountCreated.Properties.CreationTime
 		Assert-AreEqual $accountName $accountCreated.Name
 		Assert-AreEqual $location $accountCreated.Location
 		Assert-AreEqual "Microsoft.DataLakeAnalytics/accounts" $accountCreated.Type
@@ -380,7 +383,7 @@ function Test-NegativeDataLakeAnalyticsJob
 		# Attempt to Get debug data for a non-existent job
 		Assert-Throws {Get-AzureRmDataLakeAnalyticsJobDebugInfo -AccountName $accountName -JobIdentity [Guid]::Empty}
 
-		$jobsWithDateOffset = Get-AzureRmDataLakeAnalyticsJob -AccountName $accountName -SubmittedAfter $([DateTime]::Now)
+		$jobsWithDateOffset = Get-AzureRmDataLakeAnalyticsJob -AccountName $accountName -SubmittedAfter $([DateTimeOffset]$nowTime)
 
 		Assert-True {$jobsWithDateOffset.Count -eq 0} "Retrieval of jobs submitted after right now returned results and should not have"
 
@@ -549,6 +552,10 @@ function Test-DataLakeAnalyticsCatalog
 "@
 		# run the script
 		$scriptToRun = [string]::Format($scriptTemplate, $databaseName, $tableName, $tvfName, $viewName, $procName)
+
+		$guidForJob = [Microsoft.Rest.ClientRuntime.Azure.TestFramework.TestUtilities]::GenerateGuid("createCatalogJob01")
+		[Microsoft.Azure.Commands.DataLakeAnalytics.Models.DataLakeAnalyticsClient]::JobIdQueue.Enqueue($guidForJob)
+
 		$jobInfo = Submit-AzureRmDataLakeAnalyticsJob -AccountName $accountName -Name "TestJob" -Script $scriptToRun
 		$result = Wait-AzureRMDataLakeAnalyticsJob -AccountName $accountName -JobId $jobInfo.JobId
 		Assert-AreEqual "Succeeded" $result.Result
@@ -705,7 +712,9 @@ function Test-DataLakeAnalyticsCatalog
 	CREATE CREDENTIAL {1} WITH USER_NAME = "scope@rkm4grspxa", IDENTITY = "{2}";
 "@
 		$credentialJob = [string]::Format($credentialJobTemplate, $databaseName, $credentialName, $secretName)
-
+
+		$guidForJob = [Microsoft.Rest.ClientRuntime.Azure.TestFramework.TestUtilities]::GenerateGuid("createCredentialjob02")
+		[Microsoft.Azure.Commands.DataLakeAnalytics.Models.DataLakeAnalyticsClient]::JobIdQueue.Enqueue($guidForJob)
 		$jobInfo = Submit-AzureRmDataLakeAnalyticsJob -AccountName $accountName -Name "TestJobCredential" -Script $credentialJob
 		$result = Wait-AzureRMDataLakeAnalyticsJob -AccountName $accountName -JobId $jobInfo.JobId
 		Assert-AreEqual "Succeeded" $result.Result
@@ -737,7 +746,8 @@ function Test-DataLakeAnalyticsCatalog
 	DROP CREDENTIAL {1};
 "@
 		$credentialJob = [string]::Format($credentialJobTemplate, $databaseName, $credentialName)
-
+		$guidForJob = [Microsoft.Rest.ClientRuntime.Azure.TestFramework.TestUtilities]::GenerateGuid("dropCredentialJob02")
+		[Microsoft.Azure.Commands.DataLakeAnalytics.Models.DataLakeAnalyticsClient]::JobIdQueue.Enqueue($guidForJob)
 		$jobInfo = Submit-AzureRmDataLakeAnalyticsJob -AccountName $accountName -Name "TestJobCredential" -Script $credentialJob
 		$result = Wait-AzureRMDataLakeAnalyticsJob -AccountName $accountName -JobId $jobInfo.JobId
 		Assert-AreEqual "Succeeded" $result.Result