Merge pull request #3853 from begoldsm/preview

ADLA Catalog Updates and ADLS crash fix

Author: markcowl

src/ResourceManager/DataLakeAnalytics/ChangeLog.md

@@ -18,6 +18,12 @@
         - Additional information about change #1
 -->
 ## Current Release
+* Add support for catalog package get and list
+* Add support for listing the following catalog items from deeper ancestors:
+  * Table
+  * TVF
+  * View
+  * Statistics
 
 ## Version 2.8.0
 * Fix help for some commands to have the proper verbage and examples.

src/ResourceManager/DataLakeAnalytics/Commands.DataLakeAnalytics.Test/Commands.DataLakeAnalytics.Test.csproj

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
-<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <Import Project="..\..\..\packages\xunit.runner.visualstudio.2.1.0\build\net20\xunit.runner.visualstudio.props" Condition="Exists('..\..\..\packages\xunit.runner.visualstudio.2.1.0\build\net20\xunit.runner.visualstudio.props')" />
   <Import Project="..\..\..\packages\xunit.core.2.1.0\build\portable-net45+win8+wp8+wpa81\xunit.core.props" Condition="Exists('..\..\..\packages\xunit.core.2.1.0\build\portable-net45+win8+wp8+wpa81\xunit.core.props')" />
   <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
@@ -11,7 +11,7 @@
     <AppDesignerFolder>Properties</AppDesignerFolder>
     <RootNamespace>Microsoft.Azure.Commands.DataLakeAnalytics.Test</RootNamespace>
     <AssemblyName>Microsoft.Azure.Commands.DataLakeAnalytics.Test</AssemblyName>
-    <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
+    <TargetFrameworkVersion>v4.5.2</TargetFrameworkVersion>
     <FileAlignment>512</FileAlignment>
     <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\..\..\</SolutionDir>
     <RestorePackages>true</RestorePackages>
@@ -64,12 +64,11 @@
     </Reference>
     <Reference Include="Microsoft.Azure.Management.DataLake.Analytics, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\..\..\packages\Microsoft.Azure.Management.DataLake.Analytics.2.1.1-preview\lib\net45\Microsoft.Azure.Management.DataLake.Analytics.dll</HintPath>
+      <HintPath>..\..\..\packages\Microsoft.Azure.Management.DataLake.Analytics.2.2.0-preview\lib\net452\Microsoft.Azure.Management.DataLake.Analytics.dll</HintPath>
       <Private>True</Private>
     </Reference>
-    <Reference Include="Microsoft.Azure.Management.DataLake.Store, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
-      <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\..\..\packages\Microsoft.Azure.Management.DataLake.Store.2.1.1-preview\lib\net45\Microsoft.Azure.Management.DataLake.Store.dll</HintPath>
+    <Reference Include="Microsoft.Azure.Management.DataLake.Store">
+      <HintPath>..\..\..\packages\Microsoft.Azure.Management.DataLake.Store.2.1.2-preview\lib\net452\Microsoft.Azure.Management.DataLake.Store.dll</HintPath>
       <Private>True</Private>
     </Reference>
     <Reference Include="Microsoft.Azure.Management.ResourceManager, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
@@ -104,7 +103,7 @@
     </Reference>
     <Reference Include="Microsoft.Rest.ClientRuntime, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\..\..\packages\Microsoft.Rest.ClientRuntime.2.3.5\lib\net45\Microsoft.Rest.ClientRuntime.dll</HintPath>
+      <HintPath>..\..\..\packages\Microsoft.Rest.ClientRuntime.2.3.6\lib\net45\Microsoft.Rest.ClientRuntime.dll</HintPath>
       <Private>True</Private>
     </Reference>
     <Reference Include="Microsoft.Rest.ClientRuntime.Azure, Version=3.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">

src/ResourceManager/DataLakeAnalytics/Commands.DataLakeAnalytics.Test/ScenarioTests/AdlaAliasTests.ps1

@@ -358,9 +358,6 @@ function Test-DataLakeAnalyticsJob
 			Assert-False {$i -eq 60} "dataLakeAnalytics accounts not in succeeded state even after 30 min."
 		}
 
-		# For now, all Job related tests just ensure that they have a valid response and do not throw.
-		# Wait for two minutes and 30 seconds prior to attempting to submit the job in the freshly created account.
-		[Microsoft.Rest.ClientRuntime.Azure.TestFramework.TestUtilities]::Wait(150000)
 		# submit a job
 		$guidForJob = [Microsoft.Rest.ClientRuntime.Azure.TestFramework.TestUtilities]::GenerateGuid("jobTest02")
 		[Microsoft.Azure.Commands.DataLakeAnalytics.Models.DataLakeAnalyticsClient]::JobIdQueue.Enqueue($guidForJob)
@@ -603,10 +600,6 @@ function Test-DataLakeAnalyticsCatalog
 			Assert-False {$i -eq 60} "dataLakeAnalytics accounts not in succeeded state even after 30 min."
 		}
 
-		# For now, all Job related tests just ensure that they have a valid response and do not throw.
-		# Wait for two minutes prior to attempting to submit the job in the freshly created account.
-		[Microsoft.Rest.ClientRuntime.Azure.TestFramework.TestUtilities]::Wait(120000)
-	
 		# Run a job to create the catalog items (except secret and credential)
 		$scriptTemplate = @"
 	DROP DATABASE IF EXISTS {0}; CREATE DATABASE {0};
@@ -747,7 +740,24 @@ function Test-DataLakeAnalyticsCatalog
 		}
 
 		Assert-True {$found} "Could not find the table $tableName in the table list"
-	
+		# retrieve the list in the database (no schema)
+		$itemList = Get-AdlCatalogItem -AccountName $accountName -ItemType Table -Path "$databaseName"
+
+		Assert-NotNull $itemList "The table list is null"
+
+		Assert-True {$itemList.count -gt 0} "The table list is empty"
+		$found = $false
+		foreach($item in $itemList)
+		{
+			if($item.Name -eq $tableName)
+			{
+				$found = $true
+				break
+			}
+		}
+
+		Assert-True {$found} "Could not find the table $tableName in the table list"
+
 		# retrieve the specific table
 		$specificItem = Get-AdlCatalogItem -AccountName $accountName -ItemType Table -Path "$databaseName.dbo.$tableName"
 		Assert-NotNull $specificItem "Could not retrieve the table by name"
@@ -785,6 +795,24 @@ function Test-DataLakeAnalyticsCatalog
 
 		Assert-True {$found} "Could not find the TVF $tvfName in the TVF list"
 
+		# get the items from just the database (no schema)
+		$itemList = Get-AdlCatalogItem -AccountName $accountName -ItemType TableValuedFunction -Path "$databaseName"
+
+		Assert-NotNull $itemList "The TVF list is null"
+
+		Assert-True {$itemList.count -gt 0} "The TVF list is empty"
+		$found = $false
+		foreach($item in $itemList)
+		{
+			if($item.Name -eq $tvfName)
+			{
+				$found = $true
+				break
+			}
+		}
+
+		Assert-True {$found} "Could not find the TVF $tvfName in the TVF list"
+
 		# retrieve the specific TVF
 		$specificItem = Get-AdlCatalogItem -AccountName $accountName -ItemType TableValuedFunction -Path "$databaseName.dbo.$tvfName"
 		Assert-NotNull $specificItem "Could not retrieve the TVF by name"
@@ -831,6 +859,25 @@ function Test-DataLakeAnalyticsCatalog
 
 		Assert-True {$found} "Could not find the view $viewName in the view list"
 
+		# get views in database only (no schema)
+		$itemList = Get-AdlCatalogItem -AccountName $accountName -ItemType View -Path "$databaseName"
+
+		Assert-NotNull $itemList "The view list is null"
+
+		Assert-True {$itemList.count -gt 0} "The view list is empty"
+		$found = $false
+		foreach($item in $itemList)
+		{
+			if($item.Name -eq $viewName)
+			{
+				$found = $true
+				break
+			}
+		}
+	
+		Assert-True {$found} "Could not find the view $viewName in the view list"
+
+
 		# retrieve the specific view
 		$specificItem = Get-AdlCatalogItem -AccountName $accountName -ItemType View -Path "$databaseName.dbo.$viewName"
 		Assert-NotNull $specificItem "Could not retrieve the view by name"

src/ResourceManager/DataLakeAnalytics/Commands.DataLakeAnalytics.Test/ScenarioTests/AdlaTests.ps1

@@ -358,9 +358,6 @@ function Test-DataLakeAnalyticsJob
 			Assert-False {$i -eq 60} "dataLakeAnalytics accounts not in succeeded state even after 30 min."
 		}
 
-		# For now, all Job related tests just ensure that they have a valid response and do not throw.
-		# Wait for two minutes and 30 seconds prior to attempting to submit the job in the freshly created account.
-		[Microsoft.Rest.ClientRuntime.Azure.TestFramework.TestUtilities]::Wait(150000)
 		# submit a job
 		$guidForJob = [Microsoft.Rest.ClientRuntime.Azure.TestFramework.TestUtilities]::GenerateGuid("jobTest01")
 		[Microsoft.Azure.Commands.DataLakeAnalytics.Models.DataLakeAnalyticsClient]::JobIdQueue.Enqueue($guidForJob)
@@ -601,10 +598,6 @@ function Test-DataLakeAnalyticsCatalog
 			Assert-False {$i -eq 60} "dataLakeAnalytics accounts not in succeeded state even after 30 min."
 		}
 
-		# For now, all Job related tests just ensure that they have a valid response and do not throw.
-		# Wait for two minutes prior to attempting to submit the job in the freshly created account.
-		[Microsoft.Rest.ClientRuntime.Azure.TestFramework.TestUtilities]::Wait(120000)
-	
 		# Run a job to create the catalog items (except secret and credential)
 		$scriptTemplate = @"
 	DROP DATABASE IF EXISTS {0}; CREATE DATABASE {0};
@@ -748,6 +741,24 @@ function Test-DataLakeAnalyticsCatalog
 
 		Assert-True {$found} "Could not find the table $tableName in the table list"
 
+		# retrieve tables from the db (no schema)
+		$itemList = Get-AzureRMDataLakeAnalyticsCatalogItem -AccountName $accountName -ItemType Table -Path "$databaseName"
+
+		Assert-NotNull $itemList "The table list is null"
+
+		Assert-True {$itemList.count -gt 0} "The table list is empty"
+		$found = $false
+		foreach($item in $itemList)
+		{
+			if($item.Name -eq $tableName)
+			{
+				$found = $true
+				break
+			}
+		}
+
+		Assert-True {$found} "Could not find the table $tableName in the table list"
+
 		# retrieve the specific table
 		$specificItem = Get-AzureRMDataLakeAnalyticsCatalogItem -AccountName $accountName -ItemType Table -Path "$databaseName.dbo.$tableName"
 		Assert-NotNull $specificItem "Could not retrieve the table by name"
@@ -785,6 +796,24 @@ function Test-DataLakeAnalyticsCatalog
 
 		Assert-True {$found} "Could not find the TVF $tvfName in the TVF list"
 
+		# retrieve the TVF list from the db (no schema)
+		$itemList = Get-AzureRMDataLakeAnalyticsCatalogItem -AccountName $accountName -ItemType TableValuedFunction -Path "$databaseName"
+
+		Assert-NotNull $itemList "The TVF list is null"
+
+		Assert-True {$itemList.count -gt 0} "The TVF list is empty"
+		$found = $false
+		foreach($item in $itemList)
+		{
+			if($item.Name -eq $tvfName)
+			{
+				$found = $true
+				break
+			}
+		}
+
+		Assert-True {$found} "Could not find the TVF $tvfName in the TVF list"
+
 		# retrieve the specific TVF
 		$specificItem = Get-AzureRMDataLakeAnalyticsCatalogItem -AccountName $accountName -ItemType TableValuedFunction -Path "$databaseName.dbo.$tvfName"
 		Assert-NotNull $specificItem "Could not retrieve the TVF by name"
@@ -831,6 +860,25 @@ function Test-DataLakeAnalyticsCatalog
 
 		Assert-True {$found} "Could not find the view $viewName in the view list"
 
+		# get the views from the db only (no schema)
+		$itemList = Get-AzureRMDataLakeAnalyticsCatalogItem -AccountName $accountName -ItemType View -Path "$databaseName"
+
+		Assert-NotNull $itemList "The view list is null"
+
+		Assert-True {$itemList.count -gt 0} "The view list is empty"
+		$found = $false
+		foreach($item in $itemList)
+		{
+			if($item.Name -eq $viewName)
+			{
+				$found = $true
+				break
+			}
+		}
+	
+		Assert-True {$found} "Could not find the view $viewName in the view list"
+
+
 		# retrieve the specific view
 		$specificItem = Get-AzureRMDataLakeAnalyticsCatalogItem -AccountName $accountName -ItemType View -Path "$databaseName.dbo.$viewName"
 		Assert-NotNull $specificItem "Could not retrieve the view by name"