Update to latest ADL Packages

setup/azurecmdfiles.wxi

@@ -981,9 +981,6 @@
                         <Component Id="cmp3C96010EFA751762A76B66B811613835" Guid="*">
                             <File Id="fil509D1E7494560B2BB3462B8FAD7954CF" KeyPath="yes" Source="$(var.sourceDir)\ResourceManager\AzureResourceManager\AzureRM.DataLakeStore\Microsoft.Azure.Management.DataLake.Store.dll" />
                         </Component>
-                        <Component Id="cmpFECE1A517C9182672B042869C46847FC" Guid="*">
-                            <File Id="fil00C18C352C99B9BCFA42EFFACE0AEFF7" KeyPath="yes" Source="$(var.sourceDir)\ResourceManager\AzureResourceManager\AzureRM.DataLakeStore\Microsoft.Azure.Management.DataLake.StoreUploader.dll" />
-                        </Component>
                         <Component Id="cmp500DAF931B66664E6A3308FAC825093C" Guid="*">
                             <File Id="filB384FAE2600A5CAE72336384DBA3B581" KeyPath="yes" Source="$(var.sourceDir)\ResourceManager\AzureResourceManager\AzureRM.DataLakeStore\Microsoft.Azure.Management.ResourceManager.dll" />
                         </Component>
@@ -6038,7 +6035,6 @@
             <ComponentRef Id="cmp87FB43928F7A2ECF62C0141446D21A9F" />
             <ComponentRef Id="cmpD2292657EF3D9CEAC5D4FA7DEB3E7E8E" />
             <ComponentRef Id="cmp3C96010EFA751762A76B66B811613835" />
-            <ComponentRef Id="cmpFECE1A517C9182672B042869C46847FC" />
             <ComponentRef Id="cmp500DAF931B66664E6A3308FAC825093C" />
             <ComponentRef Id="cmpFCFE1AC675F0BA7336B392A419D70042" />
             <ComponentRef Id="cmpB41201E194E7639BC9C33ADECC894426" />

src/ResourceManager/DataLakeAnalytics/AzureRM.DataLakeAnalytics.psd1

@@ -92,7 +92,11 @@ CmdletsToExport = 'Get-AzureRmDataLakeAnalyticsDataSource',
                'Submit-AzureRmDataLakeAnalyticsJob', 
                'New-AzureRmDataLakeAnalyticsAccount', 
                'Remove-AzureRmDataLakeAnalyticsAccount', 
-               'Set-AzureRmDataLakeAnalyticsAccount'
+               'Set-AzureRmDataLakeAnalyticsAccount',
+			   'Add-AzureRmDataLakeAnalyticsFirewallRule',
+			   'Get-AzureRmDataLakeAnalyticsFirewallRule',
+			   'Set-AzureRmDataLakeAnalyticsFirewallRule',
+			   'Remove-AzureRmDataLakeAnalyticsFirewallRule'
 
 # Variables to export from this module
 # VariablesToExport = @()
@@ -106,7 +110,9 @@ AliasesToExport = 'Get-AdlAnalyticsDataSource', 'New-AdlCatalogCredential',
                'Test-AdlAnalyticsAccount', 'Remove-AdlAnalyticsDataSource', 
                'Add-AdlAnalyticsDataSource', 'Stop-AdlJob', 'Get-AdlJob', 
                'Get-AdlAnalyticsAccount', 'Submit-AdlJob', 'New-AdlAnalyticsAccount', 
-               'Remove-AdlAnalyticsAccount', 'Set-AdlAnalyticsAccount'
+               'Remove-AdlAnalyticsAccount', 'Set-AdlAnalyticsAccount',
+			   'Add-AdlAnalyticsFirewallRule', 'Get-AdlAnalyticsFirewallRule',
+			   'Set-AdlAnalyticsFirewallRule', 'Remove-AdlAnalyticsFirewallRule'
 
 # DSC resources to export from this module
 # DscResourcesToExport = @()

src/ResourceManager/DataLakeAnalytics/ChangeLog.md

@@ -18,6 +18,16 @@
         - Additional information about change #1
 -->
 ## Current Release
+* Add Firewall Rule support to Data Lake Analytics:
+    - Add-AzureRMDataLakeAnalyticsFirewallRule
+    - Get-AzureRMDataLakeAnalyticsFirewallRule
+    - Set-AzureRMDataLakeAnalyticsFirewallRule
+    - Remove-AzureRMDataLakeAnalyticsFirewallRule
+    - Set-AzureRMDataLakeAnalyticsAccount supports enabling/disabling the firewall and allowing/blocking Azure originating IPs through the firewall
+    - Warnings will be raised if updating firewall rules when the firewall is disabled
+* Fix Get-AzureRMDataLakeAnalyticsJob functionality:
+    - Top now correctly returns the number of jobs specified. The default number of jobs to return is 500. The more jobs requested the longer the command will take.
+* Remove explicit restrictions on resource locations. If Data Lake Analytics is not supported in a region, we will surface an error from the service.
 
 ## Version 2.5.0
 * Update Get-AdlJob to support Top parameter

src/ResourceManager/DataLakeAnalytics/Commands.DataLakeAnalytics.Test/Commands.DataLakeAnalytics.Test.csproj

@@ -64,12 +64,12 @@
     </Reference>
     <Reference Include="Microsoft.Azure.Management.DataLake.Analytics, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\..\..\packages\Microsoft.Azure.Management.DataLake.Analytics.2.0.0\lib\net45\Microsoft.Azure.Management.DataLake.Analytics.dll</HintPath>
+      <HintPath>..\..\..\packages\Microsoft.Azure.Management.DataLake.Analytics.2.0.1-preview\lib\net45\Microsoft.Azure.Management.DataLake.Analytics.dll</HintPath>
       <Private>True</Private>
     </Reference>
-    <Reference Include="Microsoft.Azure.Management.DataLake.Store, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+    <Reference Include="Microsoft.Azure.Management.DataLake.Store, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\..\..\packages\Microsoft.Azure.Management.DataLake.Store.1.0.3\lib\net45\Microsoft.Azure.Management.DataLake.Store.dll</HintPath>
+      <HintPath>..\..\..\packages\Microsoft.Azure.Management.DataLake.Store.2.0.1-preview\lib\net45\Microsoft.Azure.Management.DataLake.Store.dll</HintPath>
       <Private>True</Private>
     </Reference>
     <Reference Include="Microsoft.Azure.Management.ResourceManager, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
@@ -230,6 +230,9 @@
     <None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaAliasTests\TestAdlaCatalog.json">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
+    <None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaAliasTests\TestAdlaFirewallRules.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
     <None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaAliasTests\TestAdlaJob.json">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
@@ -248,6 +251,9 @@
     <None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaTests\TestAdlaCatalog.json">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
+    <None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaTests\TestAdlaFirewallRules.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
     <None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaTests\TestAdlaJob.json">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>

src/ResourceManager/DataLakeAnalytics/Commands.DataLakeAnalytics.Test/ScenarioTests/AdlaAliasTests.cs

@@ -38,12 +38,22 @@ public void TestAdlaAccount()
         [Trait(Category.AcceptanceType, Category.CheckIn)]
         public void TestAdlaAccountTiers()
         {
-            AdlaTestsBase.NewInstance.RunPsTest(true,
+            AdlaTestsBase.NewInstance.RunPsTest(false,
                 string.Format(
                     "Test-DataLakeAnalyticsAccountTiers -location '{0}'",
                     AdlaTestsBase.resourceGroupLocation));
         }
 
+        [Fact]
+        [Trait(Category.AcceptanceType, Category.CheckIn)]
+        public void TestAdlaFirewallRules()
+        {
+            AdlaTestsBase.NewInstance.RunPsTest(false,
+                string.Format(
+                    "Test-DataLakeAnalyticsFirewall -location '{0}'",
+                    AdlaTestsBase.resourceGroupLocation));
+        }
+
         [Fact]
         [Trait(Category.AcceptanceType, Category.CheckIn)]
         public void TestAdlaCatalog()

src/ResourceManager/DataLakeAnalytics/Commands.DataLakeAnalytics.Test/ScenarioTests/AdlaAliasTests.ps1

@@ -1,5 +1,95 @@
 <#
 .SYNOPSIS
+Tests DataLakeAnalytics Account firewall rule lifecycle (Create, Update, Get, List, Delete).
+#>
+function Test-DataLakeAnalyticsFirewall
+{
+    param
+	(
+		$resourceGroupName = (Get-ResourceGroupName),
+		$accountName = (Get-DataLakeAnalyticsAccountName),
+		$dataLakeAccountName = (Get-DataLakeStoreAccountName),
+		$location = "West US"
+	)
+
+	try
+	{
+		# Creating Account
+		New-AzureRmResourceGroup -Name $resourceGroupName -Location $location
+
+		# Test to make sure the account doesn't exist
+		Assert-False {Test-AdlAnalyticsAccount -ResourceGroupName $resourceGroupName -Name $accountName}
+		# Test it without specifying a resource group
+		Assert-False {Test-AdlAnalyticsAccount -Name $accountName}
+
+		New-AdlStore -ResourceGroupName $resourceGroupName -Name $dataLakeAccountName -Location $location
+
+		$accountCreated = New-AdlAnalyticsAccount -ResourceGroupName $resourceGroupName -Name $accountName -Location $location -DefaultDataLakeStore $dataLakeAccountName
+
+		Assert-AreEqual $accountName $accountCreated.Name
+		Assert-AreEqual $location $accountCreated.Location
+		Assert-AreEqual "Microsoft.DataLakeAnalytics/accounts" $accountCreated.Type
+		Assert-True {$accountCreated.Id -like "*$resourceGroupName*"}
+
+		# In loop to check if account exists
+		for ($i = 0; $i -le 60; $i++)
+		{
+			[array]$accountGet = Get-AdlAnalyticsAccount -ResourceGroupName $resourceGroupName -Name $accountName
+			if ($accountGet[0].ProvisioningState -like "Succeeded")
+			{
+				Assert-AreEqual $accountName $accountGet[0].Name
+				Assert-AreEqual $location $accountGet[0].Location
+				Assert-AreEqual "Microsoft.DataLakeAnalytics/accounts" $accountGet[0].Type
+				Assert-True {$accountGet[0].Id -like "*$resourceGroupName*"}
+				break
+			}
+
+			Write-Host "account not yet provisioned. current state: $($accountGet[0].ProvisioningState)"
+			[Microsoft.WindowsAzure.Commands.Utilities.Common.TestMockSupport]::Delay(30000)
+			Assert-False {$i -eq 60} " Data Lake Analytics account is not in succeeded state even after 30 min."
+		}
+
+		# Test to make sure the account does exist
+		Assert-True {Test-AdlAnalyticsAccount -ResourceGroupName $resourceGroupName -Name $accountName}
+
+		# Test to enable the firewall as well as allowing azure IPs
+		Assert-AreEqual "Disabled" $accountCreated.FirewallState 
+		Assert-AreEqual "Disabled" $accountCreated.FirewallAllowAzureIps 
+
+		$accountSet = Set-AdlAnalyticsAccount -Name $accountName -FirewallState "Enabled" -AllowAzureIpState "Enabled"
+
+		Assert-AreEqual "Enabled" $accountSet.FirewallState 
+		Assert-AreEqual "Enabled" $accountSet.FirewallAllowAzureIps
+
+		$firewallRuleName = getAssetName
+		$startIp = "127.0.0.1"
+		$endIp = "127.0.0.2"
+		# Add a firewall rule
+		Add-AdlAnalyticsFirewallRule -AccountName $accountName -Name $firewallRuleName -StartIpAddress $startIp -EndIpAddress $endIp
+
+		# Get the firewall rule
+		$result = Get-AdlAnalyticsFirewallRule -AccountName $accountName -Name $firewallRuleName
+		Assert-AreEqual $firewallRuleName $result.Name
+		Assert-AreEqual $startIp $result.StartIpAddress
+		Assert-AreEqual $endIp $result.EndIpAddress
+
+		# remove the firewall rule
+		Remove-AdlAnalyticsFirewallRule -AccountName $accountName -Name $firewallRuleName
+
+		# Make sure get throws.
+		Assert-Throws {Get-AdlAnalyticsFirewallRule -AccountName $accountName -Name $firewallRuleName}
+	}
+	finally
+	{
+		# cleanup the resource group that was used in case it still exists. This is a best effort task, we ignore failures here.
+		Invoke-HandledCmdlet -Command {Remove-AdlAnalyticsAccount -ResourceGroupName $resourceGroupName -Name $accountName -Force -ErrorAction SilentlyContinue} -IgnoreFailures
+		Invoke-HandledCmdlet -Command {Remove-AdlStore -ResourceGroupName $resourceGroupName -Name $dataLakeAccountName -Force -ErrorAction SilentlyContinue} -IgnoreFailures
+		Invoke-HandledCmdlet -Command {Remove-AzureRmResourceGroup -Name $resourceGroupName -Force -ErrorAction SilentlyContinue} -IgnoreFailures
+	}
+}
+
+<#
+.SYNOPSIS
 Tests DataLakeAnalytics Account Lifecycle (Create, Update, Get, List, Delete).
 #>
 function Test-DataLakeAnalyticsAccount

src/ResourceManager/DataLakeAnalytics/Commands.DataLakeAnalytics.Test/ScenarioTests/AdlaTests.cs

@@ -38,12 +38,22 @@ public void TestAdlaAccount()
         [Trait(Category.AcceptanceType, Category.CheckIn)]
         public void TestAdlaAccountTiers()
         {
-            AdlaTestsBase.NewInstance.RunPsTest(true,
+            AdlaTestsBase.NewInstance.RunPsTest(false,
                 string.Format(
                     "Test-DataLakeAnalyticsAccountTiers -location '{0}'",
                     AdlaTestsBase.resourceGroupLocation));
         }
 
+        [Fact]
+        [Trait(Category.AcceptanceType, Category.CheckIn)]
+        public void TestAdlaFirewallRules()
+        {
+            AdlaTestsBase.NewInstance.RunPsTest(false,
+                string.Format(
+                    "Test-DataLakeAnalyticsFirewall -location '{0}'",
+                    AdlaTestsBase.resourceGroupLocation));
+        }
+
         [Fact]
         [Trait(Category.AcceptanceType, Category.CheckIn)]
         public void TestAdlaCatalog()

src/ResourceManager/DataLakeAnalytics/Commands.DataLakeAnalytics.Test/ScenarioTests/AdlaTests.ps1

@@ -1,5 +1,95 @@
 <#
 .SYNOPSIS
+Tests DataLakeAnalytics Account firewall rule lifecycle (Create, Update, Get, List, Delete).
+#>
+function Test-DataLakeAnalyticsFirewall
+{
+    param
+	(
+		$resourceGroupName = (Get-ResourceGroupName),
+		$accountName = (Get-DataLakeAnalyticsAccountName),
+		$dataLakeAccountName = (Get-DataLakeStoreAccountName),
+		$location = "West US"
+	)
+
+	try
+	{
+		# Creating Account
+		New-AzureRmResourceGroup -Name $resourceGroupName -Location $location
+
+		# Test to make sure the account doesn't exist
+		Assert-False {Test-AzureRMDataLakeAnalyticsAccount -ResourceGroupName $resourceGroupName -Name $accountName}
+		# Test it without specifying a resource group
+		Assert-False {Test-AzureRMDataLakeAnalyticsAccount -Name $accountName}
+
+		New-AzureRMDataLakeStoreAccount -ResourceGroupName $resourceGroupName -Name $dataLakeAccountName -Location $location
+
+		$accountCreated = New-AzureRMDataLakeAnalyticsAccount -ResourceGroupName $resourceGroupName -Name $accountName -Location $location -DefaultDataLakeStore $dataLakeAccountName
+
+		Assert-AreEqual $accountName $accountCreated.Name
+		Assert-AreEqual $location $accountCreated.Location
+		Assert-AreEqual "Microsoft.DataLakeAnalytics/accounts" $accountCreated.Type
+		Assert-True {$accountCreated.Id -like "*$resourceGroupName*"}
+
+		# In loop to check if account exists
+		for ($i = 0; $i -le 60; $i++)
+		{
+			[array]$accountGet = Get-AzureRMDataLakeAnalyticsAccount -ResourceGroupName $resourceGroupName -Name $accountName
+			if ($accountGet[0].ProvisioningState -like "Succeeded")
+			{
+				Assert-AreEqual $accountName $accountGet[0].Name
+				Assert-AreEqual $location $accountGet[0].Location
+				Assert-AreEqual "Microsoft.DataLakeAnalytics/accounts" $accountGet[0].Type
+				Assert-True {$accountGet[0].Id -like "*$resourceGroupName*"}
+				break
+			}
+
+			Write-Host "account not yet provisioned. current state: $($accountGet[0].ProvisioningState)"
+			[Microsoft.WindowsAzure.Commands.Utilities.Common.TestMockSupport]::Delay(30000)
+			Assert-False {$i -eq 60} " Data Lake Analytics account is not in succeeded state even after 30 min."
+		}
+
+		# Test to make sure the account does exist
+		Assert-True {Test-AzureRMDataLakeAnalyticsAccount -ResourceGroupName $resourceGroupName -Name $accountName}
+
+		# Test to enable the firewall as well as allowing azure IPs
+		Assert-AreEqual "Disabled" $accountCreated.FirewallState 
+		Assert-AreEqual "Disabled" $accountCreated.FirewallAllowAzureIps 
+
+		$accountSet = Set-AzureRMDataLakeAnalyticsAccount -Name $accountName -FirewallState "Enabled" -AllowAzureIpState "Enabled"
+
+		Assert-AreEqual "Enabled" $accountSet.FirewallState 
+		Assert-AreEqual "Enabled" $accountSet.FirewallAllowAzureIps
+
+		$firewallRuleName = getAssetName
+		$startIp = "127.0.0.1"
+		$endIp = "127.0.0.2"
+		# Add a firewall rule
+		Add-AzureRMDataLakeAnalyticsFirewallRule -AccountName $accountName -Name $firewallRuleName -StartIpAddress $startIp -EndIpAddress $endIp
+
+		# Get the firewall rule
+		$result = Get-AzureRMDataLakeAnalyticsFirewallRule -AccountName $accountName -Name $firewallRuleName
+		Assert-AreEqual $firewallRuleName $result.Name
+		Assert-AreEqual $startIp $result.StartIpAddress
+		Assert-AreEqual $endIp $result.EndIpAddress
+
+		# remove the firewall rule
+		Remove-AzureRMDataLakeAnalyticsFirewallRule -AccountName $accountName -Name $firewallRuleName
+
+		# Make sure get throws.
+		Assert-Throws {Get-AzureRMDataLakeAnalyticsFirewallRule -AccountName $accountName -Name $firewallRuleName}
+	}
+	finally
+	{
+		# cleanup the resource group that was used in case it still exists. This is a best effort task, we ignore failures here.
+		Invoke-HandledCmdlet -Command {Remove-AzureRMDataLakeAnalyticsAccount -ResourceGroupName $resourceGroupName -Name $accountName -Force -ErrorAction SilentlyContinue} -IgnoreFailures
+		Invoke-HandledCmdlet -Command {Remove-AzureRMDataLakeStoreAccount -ResourceGroupName $resourceGroupName -Name $dataLakeAccountName -Force -ErrorAction SilentlyContinue} -IgnoreFailures
+		Invoke-HandledCmdlet -Command {Remove-AzureRmResourceGroup -Name $resourceGroupName -Force -ErrorAction SilentlyContinue} -IgnoreFailures
+	}
+}
+
+<#
+.SYNOPSIS
 Tests DataLakeAnalytics Account Lifecycle (Create, Update, Get, List, Delete).
 #>
 function Test-DataLakeAnalyticsAccount