Merge branch 'attestation-powershell-module-cmdlet' of https://github.com/shleiAmy/azure-powershell into attestation-powershell-module-cmdlet

Author: shleiAmy

.azure-pipelines/util/analyze-steps.yml

@@ -15,14 +15,14 @@ steps:
   inputs:
     command: custom
     custom: msbuild
-    arguments: 'build.proj /t:GenerateHelp /p:Configuration=${{ parameters.configuration }}'
+    arguments: 'build.proj /t:GenerateHelp /p:Configuration=${{ parameters.configuration }};PullRequestNumber=$(System.PullRequest.PullRequestNumber)'
 
 - task: DotNetCoreCLI@2
   displayName: 'Static Analysis'
   inputs:
     command: custom
     custom: msbuild
-    arguments: 'build.proj /t:StaticAnalysis /p:Configuration=${{ parameters.configuration }}'
+    arguments: 'build.proj /t:StaticAnalysis /p:Configuration=${{ parameters.configuration }};PullRequestNumber=$(System.PullRequest.PullRequestNumber)'
 
 - template: publish-artifacts-steps.yml
   parameters:

.azure-pipelines/util/build-steps.yml

@@ -9,7 +9,7 @@ steps:
   inputs:
     command: custom
     custom: msbuild
-    arguments: 'build.proj /t:Build /p:Configuration=${{ parameters.configuration }};TestFramework=${{ parameters.testFramework }}'
+    arguments: 'build.proj /t:Build /p:Configuration=${{ parameters.configuration }};TestFramework=${{ parameters.testFramework }};PullRequestNumber=$(System.PullRequest.PullRequestNumber)'
 
 - template: publish-artifacts-steps.yml
   parameters:

.azure-pipelines/util/test-steps.yml

@@ -14,7 +14,7 @@ steps:
   inputs:
     command: custom
     custom: msbuild
-    arguments: 'build.proj /t:${{ parameters.testTarget }} /p:Configuration=${{ parameters.configuration }};TestFramework=${{ parameters.testFramework }}'
+    arguments: 'build.proj /t:${{ parameters.testTarget }} /p:Configuration=${{ parameters.configuration }};TestFramework=${{ parameters.testFramework }};PullRequestNumber=$(System.PullRequest.PullRequestNumber)'
 
 - template: publish-artifacts-steps.yml
   parameters:

.github/ISSUE_TEMPLATE/feature_request.md

@@ -2,7 +2,7 @@
 name: Feature request
 about: Suggest a new feature or improvement
 title: ''
-labels: Feature Request
+labels: Azure PS Team, Feature Request
 assignees: ''
 
 ---

.gitignore

@@ -222,9 +222,8 @@ Package
 launchSettings.json
 /src/StackAdmin/AzureRM/AzureRM.psm1
 /src/StackAdmin/AzureStack/AzureStack.psm1
+/CsprojMappings.json
 /ModuleMappings.json
-/SolutionMappings.json
-/TestMappings.json
 /tools/Modules/tmp
 /tools/Az/Az.psm1
 /Azure.PowerShell.sln

ChangeLog.md

@@ -81,6 +81,8 @@
   <UsingTask Condition="$(OnPremiseBuild)" TaskName="CorporateValidation" AssemblyFile="$(CIToolsPath)/Microsoft.WindowsAzure.Tools.Build.Tasks.OnPremise.dll" />
   <Import Condition="$(OnPremiseBuild)" Project="$(CIToolsPath)/Microsoft.WindowsAzure.Build.OnPremise.msbuild" />
   <UsingTask TaskName="ESRPSignTask" AssemblyFile="$(CISignRepoPath)/tools/sdkbuildtools/tasks/MS.Az.Sdk.OnPremise.Build.Tasks.dll" />
+  <UsingTask TaskName="FilesChangedTask" AssemblyFile="$(RepoArtifacts)Microsoft.Azure.Build.Tasks/Microsoft.Azure.Build.Tasks.dll" />
+  <UsingTask TaskName="FilterTask" AssemblyFile="$(RepoArtifacts)Microsoft.Azure.Build.Tasks/Microsoft.Azure.Build.Tasks.dll" />
 
   <Target Name="Clean">
     <Message Importance="high" Text="Cleaning Cmdlets..." />
@@ -93,9 +95,13 @@
     <Exec Command="$(PowerShellCoreCommandPrefix) &quot;Get-ChildItem -Path $(MSBuildThisFileDirectory) -Recurse -Include 'bin','obj','TestResults' | Remove-Item -Recurse -Force -ErrorAction Ignore&quot;" IgnoreExitCode="true" />
   </Target>
 
-  <Target Name="FilterBuild">
+  <Target Name="FilterBuild" Condition="$(PullRequestNumber) != ''">
+    <Message Importance="high" Text="Filtering projects and modules..." />
     <Exec Command="$(PowerShellCoreCommandPrefix) &quot;. $(RepoTools)/CreateFilterMappings.ps1&quot;" />
 
+    <!-- Build the Microsoft.Azure.Build.Tasks project -->
+    <Exec Command="dotnet publish $(RepoTools)BuildPackagesTask/Microsoft.Azure.Build.Tasks/Microsoft.Azure.Build.Tasks.csproj -c $(Configuration) -f netstandard2.0" />
+
     <!-- Get all of the files changed in the given pull request -->
     <FilesChangedTask RepositoryOwner="Azure" RepositoryName="azure-powershell" PullRequestNumber="$(PullRequestNumber)">
       <Output TaskParameter="FilesChanged" ItemName="FilesChanged" />
@@ -105,33 +111,23 @@
     <FilterTask FilesChanged="@(FilesChanged)" MapFilePath="./ModuleMappings.json">
       <Output TaskParameter="Output" ItemName="ModulesChanged" />
     </FilterTask>
-    <Message Text="Filtering help generation and StaticAnalysis by the following modules:" />
-    <Message Text="%(ModulesChanged.Identity)" />
-    <Message Text="Total: @(ModulesChanged->Count())" />
-    <Message Text="" />
-
-    <!-- Get the list of solutions to build -->
-    <FilterTask FilesChanged="@(FilesChanged)" MapFilePath="./SolutionMappings.json">
-      <Output TaskParameter="Output" ItemName="CmdletSolutionsToBuild" />
-    </FilterTask>
-    <Message Text="Filtering solutions to build by the following:" />
-    <Message Text="%(CmdletSolutionsToBuild.Identity)" />
-    <Message Text="Total: @(CmdletSolutionsToBuild->Count())" />
-    <Message Text="" />
-
-    <!--Get the list of tests to be run based on files changed from a specified PullRequestNumber. Mapping between paths and test DLLs is used to produce the list.-->
-    <FilterTask FilesChanged="@(FilesChanged)" MapFilePath="./TestMappings.json">
-      <Output TaskParameter="Output" ItemName="XUnitTests" />
+    <Message Importance="high" Text="Filtering help generation and StaticAnalysis by the following modules:" />
+    <Message Importance="high" Text="%(ModulesChanged.Identity)" />
+    <Message Importance="high" Text="Total: @(ModulesChanged->Count())" />
+    <Message Importance="high" Text="" />
+
+    <!-- Get the list of projects to build -->
+    <FilterTask FilesChanged="@(FilesChanged)" MapFilePath="./CsprojMappings.json">
+      <Output TaskParameter="Output" ItemName="ProjectsToBuild" />
     </FilterTask>
-    <Message Text="Using these test assemblies:" />
-    <Message Text="%(XUnitTests.Identity)" />
-    <Message Text="Total: @(XunitTests->Count())" />
-    <Message Text="" />
+    <Message Importance="high" Text="Filtering projects to build by the following:" />
+    <Message Importance="high" Text="%(ProjectsToBuild.Identity)" />
+    <Message Importance="high" Text="Total: @(ProjectsToBuild->Count())" />
+    <Message Importance="high" Text="" />
   </Target>
 
   <!-- Build all flavors of the Cmdlets -->
-  <!-- TODO: Reimplement the FilterBuild and update RepoTasks to NetStandard -->
-  <Target Name="Build">
+  <Target Name="Build" DependsOnTargets="FilterBuild">
     <Message Importance="high" Text="Building Cmdlets..." />
 
     <MakeDir Directories="$(RepoArtifacts)" />
@@ -142,13 +138,19 @@
     <!-- Build and create package content -->
     <Exec Command="dotnet --version" />
     <Exec Command="dotnet new sln -n Azure.PowerShell -o $(RepoArtifacts) --force" />
-    <ItemGroup>
+    <ItemGroup Condition="$(PullRequestNumber) == ''">
       <CsprojFiles Include="$(RepoRoot)src/**/*.csproj" Exclude="$(RepoRoot)src/**/*.Test.csproj;$(RepoRoot)src/**/Authenticators.csproj" />
       <CsprojFiles Include="$(RepoRoot)src/**/*.Test.csproj" Exclude="$(Net472TestExclude)" Condition="'$(Configuration)' != 'Release' and '$(TestsToRun)' == 'All'" />
       <CsprojFiles Include="$(RepoRoot)src/**/*.Test.csproj" Exclude="$(CoreTests)$(Net472TestExclude)" Condition="'$(Configuration)' != 'Release' and '$(TestsToRun)' == 'NonCore'" />
       <CsprojFiles Include="$(CoreTests)" Exclude="$(Net472TestExclude)" Condition="'$(Configuration)' != 'Release' and '$(TestsToRun)' == 'Core'" />
       <CsprojFiles Include="$(RepoRoot)src/**/Authenticators.csproj" Condition="'$([MSBuild]::IsOsPlatform(&quot;Windows&quot;))' == 'true' and '$(TestFramework)' != 'net472'" />
     </ItemGroup>
+    <ItemGroup Condition="$(PullRequestNumber) != ''">
+      <!-- Always build and test common code -->
+      <CsprojFiles Include="$(LibraryRoot)src/Accounts/**/*.csproj;$(LibraryRoot)tools/TestFx/TestFx.csproj" Exclude="$(LibraryRoot)src/**/Authenticators.csproj" />
+      <CsprojFiles Include="%(ProjectsToBuild.Identity)" />
+      <CsprojFiles Include="$(LibraryRoot)src/**/Authenticators.csproj" Condition="'$([MSBuild]::IsOsPlatform(&quot;Windows&quot;))' == 'true'" />
+    </ItemGroup>
     <!-- https://github.com/dotnet/cli/issues/6295#issuecomment-346973582 -->
     <Exec Command="dotnet sln $(RepoArtifacts)Azure.PowerShell.sln add &quot;%(CsprojFiles.FullPath)&quot;" />
     <PropertyGroup>
@@ -166,9 +168,9 @@
     <!-- Delete powershell runtime files -->
     <PropertyGroup>
       <RuntimeDllsIncludeList>Microsoft.Powershell.*.dll,System*.dll,Microsoft.VisualBasic.dll,Microsoft.CSharp.dll,Microsoft.CodeAnalysis.dll,Microsoft.CodeAnalysis.CSharp.dll</RuntimeDllsIncludeList>
-      <RuntimeDllsExcludeList>System.Security.Cryptography.ProtectedData.dll,System.Configuration.ConfigurationManager.dll,System.Runtime.CompilerServices.Unsafe.dll,System.IO.FileSystem.AccessControl.dll,System.Buffers.dll,System.Text.Encodings.Web.dll,System.CodeDom.dll</RuntimeDllsExcludeList>
+      <RuntimeDllsExcludeList>System.Security.Cryptography.ProtectedData.dll,System.Configuration.ConfigurationManager.dll,System.Runtime.CompilerServices.Unsafe.dll,System.IO.FileSystem.AccessControl.dll,System.Buffers.dll,System.Text.Encodings.Web.dll,System.CodeDom.dll,System.Management.dll</RuntimeDllsExcludeList>
     </PropertyGroup>
-    <Exec Command="$(PowerShellCoreCommandPrefix) &quot;Get-ChildItem -Path $(RepoArtifacts)/$(Configuration) -Recurse -Include $(RuntimeDllsIncludeList) -Exclude $(RuntimeDllsExcludeList) | Remove-Item -Recurse -Force&quot;" />
+    <Exec Command="$(PowerShellCoreCommandPrefix) &quot;Get-ChildItem -Path $(RepoArtifacts)/$(Configuration) -Recurse -Include $(RuntimeDllsIncludeList) -Exclude $(RuntimeDllsExcludeList) | Where-Object {$_.FullName -notlike '*PreloadAssemblies*' -and $_.FullName -notlike '*NetCoreAssemblies*'} | Remove-Item -Force&quot;" />
     <Exec Command="$(PowerShellCoreCommandPrefix) &quot;Get-ChildItem -Path $(RepoArtifacts)/$(Configuration) -Recurse -Include 'runtimes' | Remove-Item -Recurse -Force&quot;" Condition="'$(CodeSign)' == 'true'" />
 
     <Exec Command="$(PowerShellCoreCommandPrefix) &quot;. $(RepoTools)/UpdateModules.ps1 -BuildConfig $(Configuration) -Scope $(Scope)&quot;" />
@@ -297,7 +299,7 @@ $(RepoArtifacts)/$(Configuration)/**/Microsoft.Azure.PowerShell.Authenticators.d
     <!-- RemoveCodeSignArtifacts.ps1 -->
     <Message Text="variables.Remove artifacts section" Importance="high" />
     <Exec Command="$(PowerShellCoreCommandPrefix) &quot;Get-ChildItem -Path $(RepoArtifacts) -Recurse -Include 'Signed','Unsigned' | Remove-Item -Recurse -Force -Confirm:$false -ErrorAction Ignore&quot;" IgnoreExitCode="true" />
-    
+
     <!-- CheckSignature.ps1 -->
     <Message Text="variables.CheckSignature section" Importance="high" />
     <Exec Command="$(PowerShellCoreCommandPrefix) &quot;. $(RepoTools)/CheckSignature.ps1 -CustomPath $(RepoArtifacts)/$(Configuration)&quot;" Condition="'$(Scope)' != 'Stack'" />
@@ -326,4 +328,4 @@ $(RepoArtifacts)/$(Configuration)/**/Microsoft.Azure.PowerShell.Authenticators.d
     <SetEnvVar EnvName="SignedMsiDir" EnvValue="$(SignedOutputRootDir)" />
   </Target>
 
-</Project>
+</Project>

build.proj

@@ -251,16 +251,33 @@ This document describes the changes between the 1.0.0 and 2.0.0 versions of Az
   ```
 
 ### Az.HDInsight
-- Removed cmdlet `Grant-AzHDInsightHttpServicesAccess` and replaced with `Set-AzHDInsightGatewayCredential`
+- Removed the `Grant-AzHDInsightHttpServicesAccess` and `Revoke-AzHDInsightHttpServicesAccess` cmdlets. These are no longer necessary because HTTP access is always enabled on all HDInsight clusters.
+- Added a new `Set-AzHDInsightGatewayCredential`  cmdlet. Use this cmdlet to change the gateway HTTP username and password (replaces `Grant-AzHDInsightHttpServicesAccess`).
+- Updated the `Get-AzHDInsightJobOutput` cmdlet to support granular role-based access to the storage key.
+    - Users with HDInsight Cluster Operator, Contributor, or Owner roles will not be affected.
+    - Users with only the Reader role will need to specify `DefaultStorageAccountKey` parameter explicitly.
+
+For more information about these role-based access changes, see [aka.ms/hdi-config-update](http://aka.ms/hdi-config-update)
+
   #### Before
   ```powershell
-  Grant-AzHDInsightHttpServicesAccess ...
+  Grant-AzHDInsightHttpServicesAccess -ClusterName $cluster -HttpCredential $credential
+  ```
+  #### After
+  ```powershell
+  Set-AzHDInsightGatewayCredential -ClusterName $cluster -HttpCredential $credential
+  ```
+
+###  Users with only Reader role for cmdlet Get-AzHDInsightJobOutput
+
+  ####  Before
+  ```powershell
+  Get-AzHDInsightJobOutput  -ClusterName $clusterName -JobId $jobId
   ```
   #### After
   ```powershell
-  Set-AzHDInsightGatewayCredential ...
+  Get-AzHDInsightJobOutput  -ClusterName $clusterName -JobId $jobId -DefaultStorageAccountKey $storageAccountKey
   ```
-- Removed cmdlet `Revoke-AzHDInsightHttpServicesAccess`
 
 ### Az.Storage
 - Namespaces for types returned from Blob, Queue, and File cmdlets have changed their namespace from `Microsoft.WindowsAzure.Storage` to `Microsoft.Azure.Storage`.  While this is not technically a breaking change according to the breaking change policy, it may require some changes in code that uses the methods from the Storage .Net SDK to interact with the objects returned from these cmdlets.

documentation/migration-guides/Az.2.0.0-migration-guide.md

@@ -156,4 +156,12 @@ If you see a version of Az.Accounts installed in the session, you can resolve th
 
 Azure PowerShell is a set of binary modules, meaning that cmdlets are defined and implemented in .NET Standard code. PowerShell loads cmdlet assemblies into a single AppDomain, and these assemblies cannot be unloaded, even if the module is removed. Therefore, a module assembly and all its dependent assemblies remain loaded in the PowerShell session once it is imported until the PowerShell session is closed. Additionally, the types defined in a .NET Standard assembly are strongly tied to the assembly version, so that the types from version `A` and version `B` of the same assembly will appear as different types in the PowerShell session. When these types are used in common by multiple assemblies (as in the authentication types in Az.Accounts), then the types that are meant to be common are actually different for each module version. This causes problems when authentication information is instantiated using types from version `X` of Az.Accounts, but an assembly requires version `Y` of these types.
 
-This underlying problem is solved in Az by requiring all authentication types used by the modules to be fully backward compatible.
+This underlying problem is solved in Az by requiring all authentication types used by the modules to be fully backward compatible.
+
+### Newtonsoft.Json versions per PowerShell edition
+
+The library `Newtonsoft.Json` is used throughout our cmdlets to handle JSON information. However, we do not have control of the version of this library that is loaded in PowerShell. To troubleshoot serialization issues, here is a table of the versions of `Newtonsoft.Json` that are loaded by the various editions of PowerShell.
+
+| | Windows PowerShell 5.1 | PowerShell Core 6.0 | PowerShell Core 6.1 | PowerShell Core 6.2
+| - | - | - | - | - |
+| **Newtonsoft.Json** | 6.0.8 | 10.0.3 | 11.0.2 | 12.0.1