Correlate the suggestion related events (#14041)

* Correlate the suggestion related events.

- Collects the event when our suggestion is displayed.
- Use SuggestionSessionId to correlate the suggestions that are provided
  by us, displayed by PSReadLine, and accepted by the user.
- Collects the client id in the telemetry events to identify if it's
  called by PSReadLine or something else.

* Update the version.

* Use new api.

* Guard against when the display mode is not known.

* Fix the package and a bug

- Use the preview packages that contain the change.
- Fix a bug that the loop infinitely when parsing the parameter values
  and the last parameter is a switch parameter.

* Rename the variable UserId to HashUserId.

* Updated release notes

* Added ChangeLog.md

* Remove the binaries that're added accidentally.

* Fixed typo in changelog

* Fix the broken merge.

* Fix the merge.

* Remove the test version suffix.

* Add a new field to the telemetry.

* Update the psreadline binaries.

* Update README.md

Updated README to reflect on the change of versions.

* Update Az.Tools.Predictor.psd1

Updated version requirements and addressed release notes comments.

* Update the doc.

* Update the link to dotnet 6.0

* Fix the list number.

* Remove .NET 6

Co-authored-by: Damien Caro <[email protected]>

Author: kceiw

tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/Az.Tools.Predictor.Test.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net5.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
     <AssemblyName>Microsoft.Azure.PowerShell.Tools.AzPredictor.Test</AssemblyName>
     <RootNamespace>Microsoft.Azure.PowerShell.Tools.AzPredictor.Test</RootNamespace>
     <IsPackable>false</IsPackable>
@@ -14,7 +14,6 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="System.Management.Automation" Version="7.1.0-preview.7" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.1" />
     <PackageReference Include="xunit" Version="2.4.1" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">

tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/AzPredictorTelemetryTests.cs

@@ -17,6 +17,7 @@
 using System.Linq;
 using System.Management.Automation.Subsystem;
 using System.Threading;
+using System.Threading.Tasks;
 using Xunit;
 
 namespace Microsoft.Azure.PowerShell.Tools.AzPredictor.Test
@@ -27,6 +28,7 @@ namespace Microsoft.Azure.PowerShell.Tools.AzPredictor.Test
     [Collection("Model collection")]
     public sealed class AzPredictorTests
     {
+        private const string AzPredictorClient = "Test";
         private readonly ModelFixture _fixture;
         private readonly MockAzPredictorTelemetryClient _telemetryClient;
         private readonly MockAzPredictorService _service;
@@ -54,65 +56,66 @@ public AzPredictorTests(ModelFixture modelFixture)
         /// Verify we replace unsupported command with <see cref="AzPredictorConstants.CommandPlaceholder"/>.
         /// </summary>
         [Fact]
-        public void VerifyRequestPredictionForOneUnsupportedCommandInHistory()
+        public async Task VerifyRequestPredictionForOneUnsupportedCommandInHistory()
         {
             IReadOnlyList<string> history = new List<string>()
             {
                 "git status"
             };
 
-            _telemetryClient.RecordedSuggestion = null;
             _service.Commands = null;
             _service.History = null;
+            _service.ResetRequestPredictionTask();
 
-            _azPredictor.StartEarlyProcessing(history);
+            _azPredictor.StartEarlyProcessing(AzPredictorTests.AzPredictorClient, history);
+            await _service.RequestPredictionTaskCompletionSource.Task;
 
             Assert.Equal(new List<string>() { AzPredictorConstants.CommandPlaceholder, AzPredictorConstants.CommandPlaceholder }, _service.Commands);
-            Assert.Equal(AzPredictorConstants.CommandPlaceholder, _telemetryClient.RecordedSuggestion.HistoryLine);
             Assert.Null(_service.History);
         }
 
         /// <summary>
         /// Verify that we masked the supported command in requesting prediction and telemetry.
         /// </summary>
         [Fact]
-        public void VerifyRequestPredictionForOneSupportedCommandInHistory()
+        public async Task VerifyRequestPredictionForOneSupportedCommandInHistory()
         {
             IReadOnlyList<string> history = new List<string>()
             {
                 "New-AzVM -Name hello -Location WestUS"
             };
 
-            _telemetryClient.RecordedSuggestion = null;
             _service.Commands = null;
             _service.History = null;
+            _service.ResetRequestPredictionTask();
 
-            _azPredictor.StartEarlyProcessing(history);
+            _azPredictor.StartEarlyProcessing(AzPredictorTests.AzPredictorClient, history);
+            await _service.RequestPredictionTaskCompletionSource.Task;
 
             string maskedCommand = "New-AzVM -Location *** -Name ***";
 
             Assert.Equal(new List<string>() { AzPredictorConstants.CommandPlaceholder, maskedCommand }, _service.Commands);
-            Assert.Equal(maskedCommand, _telemetryClient.RecordedSuggestion.HistoryLine);
             Assert.Equal(history[0], _service.History.ToString());
         }
 
         /// <summary>
         ///  Verify that we can handle the two supported command in sequences.
         /// </summary>
         [Fact]
-        public void VerifyRequestPredictionForTwoSupportedCommandInHistory()
+        public async Task VerifyRequestPredictionForTwoSupportedCommandInHistory()
         {
             IReadOnlyList<string> history = new List<string>()
             {
                 "New-AzResourceGroup -Name 'resourceGroup01'",
                 "New-AzVM -Name:hello -Location:WestUS"
             };
 
-            _telemetryClient.RecordedSuggestion = null;
             _service.Commands = null;
             _service.History = null;
+            _service.ResetRequestPredictionTask();
 
-            _azPredictor.StartEarlyProcessing(history);
+            _azPredictor.StartEarlyProcessing(AzPredictorTests.AzPredictorClient, history);
+            await _service.RequestPredictionTaskCompletionSource.Task;
 
             var maskedCommands = new List<string>()
             {
@@ -121,27 +124,27 @@ public void VerifyRequestPredictionForTwoSupportedCommandInHistory()
             };
 
             Assert.Equal(maskedCommands, _service.Commands);
-            Assert.Equal(maskedCommands[1], _telemetryClient.RecordedSuggestion.HistoryLine);
             Assert.Equal(history[1], _service.History.ToString());
         }
 
         /// <summary>
         ///  Verify that we can handle the two unsupported command in sequences.
         /// </summary>
         [Fact]
-        public void VerifyRequestPredictionForTwoUnsupportedCommandInHistory()
+        public async Task VerifyRequestPredictionForTwoUnsupportedCommandInHistory()
         {
             IReadOnlyList<string> history = new List<string>()
             {
                 "git status",
                 @"$a='ResourceGroup01'",
             };
 
-            _telemetryClient.RecordedSuggestion = null;
             _service.Commands = null;
             _service.History = null;
+            _service.ResetRequestPredictionTask();
 
-            _azPredictor.StartEarlyProcessing(history);
+            _azPredictor.StartEarlyProcessing(AzPredictorTests.AzPredictorClient, history);
+            await _service.RequestPredictionTaskCompletionSource.Task;
 
             var maskedCommands = new List<string>()
             {
@@ -150,33 +153,37 @@ public void VerifyRequestPredictionForTwoUnsupportedCommandInHistory()
             };
 
             Assert.Equal(maskedCommands, _service.Commands);
-            Assert.Equal(maskedCommands[1], _telemetryClient.RecordedSuggestion.HistoryLine);
             Assert.Null(_service.History);
         }
 
         /// <summary>
         /// Verify that we skip the unsupported commands.
         /// </summary>
         [Fact]
-        public void VerifyNotTakeUnsupportedCommands()
+        public async Task VerifyNotTakeUnsupportedCommands()
         {
             var history = new List<string>()
             {
                 "New-AzResourceGroup -Name:resourceGroup01",
                 "New-AzVM -Name hello -Location WestUS"
             };
 
-            _telemetryClient.RecordedSuggestion = null;
             _service.Commands = null;
             _service.History = null;
+            _service.ResetRequestPredictionTask();
 
-            _azPredictor.StartEarlyProcessing(history);
+            _azPredictor.StartEarlyProcessing(AzPredictorTests.AzPredictorClient, history);
+            await _service.RequestPredictionTaskCompletionSource.Task;
 
+            _service.ResetRequestPredictionTask();
             history.Add("git status");
-            _azPredictor.StartEarlyProcessing(history);
+            _azPredictor.StartEarlyProcessing(AzPredictorTests.AzPredictorClient, history);
+            // Don't need to await on _service.RequestPredictionTask, because "git" isn't a supported command and RequestPredictionsAsync isn't called.
 
+            _service.ResetRequestPredictionTask();
             history.Add(@"$a='NewResourceName'");
-            _azPredictor.StartEarlyProcessing(history);
+            _azPredictor.StartEarlyProcessing(AzPredictorTests.AzPredictorClient, history);
+            // Don't need to await on _service.RequestPredictionTask, because assignment isn't supported and RequestPredictionsAsync isn't called.
 
             // We don't take the last two unsupported command to request predictions.
             // But we send the masked one in telemetry.
@@ -188,13 +195,14 @@ public void VerifyNotTakeUnsupportedCommands()
             };
 
             Assert.Equal(maskedCommands, _service.Commands);
-            Assert.Equal(AzPredictorConstants.CommandPlaceholder, _telemetryClient.RecordedSuggestion.HistoryLine);
             Assert.Equal(history[1], _service.History.ToString());
 
             // When there is a new supported command, we'll use that for prediction.
 
+            _service.ResetRequestPredictionTask();
             history.Add("Get-AzResourceGroup -Name ResourceGroup01");
-            _azPredictor.StartEarlyProcessing(history);
+            _azPredictor.StartEarlyProcessing(AzPredictorTests.AzPredictorClient, history);
+            await _service.RequestPredictionTaskCompletionSource.Task;
 
             maskedCommands = new List<string>()
             {
@@ -203,30 +211,32 @@ public void VerifyNotTakeUnsupportedCommands()
             };
 
             Assert.Equal(maskedCommands, _service.Commands);
-            Assert.Equal(maskedCommands[1], _telemetryClient.RecordedSuggestion.HistoryLine);
             Assert.Equal(history.Last(), _service.History.ToString());
         }
 
         /// <summary>
         /// Verify that we handle the three supported command in the same order.
         /// </summary>
         [Fact]
-        public void VerifyThreeSupportedCommands()
+        public async Task VerifyThreeSupportedCommands()
         {
             var history = new List<string>()
             {
                 "New-AzResourceGroup -Name resourceGroup01",
                 "New-AzVM -Name:hello -Location:WestUS"
             };
 
-            _telemetryClient.RecordedSuggestion = null;
             _service.Commands = null;
             _service.History = null;
+            _service.ResetRequestPredictionTask();
 
-            _azPredictor.StartEarlyProcessing(history);
+            _azPredictor.StartEarlyProcessing(AzPredictorTests.AzPredictorClient, history);
+            await _service.RequestPredictionTaskCompletionSource.Task;
 
+            _service.ResetRequestPredictionTask();
             history.Add("Get-AzResourceGroup -Name resourceGroup01");
-            _azPredictor.StartEarlyProcessing(history);
+            _azPredictor.StartEarlyProcessing(AzPredictorTests.AzPredictorClient, history);
+            await _service.RequestPredictionTaskCompletionSource.Task;
 
             var maskedCommands = new List<string>()
             {
@@ -235,27 +245,27 @@ public void VerifyThreeSupportedCommands()
             };
 
             Assert.Equal(maskedCommands, _service.Commands);
-            Assert.Equal(maskedCommands[1], _telemetryClient.RecordedSuggestion.HistoryLine);
             Assert.Equal(history.Last(), _service.History.ToString());
         }
 
         /// <summary>
         /// Verify that we handle the sequence of one unsupported command and one supported command.
         /// </summary>
         [Fact]
-        public void VerifyUnsupportedAndSupportedCommands()
+        public async Task VerifyUnsupportedAndSupportedCommands()
         {
             var history = new List<string>()
             {
                 "git status",
                 "New-AzVM -Name:hello -Location:WestUS"
             };
 
-            _telemetryClient.RecordedSuggestion = null;
             _service.Commands = null;
             _service.History = null;
+            _service.ResetRequestPredictionTask();
 
-            _azPredictor.StartEarlyProcessing(history);
+            _azPredictor.StartEarlyProcessing(AzPredictorTests.AzPredictorClient, history);
+            await _service.RequestPredictionTaskCompletionSource.Task;
 
             var maskedCommands = new List<string>()
             {
@@ -264,27 +274,27 @@ public void VerifyUnsupportedAndSupportedCommands()
             };
 
             Assert.Equal(maskedCommands, _service.Commands);
-            Assert.Equal(maskedCommands[1], _telemetryClient.RecordedSuggestion.HistoryLine);
             Assert.Equal(history.Last(), _service.History.ToString());
         }
 
         /// <summary>
         /// Verify that we handle the sequence of one supported command and one unsupported command.
         /// </summary>
         [Fact]
-        public void VerifySupportedAndUnsupportedCommands()
+        public async Task VerifySupportedAndUnsupportedCommands()
         {
             var history = new List<string>()
             {
                 "New-AzVM -Name hello -Location WestUS",
                 "git status",
             };
 
-            _telemetryClient.RecordedSuggestion = null;
             _service.Commands = null;
             _service.History = null;
+            _service.ResetRequestPredictionTask();
 
-            _azPredictor.StartEarlyProcessing(history);
+            _azPredictor.StartEarlyProcessing(AzPredictorTests.AzPredictorClient, history);
+            await _service.RequestPredictionTaskCompletionSource.Task;
 
             var maskedCommands = new List<string>()
             {
@@ -293,7 +303,6 @@ public void VerifySupportedAndUnsupportedCommands()
             };
 
             Assert.Equal(maskedCommands, _service.Commands);
-            Assert.Equal(AzPredictorConstants.CommandPlaceholder, _telemetryClient.RecordedSuggestion.HistoryLine);
             Assert.Equal(history.First(), _service.History.ToString());
         }
 
@@ -307,10 +316,10 @@ public void VerifySuggestion(string userInput)
         {
             var predictionContext = PredictionContext.Create(userInput);
             var expected = this._service.GetSuggestion(predictionContext, 1, 1, CancellationToken.None);
-            var actual = this._azPredictor.GetSuggestion(predictionContext, CancellationToken.None);
+            var actual = this._azPredictor.GetSuggestion(AzPredictorTests.AzPredictorClient, predictionContext, CancellationToken.None);
 
-            Assert.Equal(expected.Count, actual.Count);
-            Assert.Equal(expected.PredictiveSuggestions.First().SuggestionText, actual.First().SuggestionText);
+            Assert.Equal(expected.Count, actual.SuggestionEntries.Count);
+            Assert.Equal(expected.PredictiveSuggestions.First().SuggestionText, actual.SuggestionEntries.First().SuggestionText);
         }
 
         /// <summary>
@@ -331,9 +340,9 @@ public void VerifySuggestionOnIncompleteCommand()
             var expected = "New-AzResourceGroup -Name 'ResourceGroup01' -Location 'Central US' -WhatIf -Tag value1";
 
             var predictionContext = PredictionContext.Create(userInput);
-            var actual = localAzPredictor.GetSuggestion(predictionContext, CancellationToken.None);
+            var actual = localAzPredictor.GetSuggestion(AzPredictorTests.AzPredictorClient, predictionContext, CancellationToken.None);
 
-            Assert.Equal(expected, actual.First().SuggestionText);
+            Assert.Equal(expected, actual.SuggestionEntries.First().SuggestionText);
         }
 
         /// <summary>
@@ -348,9 +357,9 @@ public void VerifySuggestionOnIncompleteCommand()
         public void VerifyMalFormattedCommandLine(string userInput)
         {
             var predictionContext = PredictionContext.Create(userInput);
-            var actual = _azPredictor.GetSuggestion(predictionContext, CancellationToken.None);
+            var actual = _azPredictor.GetSuggestion(AzPredictorTests.AzPredictorClient, predictionContext, CancellationToken.None);
 
-            Assert.Empty(actual);
+            Assert.Null(actual.SuggestionEntries);
         }
     }
 }

tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/AzPredictorTests.cs

@@ -0,0 +1,39 @@
+// ----------------------------------------------------------------------------------
+//
+// Copyright Microsoft Corporation
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ----------------------------------------------------------------------------------
+
+using System;
+
+namespace Microsoft.Azure.PowerShell.Tools.AzPredictor.Test.Mocks
+{
+    sealed class MockAzContext : IAzContext
+    {
+        public string HashUserId => "TestUserId";
+
+        public string MacAddress => "TestMacAddress";
+
+        public string OSVersion => "TestOSVersion";
+
+        public Version PowerShellVersion => Version.Parse("0.0.0.0");
+
+        public Version ModuleVersion => Version.Parse("0.0.0.0");
+
+        public Version AzVersion => Version.Parse("0.0.0.0");
+
+        public bool IsInternal => true;
+
+        public void UpdateContext()
+        {
+        }
+    }
+}