Merging with upstream

Author: markcowl

.gitignore

@@ -3,6 +3,9 @@ src/Publish/
 src/Package/
 drop/
 
+# external tools
+tools/CLU/7zip/
+
 obj
 TestResults
 *.orig

clu-getstart.md

@@ -1,12 +1,12 @@
-# Work on CLU cmdlets
+# Work on CLU cmdlets
 
 ## Prerequsites
 
 * Visual Studio 2015 RTM with ASP.NET. For details, check out the [installation doc](http://docs.asp.net/en/latest/getting-started/installing-on-windows.html). 
 
 Note, after done, run `dnvm list` command to check the 'coreclr' runtime is installed with right version of `1.0.0-rc1-final`. If not, run `dnvm install 1.0.0-rc1-final -r coreclr -a x64 -p`. Remember always use `-p` flag, so the selection can persist. 
 
-* Get the latest dotnet from "https://azureclu.blob.core.windows.net/tools/dotnet-win-x64.latest.zip", unzip, then add its bin folder to the PATH
+* Get the latest dotnet from "https://azureclu.blob.core.windows.net/tools/dotnet-win-x64.latest.zip", unzip, then add its bin folder to the PATH.  NOTE: With this version of dotnet, the path to dotnet.exe must not contain any spaces.
 
 ## Project Artifacts
 
@@ -25,7 +25,7 @@ CLUPackages require some additional files to direct generation of indexing, and
     | ------------- |:-------------:|
     | CommandAssemblies      | File name of cmdlets assembly(ies) |
     | NounPrefix      | ‘AzureRm’  The part of the cmdlet noun to remove in clu commands|
-    | NounFirst       | if true, the verb comes at the end of the command (e.g. az resource get)|
+    | NounFirst       | if true, the verb comes at the end of the command (e.g. az resource ls)|
 
 * \<modulename\>.nuspec.template, which contains nuspec format metadata about the package – the base temaplate is in tools\clu\Microsoft.Azure.Commands.nuspec.template.  Here are the special fields defined in this template:
    * %PackageId% - replace with the module name (Microsoft.Azure.Commands.\<rp-name\>)
@@ -60,19 +60,26 @@ There is also `<repo-root>\tools\CLU\SetupEnv.bat` which is a windows batch wrap
 
 To test on osx/linux boxes, do #1, open `<repo-root>\drop\clurun`, copy the flavor folder to your target machine, and run the "az.sh" inside. Make sure set execution permission using `chmod +x az.sh clurun`
 
-(All of those are subject to change, contact yugangw or adxsdkdev for any questions)
+### Nightly Builds
+You can also consume tarballs from the nightly build.  The latest drops are available at:
+- ubuntu: [download](https://azurecludrops.blob.core.windows.net/drops/ubuntu.14.04-x64.latest.tar.gz)
+- osx: [download](https://azurecludrops.blob.core.windows.net/drops/osx.10.10-x64.latest.tar.gz)
+- windows: [download](https://azurecludrops.blob.core.windows.net/drops/win7-x64.latest.tar.gz)
+
+
+(All of those are subject to change, contact adxsdkdev for any questions)
 
 ## Quick introductions on cmdlets
-  *  Run commands using the ‘az’ prefix, cmdlet nouns, and cmdlet verbs, for example, `az env get` maps to the cmdlet `Get-AzureRmEnvironment`
-  *  Cmdlet parameters use the double dash (--) so for example, getting a subscription with a particular name would be: `az subscription get –-SubscriptionName “name of subscription"`
+  *  Run commands using the ‘az’ prefix, cmdlet nouns, and cmdlet verbs, for example, `az env ls` maps to the cmdlet `Get-AzureRmEnvironment`
+  *  Cmdlet parameters use the double dash (--) so for example, getting a subscription with a particular name would be: `az subscription ls –-SubscriptionName “name of subscription"`
   * To log in, 3 options
-    * login interactively using device flow, this is the only option for msa account or any org-id with 2fa enforced, example: `az account add`
-    * login with user and password, this works on org-id w/o 2fa enforced, example: `az account add --Username [email protected]  --Password password1`
-    * login as service principal. Example: `az account add --ServicePrincipal --TenantId <tenant> --ApplicationId <id> --Secret <secret>`
+    * login interactively using device flow, this is the only option for msa account or any org-id with 2fa enforced, example: `az login`
+    * login with user and password, this works on org-id w/o 2fa enforced, example: `az login --Username [email protected]  --Password password1`
+    * login as service principal. Example: `az login --ServicePrincipal --TenantId <tenant> --ApplicationId <id> --Secret <secret>`
   * Piping between cmdlets should work the same way that Powerhell piping works
-    ```az subscription get --SubscriptionName | az context set```
+    ```az subscription ls --SubscriptionName | az context set```
   * You can capture piped output using redirection to a file - the result will be the json serialization of the output object.
-    ```az subscription get > subscriptions.json```
+    ```az subscription ls > subscriptions.json```
   * You can use file input to a parameter using '@' notation:
     ```az command --param1 @file1.json```
     Reads input from file1.json and attempts to deserialize the .net object that is the Parameter type for ```param1```

dockerfiles/latest-trusty/Dockerfile

@@ -0,0 +1,19 @@
+FROM ubuntu:14.04
+
+# all in one RUN to produce less, smaller layers
+RUN bash -c "\
+	sudo apt-get -qqy update && \
+	sudo apt-get -qqy upgrade && \
+	sudo apt-get -qqy install wget libicu-dev libunwind8-dev libcurl3 libcurl4-gnutls-dev && \
+	sudo apt-get clean && \
+	sudo rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*"
+
+RUN bash -c "\
+	mkdir /opt/az && \
+	cd /opt/az && \
+	wget https://azurecludrops.blob.core.windows.net/drops/ubuntu.14.04-x64.latest.tar.gz && \
+	tar xvzf ubuntu.14.04-x64.latest.tar.gz"
+
+ENV PATH "$PATH:/opt/az/ubuntu.14.04-x64/"
+
+WORKDIR /opt/az/ubuntu.14.04-x64/

src/CLU/CLUCoreCLR.sln

@@ -66,6 +66,8 @@ Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Sync", "Sync\Sync.xproj", "
 EndProject
 Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Microsoft.Azure.Commands.Network", "Microsoft.Azure.Commands.Network\Microsoft.Azure.Commands.Network.xproj", "{C4DCF4EA-62E7-431E-ADB5-16FD6CFEA5D6}"
 EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Microsoft.CLU.Test", "Microsoft.CLU.Test\Microsoft.CLU.Test.xproj", "{13C34370-51A4-4726-81B8-BE0996FC9CF0}"
+EndProject
 Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "StaticAnalysis", "StaticAnalysis\StaticAnalysis.xproj", "{84F9573A-449E-4159-8493-EFD6C3C8F0A5}"
 EndProject
 Global
@@ -168,6 +170,10 @@ Global
 		{C4DCF4EA-62E7-431E-ADB5-16FD6CFEA5D6}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C4DCF4EA-62E7-431E-ADB5-16FD6CFEA5D6}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{C4DCF4EA-62E7-431E-ADB5-16FD6CFEA5D6}.Release|Any CPU.Build.0 = Release|Any CPU
+		{13C34370-51A4-4726-81B8-BE0996FC9CF0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{13C34370-51A4-4726-81B8-BE0996FC9CF0}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{13C34370-51A4-4726-81B8-BE0996FC9CF0}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{13C34370-51A4-4726-81B8-BE0996FC9CF0}.Release|Any CPU.Build.0 = Release|Any CPU
 		{84F9573A-449E-4159-8493-EFD6C3C8F0A5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{84F9573A-449E-4159-8493-EFD6C3C8F0A5}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{84F9573A-449E-4159-8493-EFD6C3C8F0A5}.Release|Any CPU.ActiveCfg = Release|Any CPU

src/CLU/Commands.Common.Authentication/Models/AzureRMProfile.cs

@@ -71,20 +71,21 @@ public AzureRMProfile(IDataStore dataStore)
         [JsonIgnore]
         public string ProfilePath { get; private set; }
 
+        /// <summary>
+        /// When set to true, collects telemetry information.
+        /// </summary>
+        public bool? IsTelemetryCollectionEnabled { get; set; }
+        
         private void Load(string path)
         {
             this.ProfilePath = path;
             if (_dataStore.FileExists(ProfilePath))
             {
                 string contents = _dataStore.ReadFileAsText(ProfilePath);
-                AzureRMProfile profile = JsonConvert.DeserializeObject<AzureRMProfile>(contents);
-                Debug.Assert(profile != null);
-                this.Context = profile.Context;
-                this.Environments = profile.Environments;
+                JsonConvert.PopulateObject(contents, this);
             }
         }
 
-
         /// <summary>
         /// Writes profile to the disk it was opened from disk.
         /// </summary>

src/CLU/Commands.Common/AzureDataCmdlet.cs

@@ -60,64 +60,11 @@ public AzureRMProfile RMProfile
             get { return GetSessionVariableValue<PSAzureProfile>(AzurePowerShell.ProfileVariable, null); }
         }
 
-        protected override void SaveDataCollectionProfile()
+        protected override bool IsTelemetryCollectionEnabled
         {
-             if (_dataCollectionProfile == null)
-            {
-                InitializeDataCollectionProfile();
-            }
-
-            string fileFullPath = Path.Combine(AzurePowerShell.ProfileDirectory, AzurePSDataCollectionProfile.DefaultFileName);
-            var contents = JsonConvert.SerializeObject(_dataCollectionProfile);
-            if (!DataStore.DirectoryExists(AzurePowerShell.ProfileDirectory))
-            {
-                DataStore.CreateDirectory(AzurePowerShell.ProfileDirectory);
-            }
-            DataStore.WriteFile(fileFullPath, contents);
-            WriteWarning(string.Format(Resources.DataCollectionSaveFileInformation, fileFullPath));
-       }
-
-        protected override void PromptForDataCollectionProfileIfNotExists()
-        {
-            // Initialize it from the environment variable or profile file.
-            InitializeDataCollectionProfile();
-
-            if (!_dataCollectionProfile.EnableAzureDataCollection.HasValue && CheckIfInteractive())
+            get
             {
-                WriteWarning(Resources.DataCollectionPrompt);
-
-                const double timeToWaitInSeconds = 60;
-                var status = string.Format(Resources.DataCollectionConfirmTime, timeToWaitInSeconds);
-                ProgressRecord record = new ProgressRecord(0, Resources.DataCollectionActivity, status);
-
-                var startTime = DateTime.Now;
-                var endTime = DateTime.Now;
-                //double elapsedSeconds = 0;
-
-                //while (!this.Host.UI.RawUI.KeyAvailable && elapsedSeconds < timeToWaitInSeconds)
-                //{
-                //    Thread.Sleep(TimeSpan.FromMilliseconds(10));
-                //    endTime = DateTime.Now;
-
-                //    elapsedSeconds = (endTime - startTime).TotalSeconds;
-                //    record.PercentComplete = ((int) elapsedSeconds*100/(int) timeToWaitInSeconds);
-                //    WriteProgress(record);
-                //}
-
-                bool enabled = false;
-                //if (this.Host.UI.RawUI.KeyAvailable)
-                //{
-                //    KeyInfo keyInfo =
-                //        this.Host.UI.RawUI.ReadKey(ReadKeyOptions.NoEcho | ReadKeyOptions.AllowCtrlC |
-                //                                   ReadKeyOptions.IncludeKeyDown);
-                //    enabled = (keyInfo.Character == 'Y' || keyInfo.Character == 'y');
-                //}
-
-                _dataCollectionProfile.EnableAzureDataCollection = enabled;
-
-                WriteWarning(enabled ? Resources.DataCollectionConfirmYes : Resources.DataCollectionConfirmNo);
-
-                SaveDataCollectionProfile();
+                return false;
             }
         }
 

src/CLU/Commands.Common/AzurePSCmdlet.cs

@@ -41,10 +41,7 @@ namespace Microsoft.Azure.Commands.Utilities.Common
     public abstract class AzurePSCmdlet : PSCmdlet, IDisposable
     {
         protected readonly ConcurrentQueue<string> _debugMessages;
-
-
         private DebugStreamTraceListener _adalListener;
-        protected static AzurePSDataCollectionProfile _dataCollectionProfile = null;
         protected static string _errorRecordFolderPath = null;
         protected const string _fileTimeStampSuffixFormat = "yyyy-MM-dd-THH-mm-ss-fff";
         protected string _clientRequestId = Guid.NewGuid().ToString();
@@ -56,7 +53,8 @@ public abstract class AzurePSCmdlet : PSCmdlet, IDisposable
 
         protected AzurePSQoSEvent QosEvent;
 
-        protected virtual bool IsUsageMetricEnabled {
+        protected virtual bool IsUsageMetricEnabled
+        {
             get { return true; }
         }
 
@@ -67,9 +65,9 @@ protected virtual bool IsErrorMetricEnabled
 
         /// <summary>
         /// Gets the PowerShell module name used for user agent header.
-        /// By default uses "Azurepowershell"
+        /// By default uses "AzureCLU"
         /// </summary>
-        protected virtual string ModuleName { get { return "AzurePowershell"; } }
+        protected virtual string ModuleName { get { return "AzureCLU"; } }
 
         /// <summary>
         /// Gets PowerShell module version used for user agent header.
@@ -158,128 +156,21 @@ protected virtual void SetSessionVariable<T>(string name, T value) where T : cla
                 DataStore.WriteFile(variablePath, JsonConvert.SerializeObject(value));
             }
         }
-        /// <summary>
-        /// Initialize the data collection profile
-        /// </summary>
-        protected static void InitializeDataCollectionProfile()
-        {
-            if (_dataCollectionProfile != null && _dataCollectionProfile.EnableAzureDataCollection.HasValue)
-            {
-                return;
-            }
-
-            // Get the value of the environment variable for Azure PS data collection setting.
-            string value = Environment.GetEnvironmentVariable(AzurePSDataCollectionProfile.EnvironmentVariableName);
-            if (!string.IsNullOrWhiteSpace(value))
-            {
-                if (string.Equals(value, bool.FalseString, StringComparison.OrdinalIgnoreCase))
-                {
-                    // Disable data collection only if it is explicitly set to 'false'.
-                    _dataCollectionProfile = new AzurePSDataCollectionProfile(true);
-                }
-                else if (string.Equals(value, bool.TrueString, StringComparison.OrdinalIgnoreCase))
-                {
-                    // Enable data collection only if it is explicitly set to 'true'.
-                    _dataCollectionProfile = new AzurePSDataCollectionProfile(false);
-                }
-            }
-
-            // If the environment value is null or empty, or not correctly set, try to read the setting from default file location.
-            if (_dataCollectionProfile == null)
-            {
-                string fileFullPath = Path.Combine(AzurePowerShell.ProfileDirectory, AzurePSDataCollectionProfile.DefaultFileName);
-                if (File.Exists(fileFullPath))
-                {
-                    string contents = File.ReadAllText(fileFullPath);
-                    _dataCollectionProfile = JsonConvert.DeserializeObject<AzurePSDataCollectionProfile>(contents);
-                }
-            }
-
-            // If the environment variable or file content is not set, create a new profile object.
-            if (_dataCollectionProfile == null)
-            {
-                _dataCollectionProfile = new AzurePSDataCollectionProfile();
-            }
-        }
-
-        /// <summary>
-        /// Get the data collection profile
-        /// </summary>
-        protected static AzurePSDataCollectionProfile GetDataCollectionProfile()
-        {
-            if (_dataCollectionProfile == null)
-            {
-                InitializeDataCollectionProfile();
-            }
-
-            return _dataCollectionProfile;
-        }
 
         /// <summary>
         /// Check whether the data collection is opted in from user
         /// </summary>
         /// <returns>true if allowed</returns>
-        public static bool IsDataCollectionAllowed()
+        protected abstract bool IsTelemetryCollectionEnabled
         {
-            //TODO: CLU - remove before final release
-            return true;
-
-            //if (_dataCollectionProfile != null &&
-            //    _dataCollectionProfile.EnableAzureDataCollection.HasValue &&
-            //    _dataCollectionProfile.EnableAzureDataCollection.Value)
-            //{
-            //    return true;
-            //}
-
-            //return false;
+            get;
         }
 
-        /// <summary>
-        /// Save the current data collection profile Json data into the default file path
-        /// </summary>
-        protected abstract void SaveDataCollectionProfile();
-
-        protected bool CheckIfInteractive()
-        {
-            bool interactive = false;
-            //if (this.Host == null || 
-            //    this.Host.UI == null || 
-            //    this.Host.UI.RawUI == null ||
-            //    Environment.GetCommandLineArgs().Any(s => s.Equals("-NonInteractive", StringComparison.OrdinalIgnoreCase)))
-            //{
-            //    interactive = false;
-            //}
-            //else
-            //{
-            //    try
-            //    {
-            //        var test = this.Host.UI.RawUI.KeyAvailable;
-            //    }
-            //    catch
-            //    {
-            //        interactive = false;
-            //    }
-            //}
-
-            if (!interactive && !_dataCollectionProfile.EnableAzureDataCollection.HasValue)
-            {
-                _dataCollectionProfile.EnableAzureDataCollection = false;
-            }
-            return interactive;
-        }
-
-        /// <summary>
-        /// Prompt for the current data collection profile
-        /// </summary>
-        /// <param name="profile"></param>
-        protected abstract void PromptForDataCollectionProfileIfNotExists();
-
         /// <summary>
         /// Cmdlet begin process. Write to logs, setup Http Tracing and initialize profile
         /// </summary>
         protected override void BeginProcessing()
         {
-            PromptForDataCollectionProfileIfNotExists();
             InitializeQosEvent();
             if (string.IsNullOrEmpty(ParameterSetName))
             {
@@ -352,7 +243,7 @@ protected bool IsVerbose()
 
         protected new void WriteError(ErrorRecord errorRecord)
         {
-            FlushDebugMessages(IsDataCollectionAllowed());
+            FlushDebugMessages(IsTelemetryCollectionEnabled);
             if (QosEvent != null && errorRecord != null)
             {
                 QosEvent.Exception = errorRecord.Exception;
@@ -524,7 +415,7 @@ protected void LogQosEvent()
                 return;
             }
 
-            if (!IsDataCollectionAllowed())
+            if (!IsTelemetryCollectionEnabled)
             {
                 return;
             }
@@ -533,8 +424,10 @@ protected void LogQosEvent()
 
             try
             {
-                MetricHelper.LogQoSEvent(QosEvent, IsUsageMetricEnabled, IsErrorMetricEnabled);
-                MetricHelper.FlushMetric();
+                MetricHelper.LogQoSEvent(QosEvent, 
+                    IsUsageMetricEnabled && IsTelemetryCollectionEnabled, 
+                    IsErrorMetricEnabled && IsTelemetryCollectionEnabled);
+                MetricHelper.FlushMetric(IsTelemetryCollectionEnabled);
                 WriteDebug("Finish sending metric.");
             }
             catch (Exception e)