Merge pull request Azure#9364 from arolshan/OperationalInsights-DataSources-FixCustomLogModel

markcowl · web-flow · commit 146c409ad8d9 · 2019-06-24T12:48:40.000-07:00
Fix DateTimeExtraction model - Bug Fix
diff --git a/src/OperationalInsights/OperationalInsights.Test/ScenarioTests/DataSourceTests.ps1 b/src/OperationalInsights/OperationalInsights.Test/ScenarioTests/DataSourceTests.ps1
@@ -132,10 +132,14 @@ function Test-CreateAllKindsOfDataSource
     # linux performance data source
     $linuxPerfDataSource = New-AzOperationalInsightsLinuxPerformanceObjectDataSource -Workspace $workspace -Name "MemoryLinux" -ObjectName "Memory" -InstanceName * -CounterNames "Available bytes" -IntervalSeconds 10
 
-    # customlog
-    $customLogRawJson = '{"customLogName":"Validation_CL","description":"test","inputs":[{"location":{"fileSystemLocations":{"linuxFileTypeLogPaths":null,"windowsFileTypeLogPaths":["C:\\e2e\\Evan\\ArubaSECURITY\\*.log"]}},"recordDelimiter":{"regexDelimiter":{"pattern":"\\n","matchIndex":0}}}],"extractions":[{"extractionName":"TimeGenerated","extractionType":"DateTime","extractionProperties":{"dateTimeExtraction":{"regex":null,"joinStringRegex":null}}}]}'
+    # customlog, regex string
+    $customLogRawJson = '{"customLogName":"Validation_CL","description":"test","inputs":[{"location":{"fileSystemLocations":{"linuxFileTypeLogPaths":null,"windowsFileTypeLogPaths":["C:\\e2e\\Evan\\ArubaSECURITY\\*.log"]}},"recordDelimiter":{"regexDelimiter":{"pattern":"\\n","matchIndex":0}}}],"extractions":[{"extractionName":"TimeGenerated","extractionType":"DateTime","extractionProperties":{"dateTimeExtraction":{"regex":"((\\d{2})|(\\d{4}))-([0-1]\\d)-(([0-3]\\d)|(\\d))\\s((\\d)|([0-1]\\d)|(2[0-4])):[0-5][0-9]:[0-5][0-9]","joinStringRegex":null}}}]}'
     $customLogDataSource = New-AzOperationalInsightsCustomLogDataSource -Workspace $workspace -CustomLogRawJson $customLogRawJson -Name "MyCustomLog"
 	
+    # customlog
+    $customLogRawJson2 = '{"customLogName":"Validation_CL2","description":"test","inputs":[{"location":{"fileSystemLocations":{"linuxFileTypeLogPaths":null,"windowsFileTypeLogPaths":["C:\\e2e\\Evan\\ArubaSECURITY\\*.log"]}},"recordDelimiter":{"regexDelimiter":{"pattern":"\\n","matchIndex":0}}}],"extractions":[{"extractionName":"TimeGenerated","extractionType":"DateTime","extractionProperties":{"dateTimeExtraction":{"regex":[{"matchIndex":0,"numberdGroup":null,"pattern":"((\\d{2})|(\\d{4}))-([0-1]\\d)-(([0-3]\\d)|(\\d))\\s((\\d)|([0-1]\\d)|(2[0-4])):[0-5][0-9]:[0-5][0-9]"}],"joinStringRegex":null}}}]}'
+    $customLogDataSource2 = New-AzOperationalInsightsCustomLogDataSource -Workspace $workspace -CustomLogRawJson $customLogRawJson2 -Name "MyCustomLog2"
+
     # ApplicationInsights data source
     $applicationInsightsDataSource1 = New-AzOperationalInsightsApplicationInsightsDataSource -Workspace $workspace -ApplicationSubscriptionId $subId1 -ApplicationResourceGroupName $rgname -ApplicationName "ai-app"
     Assert-NotNull $applicationInsightsDataSource1
@@ -183,4 +187,4 @@ function Test-ToggleSingletonDataSourceState
     Enable-AzOperationalInsightsLinuxSyslogCollection -Workspace $workspace
     Disable-AzOperationalInsightsLinuxSyslogCollection -Workspace $workspace
 
-}
+}
diff --git a/src/OperationalInsights/OperationalInsights.Test/SessionRecords/Microsoft.Azure.Commands.OperationalInsights.Test.DataSourceTests/TestCreateAllKindsOfDataSource.json b/src/OperationalInsights/OperationalInsights.Test/SessionRecords/Microsoft.Azure.Commands.OperationalInsights.Test.DataSourceTests/TestCreateAllKindsOfDataSource.json
@@ -1066,6 +1066,144 @@
       },
       "StatusCode": 200
     },
+    {
+      "RequestUri": "/subscriptions/a6383be3-f0e8-4968-93d5-10f2625f5bb5/resourcegroups/onesdk9967/providers/Microsoft.OperationalInsights/workspaces/AzTestonesdk5206/dataSources/MyCustomLog2?api-version=2015-11-01-preview",
+      "EncodedRequestUri": "L3N1YnNjcmlwdGlvbnMvYTYzODNiZTMtZjBlOC00OTY4LTkzZDUtMTBmMjYyNWY1YmI1L3Jlc291cmNlZ3JvdXBzL29uZXNkazk5NjcvcHJvdmlkZXJzL01pY3Jvc29mdC5PcGVyYXRpb25hbEluc2lnaHRzL3dvcmtzcGFjZXMvQXpUZXN0b25lc2RrNTIwNi9kYXRhU291cmNlcy9NeUN1c3RvbUxvZzI/YXBpLXZlcnNpb249MjAxNS0xMS0wMS1wcmV2aWV3",
+      "RequestMethod": "GET",
+      "RequestBody": "",
+      "RequestHeaders": {
+        "x-ms-client-request-id": [
+          "02f6042b-0d04-4d64-a7ae-ef13512cabe3"
+        ],
+        "accept-language": [
+          "en-US"
+        ],
+        "User-Agent": [
+          "FxVersion/4.7.2046.0",
+          "OSName/Windows_10_Enterprise",
+          "OSVersion/6.3.15063",
+          "Microsoft.Azure.Management.OperationalInsights.OperationalInsightsManagementClient/0.18.0-preview"
+        ]
+      },
+      "ResponseBody": "{\r\n  \"error\": {\r\n    \"code\": \"DataSourceNotFound\",\r\n    \"message\": \"DataSource 'MyCustomLog2' could not be found.\"\r\n  }\r\n}",
+      "ResponseHeaders": {
+        "Content-Length": [
+          "96"
+        ],
+        "Content-Type": [
+          "application/json; charset=utf-8"
+        ],
+        "Expires": [
+          "-1"
+        ],
+        "Pragma": [
+          "no-cache"
+        ],
+        "Strict-Transport-Security": [
+          "max-age=31536000; includeSubDomains"
+        ],
+        "X-Content-Type-Options": [
+          "nosniff"
+        ],
+        "x-ms-ratelimit-remaining-subscription-reads": [
+          "14823"
+        ],
+        "x-ms-request-id": [
+          "c94b019e-bd8a-4546-b433-ddb5aa7635ef"
+        ],
+        "x-ms-correlation-request-id": [
+          "c94b019e-bd8a-4546-b433-ddb5aa7635ef"
+        ],
+        "x-ms-routing-request-id": [
+          "WESTUS2:20170504T002631Z:c94b019e-bd8a-4546-b433-ddb5aa7635ef"
+        ],
+        "Cache-Control": [
+          "no-cache"
+        ],
+        "Date": [
+          "Thu, 04 May 2017 00:26:30 GMT"
+        ],
+        "Server": [
+          "Microsoft-IIS/8.5"
+        ],
+        "X-Powered-By": [
+          "ASP.NET"
+        ]
+      },
+      "StatusCode": 404
+    },
+    {
+      "RequestUri": "/subscriptions/a6383be3-f0e8-4968-93d5-10f2625f5bb5/resourcegroups/onesdk9967/providers/Microsoft.OperationalInsights/workspaces/AzTestonesdk5206/dataSources/MyCustomLog2?api-version=2015-11-01-preview",
+      "EncodedRequestUri": "L3N1YnNjcmlwdGlvbnMvYTYzODNiZTMtZjBlOC00OTY4LTkzZDUtMTBmMjYyNWY1YmI1L3Jlc291cmNlZ3JvdXBzL29uZXNkazk5NjcvcHJvdmlkZXJzL01pY3Jvc29mdC5PcGVyYXRpb25hbEluc2lnaHRzL3dvcmtzcGFjZXMvQXpUZXN0b25lc2RrNTIwNi9kYXRhU291cmNlcy9NeUN1c3RvbUxvZzI/YXBpLXZlcnNpb249MjAxNS0xMS0wMS1wcmV2aWV3",
+      "RequestMethod": "PUT",
+      "RequestBody": "{\r\n  \"properties\": {\r\n    \"customLogName\": \"Validation_CL2\",\r\n    \"description\": \"test\",\r\n    \"extractions\": [\r\n      {\r\n        \"extractionName\": \"TimeGenerated\",\r\n        \"extractionProperties\": {\r\n          \"dateTimeExtraction\": {}\r\n        },\r\n        \"extractionType\": \"DateTime\"\r\n      }\r\n    ],\r\n    \"inputs\": [\r\n      {\r\n        \"location\": {\r\n          \"fileSystemLocations\": {\r\n            \"windowsFileTypeLogPaths\": [\r\n              \"C:\\\\e2e\\\\Evan\\\\ArubaSECURITY\\\\*.log\"\r\n            ]\r\n          }\r\n        },\r\n        \"recordDelimiter\": {\r\n          \"regexDelimiter\": {\r\n            \"matchIndex\": 0,\r\n            \"pattern\": \"\\\\n\"\r\n          }\r\n        }\r\n      }\r\n    ]\r\n  },\r\n  \"kind\": \"CustomLog\"\r\n}",
+      "RequestHeaders": {
+        "Content-Type": [
+          "application/json; charset=utf-8"
+        ],
+        "Content-Length": [
+          "713"
+        ],
+        "x-ms-client-request-id": [
+          "3a712d5e-832f-4dd2-8cfe-b39dc4f3b80b"
+        ],
+        "accept-language": [
+          "en-US"
+        ],
+        "User-Agent": [
+          "FxVersion/4.7.2046.0",
+          "OSName/Windows_10_Enterprise",
+          "OSVersion/6.3.15063",
+          "Microsoft.Azure.Management.OperationalInsights.OperationalInsightsManagementClient/0.18.0-preview"
+        ]
+      },
+      "ResponseBody": "{\r\n  \"kind\": \"CustomLog\",\r\n  \"properties\": {\r\n    \"customLogName\": \"Validation_CL2\",\r\n    \"description\": \"test\",\r\n    \"extractions\": [\r\n      {\r\n        \"extractionName\": \"TimeGenerated\",\r\n        \"extractionProperties\": {\r\n          \"dateTimeExtraction\": {\"regex\":[{\"matchIndex\":0,\"numberdGroup\":null,\"pattern\":\"((\\\\d{2})|(\\\\d{4}))-([0-1]\\\\d)-(([0-3]\\\\d)|(\\\\d))\\\\s((\\\\d)|([0-1]\\\\d)|(2[0-4])):[0-5][0-9]:[0-5][0-9]\"}]}\r\n        },\r\n        \"extractionType\": \"DateTime\"\r\n      }\r\n    ],\r\n    \"inputs\": [\r\n      {\r\n        \"location\": {\r\n          \"fileSystemLocations\": {\r\n            \"linuxFileTypeLogPaths\": null,\r\n            \"windowsFileTypeLogPaths\": [\r\n              \"C:\\\\e2e\\\\Evan\\\\ArubaSECURITY\\\\*.log\"\r\n            ]\r\n          }\r\n        },\r\n        \"recordDelimiter\": {\r\n          \"regexDelimiter\": {\r\n            \"matchIndex\": 0,\r\n            \"numberdGroup\": null,\r\n            \"pattern\": \"\\\\n\"\r\n          }\r\n        }\r\n      }\r\n    ]\r\n  },\r\n  \"id\": \"/subscriptions/a6383be3-f0e8-4968-93d5-10f2625f5bb5/resourceGroups/onesdk9967/providers/Microsoft.OperationalInsights/workspaces/AzTestonesdk5206/datasources/MyCustomLog2\",\r\n  \"etag\": \"W/\\\"datetime'2017-05-04T00%3A26%3A31.2865595Z'\\\"\",\r\n  \"name\": \"MyCustomLog\",\r\n  \"type\": \"Microsoft.OperationalInsights/workspaces/datasources\"\r\n}",
+      "ResponseHeaders": {
+        "Content-Length": [
+          "769"
+        ],
+        "Content-Type": [
+          "application/json; charset=utf-8"
+        ],
+        "Expires": [
+          "-1"
+        ],
+        "Pragma": [
+          "no-cache"
+        ],
+        "Strict-Transport-Security": [
+          "max-age=31536000; includeSubDomains"
+        ],
+        "X-Content-Type-Options": [
+          "nosniff"
+        ],
+        "x-ms-ratelimit-remaining-subscription-writes": [
+          "1193"
+        ],
+        "x-ms-request-id": [
+          "f84325f6-3ea3-4b44-89d8-1fc05971adec"
+        ],
+        "x-ms-correlation-request-id": [
+          "f84325f6-3ea3-4b44-89d8-1fc05971adec"
+        ],
+        "x-ms-routing-request-id": [
+          "WESTUS2:20170504T002713Z:f84325f6-3ea3-4b44-89d8-1fc05971adec"
+        ],
+        "Cache-Control": [
+          "no-cache"
+        ],
+        "Date": [
+          "Thu, 04 May 2017 00:27:12 GMT"
+        ],
+        "Server": [
+          "Microsoft-IIS/8.5"
+        ],
+        "X-Powered-By": [
+          "ASP.NET"
+        ]
+      },
+      "StatusCode": 200
+    },
     {
       "RequestUri": "/subscriptions/a6383be3-f0e8-4968-93d5-10f2625f5bb5/resourcegroups/onesdk9967/providers/Microsoft.OperationalInsights/workspaces/AzTestonesdk5206/dataSources/ApplicationInsights?api-version=2015-11-01-preview",
       "EncodedRequestUri": "L3N1YnNjcmlwdGlvbnMvYTYzODNiZTMtZjBlOC00OTY4LTkzZDUtMTBmMjYyNWY1YmI1L3Jlc291cmNlZ3JvdXBzL29uZXNkazk5NjcvcHJvdmlkZXJzL01pY3Jvc29mdC5PcGVyYXRpb25hbEluc2lnaHRzL3dvcmtzcGFjZXMvQXpUZXN0b25lc2RrNTIwNi9kYXRhU291cmNlcy9BcHBsaWNhdGlvbkluc2lnaHRzP2FwaS12ZXJzaW9uPTIwMTUtMTEtMDEtcHJldmlldw==",
diff --git a/src/OperationalInsights/OperationalInsights/ChangeLog.md b/src/OperationalInsights/OperationalInsights/ChangeLog.md
@@ -18,6 +18,7 @@
         - Additional information about change #1
 -->
 ## Upcoming Release
+* Fixed CustomLog datasource model returned in Get-AzOperationalInsightsDataSource
 
 ## Version 1.3.0
 * Enable **pergb2018** pricing tier in `New-AzureRmOperationalInsightsWorkspace` command
diff --git a/src/OperationalInsights/OperationalInsights/DataSources/NewDataSourceCmdletsPerKind/NewAzureOperationalInsightsCustomLogDataSourceCommand.cs b/src/OperationalInsights/OperationalInsights/DataSources/NewDataSourceCmdletsPerKind/NewAzureOperationalInsightsCustomLogDataSourceCommand.cs
@@ -20,6 +20,7 @@
 using System;
 using System.Linq;
 using System.Management.Automation;
+using Microsoft.WindowsAzure.Commands.Common.CustomAttributes;
 
 namespace Microsoft.Azure.Commands.OperationalInsights
 {
diff --git a/src/OperationalInsights/OperationalInsights/Models/DataSourcePropertiesPerKind/PSCustomLogDataSourceProperties.cs b/src/OperationalInsights/OperationalInsights/Models/DataSourcePropertiesPerKind/PSCustomLogDataSourceProperties.cs
@@ -19,6 +19,8 @@
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
+using Microsoft.WindowsAzure.Commands.Common.CustomAttributes;
+using Newtonsoft.Json.Linq;
 
 namespace Microsoft.Azure.Commands.OperationalInsights.Models
 {
@@ -63,7 +65,8 @@ public class Location
         /// <summary>
         /// Gets or sets the file system locations.
         /// </summary>
-        public FileSystemLocations fileSystemLocations { get; set; }
+        [JsonProperty(PropertyName = "fileSystemLocations")]
+        public FileSystemLocations FileSystemLocations { get; set; }
 
         #endregion
     }
@@ -78,12 +81,14 @@ public class FileSystemLocations
         /// <summary>
         /// Gets or sets the linux file type log paths.
         /// </summary>
-        public string[] linuxFileTypeLogPaths { get; set; }
+        [JsonProperty(PropertyName = "linuxFileTypeLogPaths")]
+        public string[] LinuxFileTypeLogPaths { get; set; }
 
         /// <summary>
         /// Gets or sets the windows file type log paths.
         /// </summary>
-        public string[] windowsFileTypeLogPaths { get; set; }
+        [JsonProperty(PropertyName = "windowsFileTypeLogPaths")]
+        public string[] WindowsFileTypeLogPaths { get; set; }
 
         #endregion
     }
@@ -98,7 +103,8 @@ public class RecordDelimiter
         /// <summary>
         /// Gets or sets the regex delimiter.
         /// </summary>
-        public RegexDelimiter regexDelimiter { get; set; }
+        [JsonProperty(PropertyName = "regexDelimiter")]
+        public RegexDelimiter RegexDelimiter { get; set; }
 
         #endregion
     }
@@ -113,17 +119,20 @@ public class RegexDelimiter
         /// <summary>
         /// Gets or sets the match index.
         /// </summary>
-        public int matchIndex { get; set; }
+        [JsonProperty(PropertyName = "matchIndex")]
+        public int MatchIndex { get; set; }
 
         /// <summary>
         /// Gets or sets the numberd group.
         /// </summary>
-        public string numberdGroup { get; set; }
+        [JsonProperty(PropertyName = "numberdGroup")]
+        public string NumberdGroup { get; set; }
 
         /// <summary>
         /// Gets or sets the pattern.
         /// </summary>
-        public string pattern { get; set; }
+        [JsonProperty(PropertyName = "pattern")]
+        public string Pattern { get; set; }
 
         #endregion
     }
@@ -138,12 +147,14 @@ public class CustomLogInput
         /// <summary>
         /// Gets or sets the location.
         /// </summary>
-        public Location location { get; set; }
+        [JsonProperty(PropertyName = "location")]
+        public Location Location { get; set; }
 
         /// <summary>
         /// Gets or sets the record delimiter.
         /// </summary>
-        public RecordDelimiter recordDelimiter { get; set; }
+        [JsonProperty(PropertyName = "recordDelimiter")]
+        public RecordDelimiter RecordDelimiter { get; set; }
 
         #endregion
     }
@@ -158,12 +169,28 @@ public class DateTimeExtraction
         /// <summary>
         /// Gets or sets the join string regex.
         /// </summary>
-        public string joinStringRegex { get; set; }
+        [JsonProperty("joinStringRegex")]
+        public string JoinStringRegex { get; set; }
+
+        /// <summary>
+        /// Gets or sets the regex string.
+        /// </summary>
+        [JsonIgnore]
+        public string Regex { get; set; }
 
         /// <summary>
         /// Gets or sets the regex.
         /// </summary>
-        public string regex { get; set; }
+        [CmdletParameterBreakingChange(nameof(Regex), OldParamaterType = typeof(string), NewParameterTypeName = nameof(RegexDelimiter))]
+        [JsonConverter(typeof(RegexDelimiterJsonConverter))]
+        [JsonProperty("regex")]
+        public RegexDelimiter[] RegexDelimiters { get; set; }
+
+        ///<summary>
+        /// Gets or sets the FormatString
+        /// </summary>
+        [JsonProperty("formatString")]
+        public string FormatString { get; set; }
 
         #endregion
     }
@@ -178,7 +205,8 @@ public class ExtractionProperties
         /// <summary>
         /// Gets or sets the date time extraction.
         /// </summary>
-        public DateTimeExtraction dateTimeExtraction { get; set; }
+        [JsonProperty(PropertyName = "dateTimeExtraction")]
+        public DateTimeExtraction DateTimeExtraction { get; set; }
 
         #endregion
     }
@@ -193,18 +221,44 @@ public class CustomLogExtraction
         /// <summary>
         /// Gets or sets the extraction name.
         /// </summary>
-        public string extractionName { get; set; }
+        [JsonProperty(PropertyName = "extractionName")]
+        public string ExtractionName { get; set; }
 
         /// <summary>
         /// Gets or sets the extraction properties.
         /// </summary>
-        public ExtractionProperties extractionProperties { get; set; }
+        [JsonProperty(PropertyName = "extractionProperties")]
+        public ExtractionProperties ExtractionProperties { get; set; }
 
         /// <summary>
         /// Gets or sets the extraction type.
         /// </summary>
-        public string extractionType { get; set; }
+        [JsonProperty(PropertyName = "extractionType")]
+        public string ExtractionType { get; set; }
 
         #endregion
     }
+
+    // This converter allows backward compatibility
+    public class RegexDelimiterJsonConverter : JsonConverter
+    {
+        public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+        {
+            switch (reader.TokenType)
+            {
+                case JsonToken.StartArray:
+                    JToken token = JToken.Load(reader);
+                    return token.ToObject<RegexDelimiter[]>();
+                case JsonToken.String:
+                    // Satisfy case in which user uses the old regex property in input
+                    return new[] {new RegexDelimiter {MatchIndex = 0, Pattern = (string) reader.Value}};
+                default:
+                    throw new JsonSerializationException();
+            }
+        }
+
+        public override bool CanWrite => false;
+        public override bool CanConvert(Type objectType) => throw new NotImplementedException();
+        public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) => throw new NotImplementedException();
+    }
 }

Original file line number	Diff line number	Diff line change
`@@ -20,6 +20,7 @@`
`20`	`20`	`using System;`
`21`	`21`	`using System.Linq;`
`22`	`22`	`using System.Management.Automation;`
	`23`	`+using Microsoft.WindowsAzure.Commands.Common.CustomAttributes;`
`23`	`24`
`24`	`25`	`namespace Microsoft.Azure.Commands.OperationalInsights`
`25`	`26`	`{`