Integrating the new custom dataplane sdk for ADLS to the powershell cmdlets

src/ResourceManager/Common/Commands.ScenarioTests.ResourceManager.Common/Assert.ps1

@@ -277,7 +277,28 @@ function Assert-AreEqual
 
   return $true
 }
-
+###################
+#
+# Verify that if actual falls in an acceptable range of expected
+#
+#    param [long] $expected : The expected number
+#    param [long] $actual   : The actual number
+#	 param [long] $interval : The acceptable offset either side of the expected
+#    param [string] $message  : The message to return if the given objects are not equal
+####################
+function Assert-NumAreInRange
+{
+	param([long] $expected, [long] $actual, [long] $interval, [string] $message)
+	if (!$message)
+	{
+		$message = "Assertion failed because expected '$expected' does not fall in accepted range of 'interval' of actual '$actual'"
+	}
+	if(!($actual -ge ($expected-$interval) -and $actual -le ($expected+$interval)))
+	{
+        throw $message
+    }
+	return $true
+}
 ###################
 #
 # Verify that two given arrays are equal

src/ResourceManager/DataLakeStore/ChangeLog.md

@@ -19,6 +19,42 @@
 -->
 ## Current Release
 
+* Moved from AutoRest sdk to our version of Dataplane sdk for Azure data lake.
+* Refactored following cmdlets to use the new version of SDK:
+	* Add-AzureRmDataLakeStoreItemContent
+	* Export-AzureRmDataLakeStoreItem
+	* Get-AzureRmDataLakeStoreChildItem
+	* Get-AzureRmDataLakeStoreItem
+	* Get-AzureRmDataLakeStoreItemAclEntry
+	* Get-AzureRmDataLakeStoreItemContent
+	* Get-AzureRmDataLakeStoreItemOwner
+	* Get-AzureRmDataLakeStoreItemPermission
+	* Import-AzureRmDataLakeStoreItem
+	* Join-AzureRmDataLakeStoreItem
+	* Move-AzureRmDataLakeStoreItem
+	* New-AzureRmDataLakeStoreItem
+	* Remove-AzureRmDataLakeStoreItem
+	* Remove-AzureRmDataLakeStoreItemAcl
+	* Remove-AzureRmDataLakeStoreItemAclEntry
+	* Set-AzureRmDataLakeStoreItemAcl
+	* Set-AzureRmDataLakeStoreItemAclEntry
+	* Set-AzureRmDataLakeStoreItemExpiry
+	* Set-AzureRmDataLakeStoreItemOwner
+	* Set-AzureRmDataLakeStoreItemPermission
+	* Test-AzureRmDataLakeStoreItem
+* Refactored following models to use models from new SDK:
+	* DataLakeStoreEnums
+	* DataLakeStoreFileSystemClient
+	* DataLakeStoreItem
+	* DataLakeStoreItemAce
+	* DataLakeStorePathInstance
+* Moved the commands and models related to the dataplane into separate folders
+* Export-AzureRmDataLakeStoreItem (documented in the breaking changes) - Deprecated parameters PerFileThreadCount, ConcurrentFileCount and introduced parameter Concurrency
+* Import-AzureRMDataLakeStoreItem (documented in the breaking changes) -Deprecated parametersPerFileThreadCount, ConcurrentFileCount and introduced parameter Concurrency
+* Get-AzureRMDataLakeStoreItemContent - Fixed the tail behavior for contents greater than 4MB
+* Set-AzureRMDataLakeStoreItemExpiry - Introduced new parameter set for setting relative expiration time
+* Remove-AzureRmDataLakeStoreItem (documented in the breaking changes) - Deprecated parameter Clean.
+
 ## Version 5.0.0
 * NOTE: This is a breaking change release. Please see the migration guide (https://aka.ms/azps-migration-guide) for a full list of breaking changes introduced.
 * Removed the Obsolete Properties fields in PSDataLakeStoreAccount.cs and its assoicated files

src/ResourceManager/DataLakeStore/Commands.DataLakeStore.Test/Commands.DataLakeStore.Test.csproj

@@ -196,6 +196,10 @@
       <Project>{5ee72c53-1720-4309-b54b-5fb79703195f}</Project>
       <Name>Commands.Common</Name>
     </ProjectReference>
+    <ProjectReference Include="..\Commands.DataLakeStore\Commands.DataLakeStore.csproj">
+      <Project>{8aab43e6-e8f6-4f91-af8a-6a63cd78ef1e}</Project>
+      <Name>Commands.DataLakeStore</Name>
+    </ProjectReference>
   </ItemGroup>
   <ItemGroup>
     <None Include="MSSharedLibKey.snk" />

src/ResourceManager/DataLakeStore/Commands.DataLakeStore.Test/ScenarioTests/AdlsAliasTests.cs

@@ -12,6 +12,8 @@
 // limitations under the License.
 // ----------------------------------------------------------------------------------
 
+using System;
+
 namespace Microsoft.Azure.Commands.DataLakeStore.Test.ScenarioTests
 {
     using Microsoft.WindowsAzure.Commands.ScenarioTest;

src/ResourceManager/DataLakeStore/Commands.DataLakeStore.Test/ScenarioTests/AdlsAliasTests.ps1

@@ -424,38 +424,46 @@ function Test-DataLakeStoreFileSystem
 		$result = Get-AdlStoreItem -Account $accountName -path $folderToCreate
 		Assert-NotNull $result "No value was returned on folder get"
 		Assert-AreEqual "Directory" $result.Type
+
 		# Create and get Empty File
 		$result = New-AdlStoreItem -Account $accountName -path $emptyFilePath
 		Assert-NotNull $result "No value was returned on empty file creation"
 		$result = Get-AdlStoreItem -Account $accountName -path $emptyFilePath
+		$emptyFileCreationDate=$result.LastWriteTime # To be used later
 		Assert-NotNull $result "No value was returned on empty file get"
 		Assert-AreEqual "File" $result.Type
 		Assert-AreEqual 0 $result.Length
+
 		# Create and get file with content
 		$result = New-AdlStoreItem -Account $accountName -path $contentFilePath -Value $content
 		Assert-NotNull $result "No value was returned on content file creation"
 		$result = Get-AdlStoreItem -Account $accountName -path $contentFilePath
 		Assert-NotNull $result "No value was returned on content file get"
 		Assert-AreEqual "File" $result.Type
 		Assert-AreEqual $content.length $result.Length
-		# set and validate expiration for a file
+
+		# set absolute expiration for content file
 		Assert-True {253402300800000 -ge $result.ExpirationTime -or 0 -le $result.ExpirationTime} # validate that expiration is currently max value
 		[DateTimeOffset]$timeToUse = [Microsoft.Azure.Test.HttpRecorder.HttpMockServer]::GetVariable("absoluteTime", [DateTimeOffset]::UtcNow.AddSeconds(120))
 		$result = Set-AdlStoreItemExpiry -Account $accountName -path $contentFilePath -Expiration $timeToUse
-		Assert-AreEqual $timeToUse.UtcTicks $result.Expiration.UtcTicks
+		Assert-NumAreInRange $timeToUse.UtcTicks $result.Expiration.UtcTicks 500000 # range of 50 milliseconds
+
 		# set it back to "never expire"
 		$result = Set-AdlStoreItemExpiry -Account $accountName -path $contentFilePath
 		Assert-True {253402300800000 -ge $result.ExpirationTime -or 0 -le $result.ExpirationTime} # validate that expiration is currently max value
+
 		# list files
 		$result = Get-AdlStoreChildItem -Account $accountName -path $folderToCreate
 		Assert-NotNull $result "No value was returned on folder list"
 		Assert-AreEqual 2 $result.length
+
 		# add content to empty file
 		Add-AdlStoreItemContent -Account $accountName -Path $emptyFilePath -Value $content
 		$result = Get-AdlStoreItem -Account $accountName -path $emptyFilePath
 		Assert-NotNull $result "No value was returned on empty file get with content added"
 		Assert-AreEqual "File" $result.Type
 		Assert-AreEqual $content.length $result.Length
+
 		# concat files
 		$result = Join-AdlStoreItem -Account $accountName -Paths $emptyFilePath,$contentFilePath -Destination $concatFile
 		Assert-NotNull $result "No value was returned on concat file"
@@ -506,6 +514,7 @@ function Test-DataLakeStoreFileSystem
 		Assert-NotNull $result "No value was returned on import file get"
 		Assert-AreEqual "File" $result.Type
 		Assert-AreEqual $localFileInfo.length $result.Length
+
 		# download file
 		$currentDir = Split-Path $fileToCopy
 		$targetFile = Join-Path $currentDir "adlspstestdownload.txt"
@@ -527,6 +536,7 @@ function Test-DataLakeStoreFileSystem
 		Assert-AreEqual "File" $result.Type
 		Assert-AreEqual $($content.length*2) $result.Length
 		Assert-Throws {Get-AdlStoreItem -Account $accountName -path $concatFile}
+
 		# move a folder
 		$result = Move-AdlStoreItem -Account $accountName -Path $folderToCreate -Destination $moveFolder
 		Assert-NotNull $result "No value was returned on move folder"
@@ -535,9 +545,11 @@ function Test-DataLakeStoreFileSystem
 		Assert-AreEqual "Directory" $result.Type
 		Assert-AreEqual 0 $result.Length
 		Assert-Throws {Get-AdlStoreItem -Account $accountName -path $folderToCreate}
+
 		# delete a file
 		Assert-True {Remove-AdlStoreItem -Account $accountName -paths "$moveFolder/movefile.txt" -force -passthru } "Remove File Failed"
 		Assert-Throws {Get-AdlStoreItem -Account $accountName -path $moveFile}
+
 		# delete a folder
 		Assert-True {Remove-AdlStoreItem -Account $accountName -paths $moveFolder -force -recurse -passthru} "Remove folder failed"
 		Assert-Throws {Get-AdlStoreItem -Account $accountName -path $moveFolder}
@@ -601,6 +613,17 @@ function Test-DataLakeStoreFileSystemPermissions
 		# define the permissions to add/remove
 		$aceUserId = "027c28d5-c91d-49f0-98c5-d10134b169b3"
 
+		#set owner
+        New-AdlStoreItem -Account $accountName -Path "/temp"
+        $prevOwner=Get-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type User
+        $prevGroup=Get-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type Group
+        $currentOwner=Set-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type User -Id $aceUserId -PassThru
+        $currentGroup=Get-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type Group
+        Assert-AreEqual $aceUserId $currentOwner
+        Assert-AreNotEqual $prevOwner $currentOwner
+        Assert-AreEqual $prevGroup $currentGroup
+        Remove-AdlStoreItem -Account $accountName -paths "/temp" -force
+
 		# Set and get all the permissions
 		$result = Get-AdlStoreItemAclEntry -Account $accountName -path "/"
 		Assert-NotNull $result "Did not get any result from ACL get" 
@@ -635,14 +658,17 @@ function Test-DataLakeStoreFileSystemPermissions
 		Set-AdlStoreItemAclEntry -Account $accountName -path "/" -AceType User -Id $aceUserId -Permissions All
 		$result = Get-AdlStoreItemAclEntry -Account $accountName -path "/"
 		Assert-AreEqual $($currentCount+1) $result.Count
+
 		# remove a specific permission with friendly remove
 		Remove-AdlStoreItemAclEntry -Account $accountName -path "/" -AceType User -Id $aceUserId
 		$result = Get-AdlStoreItemAclEntry -Account $accountName -path "/"
 		Assert-AreEqual $($currentCount) $result.Count
+
 		# set and get a specific permission with the ACE string
 		Set-AdlStoreItemAclEntry -Account $accountName -path "/" -Acl $([string]::Format("user:{0}:rwx", $aceUserId))
 		$result = Get-AdlStoreItemAclEntry -Account $accountName -path "/"
 		Assert-AreEqual $($currentCount+1) $result.Count
+
 		# remove a specific permission with the ACE string
 		Remove-AdlStoreItemAclEntry -Account $accountName -path "/" -Acl $([string]::Format("user:{0}:---", $aceUserId))
 		$result = Get-AdlStoreItemAclEntry -Account $accountName -path "/"

src/ResourceManager/DataLakeStore/Commands.DataLakeStore.Test/ScenarioTests/AdlsTests.ps1

@@ -423,38 +423,46 @@ function Test-DataLakeStoreFileSystem
 		$result = Get-AzureRMDataLakeStoreItem -Account $accountName -path $folderToCreate
 		Assert-NotNull $result "No value was returned on folder get"
 		Assert-AreEqual "Directory" $result.Type
+
 		# Create and get Empty File
 		$result = New-AzureRMDataLakeStoreItem -Account $accountName -path $emptyFilePath
 		Assert-NotNull $result "No value was returned on empty file creation"
 		$result = Get-AzureRMDataLakeStoreItem -Account $accountName -path $emptyFilePath
+		$emptyFileCreationDate=$result.LastWriteTime # To be used later
 		Assert-NotNull $result "No value was returned on empty file get"
 		Assert-AreEqual "File" $result.Type
 		Assert-AreEqual 0 $result.Length
+
 		# Create and get file with content
 		$result = New-AzureRMDataLakeStoreItem -Account $accountName -path $contentFilePath -Value $content
 		Assert-NotNull $result "No value was returned on content file creation"
 		$result = Get-AzureRMDataLakeStoreItem -Account $accountName -path $contentFilePath
 		Assert-NotNull $result "No value was returned on content file get"
 		Assert-AreEqual "File" $result.Type
 		Assert-AreEqual $content.length $result.Length
-		# set and validate expiration for a file
+
+		# set absolute expiration for content file
 		Assert-True {253402300800000 -ge $result.ExpirationTime -or 0 -le $result.ExpirationTime} # validate that expiration is currently max value
 		[DateTimeOffset]$timeToUse = [Microsoft.Azure.Test.HttpRecorder.HttpMockServer]::GetVariable("absoluteTime", [DateTimeOffset]::UtcNow.AddSeconds(120))
-		$result = Set-AzureRmDataLakeStoreItemExpiry -Account $accountName -path $contentFilePath -Expiration $timeToUse
-		Assert-AreEqual $timeToUse.UtcTicks $result.Expiration.UtcTicks
+		$result = Set-AdlStoreItemExpiry -Account $accountName -path $contentFilePath -Expiration $timeToUse
+		Assert-NumAreInRange $timeToUse.UtcTicks $result.Expiration.UtcTicks 500000 # range of 50 milliseconds
+
 		# set it back to "never expire"
-		$result = Set-AzureRmDataLakeStoreItemExpiry -Account $accountName -path $contentFilePath
+		$result = Set-AdlStoreItemExpiry -Account $accountName -path $contentFilePath
 		Assert-True {253402300800000 -ge $result.ExpirationTime -or 0 -le $result.ExpirationTime} # validate that expiration is currently max value
+
 		# list files
 		$result = Get-AzureRMDataLakeStoreChildItem -Account $accountName -path $folderToCreate
 		Assert-NotNull $result "No value was returned on folder list"
 		Assert-AreEqual 2 $result.length
+
 		# add content to empty file
 		Add-AzureRMDataLakeStoreItemContent -Account $accountName -Path $emptyFilePath -Value $content
 		$result = Get-AzureRMDataLakeStoreItem -Account $accountName -path $emptyFilePath
 		Assert-NotNull $result "No value was returned on empty file get with content added"
 		Assert-AreEqual "File" $result.Type
 		Assert-AreEqual $content.length $result.Length
+
 		# concat files
 		$result = Join-AzureRMDataLakeStoreItem -Account $accountName -Paths $emptyFilePath,$contentFilePath -Destination $concatFile
 		Assert-NotNull $result "No value was returned on concat file"
@@ -505,6 +513,7 @@ function Test-DataLakeStoreFileSystem
 		Assert-NotNull $result "No value was returned on import file get"
 		Assert-AreEqual "File" $result.Type
 		Assert-AreEqual $localFileInfo.length $result.Length
+
 		# download file
 		$currentDir = Split-Path $fileToCopy
 		$targetFile = Join-Path $currentDir "adlspstestdownload.txt"
@@ -517,7 +526,7 @@ function Test-DataLakeStoreFileSystem
 		$downloadedFileInfo = Get-ChildItem $targetFile
 		Assert-AreEqual $($content.length*2) $downloadedFileInfo.length
 		Remove-Item -path $targetFile -force -confirm:$false
-
+		
 		# move a file
 		$result = Move-AzureRMDataLakeStoreItem -Account $accountName -Path $concatFile -Destination $moveFile
 		Assert-NotNull $result "No value was returned on move file"
@@ -526,6 +535,7 @@ function Test-DataLakeStoreFileSystem
 		Assert-AreEqual "File" $result.Type
 		Assert-AreEqual $($content.length*2) $result.Length
 		Assert-Throws {Get-AzureRMDataLakeStoreItem -Account $accountName -path $concatFile}
+
 		# move a folder
 		$result = Move-AzureRMDataLakeStoreItem -Account $accountName -Path $folderToCreate -Destination $moveFolder
 		Assert-NotNull $result "No value was returned on move folder"
@@ -534,9 +544,11 @@ function Test-DataLakeStoreFileSystem
 		Assert-AreEqual "Directory" $result.Type
 		Assert-AreEqual 0 $result.Length
 		Assert-Throws {Get-AzureRMDataLakeStoreItem -Account $accountName -path $folderToCreate}
+
 		# delete a file
 		Assert-True {Remove-AzureRMDataLakeStoreItem -Account $accountName -paths "$moveFolder/movefile.txt" -force -passthru } "Remove File Failed"
 		Assert-Throws {Get-AzureRMDataLakeStoreItem -Account $accountName -path $moveFile}
+
 		# delete a folder
 		Assert-True {Remove-AzureRMDataLakeStoreItem -Account $accountName -paths $moveFolder -force -recurse -passthru} "Remove folder failed"
 		Assert-Throws {Get-AzureRMDataLakeStoreItem -Account $accountName -path $moveFolder}
@@ -600,6 +612,17 @@ function Test-DataLakeStoreFileSystemPermissions
 		# define the permissions to add/remove
 		$aceUserId = "027c28d5-c91d-49f0-98c5-d10134b169b3"
 
+		#set owner
+        New-AdlStoreItem -Account $accountName -Path "/temp"
+        $prevOwner=Get-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type User
+        $prevGroup=Get-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type Group
+        $currentOwner=Set-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type User -Id $aceUserId -PassThru
+        $currentGroup=Get-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type Group
+        Assert-AreEqual $aceUserId $currentOwner
+        Assert-AreNotEqual $prevOwner $currentOwner
+        Assert-AreEqual $prevGroup $currentGroup
+        Remove-AdlStoreItem -Account $accountName -paths "/temp" -force
+
 		# Set and get all the permissions
 		$result = Get-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/"
 		Assert-NotNull $result "Did not get any result from ACL get" 
@@ -636,14 +659,17 @@ function Test-DataLakeStoreFileSystemPermissions
 		Set-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/" -AceType User -Id $aceUserId -Permissions All
 		$result = Get-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/"
 		Assert-AreEqual $($currentCount+1) $result.Count
+
 		# remove a specific permission with friendly remove
 		Remove-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/" -AceType User -Id $aceUserId
 		$result = Get-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/"
 		Assert-AreEqual $($currentCount) $result.Count
+
 		# set and get a specific permission with the ACE string
 		Set-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/" -Acl $([string]::Format("user:{0}:rwx", $aceUserId))
 		$result = Get-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/"
 		Assert-AreEqual $($currentCount+1) $result.Count
+
 		# remove a specific permission with the ACE string
 		Remove-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/" -Acl $([string]::Format("user:{0}:---", $aceUserId))
 		$result = Get-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/"