Merge Adls data plane to preview

src/ResourceManager/Common/Commands.ScenarioTests.ResourceManager.Common/Assert.ps1

@@ -277,7 +277,28 @@ function Assert-AreEqual
 
   return $true
 }
-
+###################
+#
+# Verify that if actual falls in an acceptable range of expected
+#
+#    param [long] $expected : The expected number
+#    param [long] $actual   : The actual number
+#	 param [long] $interval : The acceptable offset either side of the expected
+#    param [string] $message  : The message to return if the given objects are not equal
+####################
+function Assert-NumAreInRange
+{
+	param([long] $expected, [long] $actual, [long] $interval, [string] $message)
+	if (!$message)
+	{
+		$message = "Assertion failed because expected '$expected' does not fall in accepted range of 'interval' of actual '$actual'"
+	}
+	if(!($actual -ge ($expected-$interval) -and $actual -le ($expected+$interval)))
+	{
+        throw $message
+    }
+	return $true
+}
 ###################
 #
 # Verify that two given arrays are equal

src/ResourceManager/DataLakeStore/AzureRM.DataLakeStore.psd1

@@ -12,7 +12,7 @@
 # RootModule = ''
 
 # Version number of this module.
-ModuleVersion = '5.1.1'
+ModuleVersion = '5.2.0'
 
 # Supported PSEditions
 # CompatiblePSEditions = @()
@@ -54,7 +54,7 @@ CLRVersion = '4.0'
 RequiredModules = @(@{ModuleName = 'AzureRM.Profile'; ModuleVersion = '4.5.0'; })
 
 # Assemblies that must be loaded prior to importing this module
-RequiredAssemblies = '.\Microsoft.Azure.Management.DataLake.Store.dll'
+RequiredAssemblies = '.\Microsoft.Azure.Management.DataLake.Store.dll','.\Microsoft.Azure.DataLake.Store.dll','.\NLog.dll'
 
 # Script files (.ps1) that are run in the caller's environment prior to importing this module.
 # ScriptsToProcess = @()
@@ -63,8 +63,7 @@ RequiredAssemblies = '.\Microsoft.Azure.Management.DataLake.Store.dll'
 # TypesToProcess = @()
 
 # Format files (.ps1xml) to be loaded when importing this module
-FormatsToProcess = 
-               '.\Microsoft.Azure.Commands.DataLakeStoreFileSystem.format.ps1xml'
+FormatsToProcess = '.\Microsoft.Azure.Commands.DataLakeStoreFileSystem.format.ps1xml'
 
 # Modules to import as nested modules of the module specified in RootModule/ModuleToProcess
 NestedModules = @('.\Microsoft.Azure.Commands.DataLakeStore.dll')
@@ -152,12 +151,16 @@ PrivateData = @{
         # IconUri = ''
 
         # ReleaseNotes of this module
-        ReleaseNotes = '* Corrected usage of ''Login-AzureRmAccount'' to use ''Connect-AzureRmAccount''
-* Corrected the error message of ''Test-AzureRmDataLakeStoreAccount'' when running this cmdlet without having logged in with ''Login-AzureRmAccount'''
-
-        # Prerelease string of this module
-        # Prerelease = ''
-
+        ReleaseNotes = '
+		* Export-AzureRmDataLakeStoreItem (https://github.com/Azure/azure-powershell/blob/adls-data-plane/src/ResourceManager/DataLakeStore/documentation/upcoming-breaking-changes.md) - Deprecated parameters PerFileThreadCount, ConcurrentFileCount and introduced parameter Concurrency
+		* Import-AzureRMDataLakeStoreItem (https://github.com/Azure/azure-powershell/blob/adls-data-plane/src/ResourceManager/DataLakeStore/documentation/upcoming-breaking-changes.md) -Deprecated parametersPerFileThreadCount, ConcurrentFileCount and introduced parameter Concurrency
+		* Get-AzureRMDataLakeStoreItemContent - Fixed the tail behavior for contents greater than 4MB
+		* Set-AzureRMDataLakeStoreItemExpiry - Introduced new parameter set SetRelativeExpiry for setting relative expiration time
+		* Remove-AzureRmDataLakeStoreItem (https://github.com/Azure/azure-powershell/blob/adls-data-plane/src/ResourceManager/DataLakeStore/documentation/upcoming-breaking-changes.md) - Deprecated parameter Clean.
+		'
+
+		# Prerelease string of this module
+		Prerelease = 'preview'
         # Flag to indicate whether the module requires explicit user acceptance for install/update
         # RequireLicenseAcceptance = $false
 

src/ResourceManager/DataLakeStore/ChangeLog.md

@@ -19,10 +19,13 @@
 -->
 ## Current Release
 * Updated to the latest version of the Azure ClientRuntime
-
-## Version 5.1.1
-* Corrected usage of 'Login-AzureRmAccount' to use 'Connect-AzureRmAccount'
-* Corrected the error message of 'Test-AzureRmDataLakeStoreAccount' when running this cmdlet without having logged in with 'Login-AzureRmAccount'
+* Add debug functionality
+* Update the version of the ADLS dataplane SDK to 1.1.2
+* Export-AzureRmDataLakeStoreItem (https://github.com/Azure/azure-powershell/blob/adls-data-plane/src/ResourceManager/DataLakeStore/documentation/upcoming-breaking-changes.md) - Deprecated parameters PerFileThreadCount, ConcurrentFileCount and introduced parameter Concurrency
+* Import-AzureRMDataLakeStoreItem (https://github.com/Azure/azure-powershell/blob/adls-data-plane/src/ResourceManager/DataLakeStore/documentation/upcoming-breaking-changes.md) -Deprecated parametersPerFileThreadCount, ConcurrentFileCount and introduced parameter Concurrency
+* Get-AzureRMDataLakeStoreItemContent - Fixed the tail behavior for contents greater than 4MB
+* Set-AzureRMDataLakeStoreItemExpiry - Introduced new parameter set SetRelativeExpiry for setting relative expiration time
+* Remove-AzureRmDataLakeStoreItem (https://github.com/Azure/azure-powershell/blob/adls-data-plane/src/ResourceManager/DataLakeStore/documentation/upcoming-breaking-changes.md) - Deprecated parameter Clean.
 
 ## Version 5.1.0
 * Added Location Completer to -Location parameters allowing tab completion through valid Locations

src/ResourceManager/DataLakeStore/Commands.DataLakeStore.Test/Commands.DataLakeStore.Test.csproj

@@ -188,6 +188,10 @@
       <Project>{5ee72c53-1720-4309-b54b-5fb79703195f}</Project>
       <Name>Commands.Common</Name>
     </ProjectReference>
+    <ProjectReference Include="..\Commands.DataLakeStore\Commands.DataLakeStore.csproj">
+      <Project>{8aab43e6-e8f6-4f91-af8a-6a63cd78ef1e}</Project>
+      <Name>Commands.DataLakeStore</Name>
+    </ProjectReference>
   </ItemGroup>
   <ItemGroup>
     <None Include="MSSharedLibKey.snk" />

src/ResourceManager/DataLakeStore/Commands.DataLakeStore.Test/ScenarioTests/AdlsAliasTests.cs

@@ -12,6 +12,8 @@
 // limitations under the License.
 // ----------------------------------------------------------------------------------
 
+using System;
+
 namespace Microsoft.Azure.Commands.DataLakeStore.Test.ScenarioTests
 {
     using Microsoft.WindowsAzure.Commands.ScenarioTest;

src/ResourceManager/DataLakeStore/Commands.DataLakeStore.Test/ScenarioTests/AdlsAliasTests.ps1

@@ -424,38 +424,46 @@ function Test-DataLakeStoreFileSystem
 		$result = Get-AdlStoreItem -Account $accountName -path $folderToCreate
 		Assert-NotNull $result "No value was returned on folder get"
 		Assert-AreEqual "Directory" $result.Type
+
 		# Create and get Empty File
 		$result = New-AdlStoreItem -Account $accountName -path $emptyFilePath
 		Assert-NotNull $result "No value was returned on empty file creation"
 		$result = Get-AdlStoreItem -Account $accountName -path $emptyFilePath
+		$emptyFileCreationDate=$result.LastWriteTime # To be used later
 		Assert-NotNull $result "No value was returned on empty file get"
 		Assert-AreEqual "File" $result.Type
 		Assert-AreEqual 0 $result.Length
+
 		# Create and get file with content
 		$result = New-AdlStoreItem -Account $accountName -path $contentFilePath -Value $content
 		Assert-NotNull $result "No value was returned on content file creation"
 		$result = Get-AdlStoreItem -Account $accountName -path $contentFilePath
 		Assert-NotNull $result "No value was returned on content file get"
 		Assert-AreEqual "File" $result.Type
 		Assert-AreEqual $content.length $result.Length
-		# set and validate expiration for a file
+
+		# set absolute expiration for content file
 		Assert-True {253402300800000 -ge $result.ExpirationTime -or 0 -le $result.ExpirationTime} # validate that expiration is currently max value
 		[DateTimeOffset]$timeToUse = [Microsoft.Azure.Test.HttpRecorder.HttpMockServer]::GetVariable("absoluteTime", [DateTimeOffset]::UtcNow.AddSeconds(120))
 		$result = Set-AdlStoreItemExpiry -Account $accountName -path $contentFilePath -Expiration $timeToUse
-		Assert-AreEqual $timeToUse.UtcTicks $result.Expiration.UtcTicks
+		Assert-NumAreInRange $timeToUse.UtcTicks $result.Expiration.UtcTicks 500000 # range of 50 milliseconds
+
 		# set it back to "never expire"
 		$result = Set-AdlStoreItemExpiry -Account $accountName -path $contentFilePath
 		Assert-True {253402300800000 -ge $result.ExpirationTime -or 0 -le $result.ExpirationTime} # validate that expiration is currently max value
+
 		# list files
 		$result = Get-AdlStoreChildItem -Account $accountName -path $folderToCreate
 		Assert-NotNull $result "No value was returned on folder list"
 		Assert-AreEqual 2 $result.length
+
 		# add content to empty file
 		Add-AdlStoreItemContent -Account $accountName -Path $emptyFilePath -Value $content
 		$result = Get-AdlStoreItem -Account $accountName -path $emptyFilePath
 		Assert-NotNull $result "No value was returned on empty file get with content added"
 		Assert-AreEqual "File" $result.Type
 		Assert-AreEqual $content.length $result.Length
+
 		# concat files
 		$result = Join-AdlStoreItem -Account $accountName -Paths $emptyFilePath,$contentFilePath -Destination $concatFile
 		Assert-NotNull $result "No value was returned on concat file"
@@ -506,6 +514,7 @@ function Test-DataLakeStoreFileSystem
 		Assert-NotNull $result "No value was returned on import file get"
 		Assert-AreEqual "File" $result.Type
 		Assert-AreEqual $localFileInfo.length $result.Length
+
 		# download file
 		$currentDir = Split-Path $fileToCopy
 		$targetFile = Join-Path $currentDir "adlspstestdownload.txt"
@@ -527,6 +536,7 @@ function Test-DataLakeStoreFileSystem
 		Assert-AreEqual "File" $result.Type
 		Assert-AreEqual $($content.length*2) $result.Length
 		Assert-Throws {Get-AdlStoreItem -Account $accountName -path $concatFile}
+
 		# move a folder
 		$result = Move-AdlStoreItem -Account $accountName -Path $folderToCreate -Destination $moveFolder
 		Assert-NotNull $result "No value was returned on move folder"
@@ -535,9 +545,11 @@ function Test-DataLakeStoreFileSystem
 		Assert-AreEqual "Directory" $result.Type
 		Assert-AreEqual 0 $result.Length
 		Assert-Throws {Get-AdlStoreItem -Account $accountName -path $folderToCreate}
+
 		# delete a file
 		Assert-True {Remove-AdlStoreItem -Account $accountName -paths "$moveFolder/movefile.txt" -force -passthru } "Remove File Failed"
 		Assert-Throws {Get-AdlStoreItem -Account $accountName -path $moveFile}
+
 		# delete a folder
 		Assert-True {Remove-AdlStoreItem -Account $accountName -paths $moveFolder -force -recurse -passthru} "Remove folder failed"
 		Assert-Throws {Get-AdlStoreItem -Account $accountName -path $moveFolder}
@@ -601,6 +613,17 @@ function Test-DataLakeStoreFileSystemPermissions
 		# define the permissions to add/remove
 		$aceUserId = "027c28d5-c91d-49f0-98c5-d10134b169b3"
 
+		#set owner
+        New-AdlStoreItem -Account $accountName -Path "/temp"
+        $prevOwner=Get-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type User
+        $prevGroup=Get-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type Group
+        $currentOwner=Set-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type User -Id $aceUserId -PassThru
+        $currentGroup=Get-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type Group
+        Assert-AreEqual $aceUserId $currentOwner
+        Assert-AreNotEqual $prevOwner $currentOwner
+        Assert-AreEqual $prevGroup $currentGroup
+        Remove-AdlStoreItem -Account $accountName -paths "/temp" -force
+
 		# Set and get all the permissions
 		$result = Get-AdlStoreItemAclEntry -Account $accountName -path "/"
 		Assert-NotNull $result "Did not get any result from ACL get" 
@@ -635,14 +658,17 @@ function Test-DataLakeStoreFileSystemPermissions
 		Set-AdlStoreItemAclEntry -Account $accountName -path "/" -AceType User -Id $aceUserId -Permissions All
 		$result = Get-AdlStoreItemAclEntry -Account $accountName -path "/"
 		Assert-AreEqual $($currentCount+1) $result.Count
+
 		# remove a specific permission with friendly remove
 		Remove-AdlStoreItemAclEntry -Account $accountName -path "/" -AceType User -Id $aceUserId
 		$result = Get-AdlStoreItemAclEntry -Account $accountName -path "/"
 		Assert-AreEqual $($currentCount) $result.Count
+
 		# set and get a specific permission with the ACE string
 		Set-AdlStoreItemAclEntry -Account $accountName -path "/" -Acl $([string]::Format("user:{0}:rwx", $aceUserId))
 		$result = Get-AdlStoreItemAclEntry -Account $accountName -path "/"
 		Assert-AreEqual $($currentCount+1) $result.Count
+
 		# remove a specific permission with the ACE string
 		Remove-AdlStoreItemAclEntry -Account $accountName -path "/" -Acl $([string]::Format("user:{0}:---", $aceUserId))
 		$result = Get-AdlStoreItemAclEntry -Account $accountName -path "/"

src/ResourceManager/DataLakeStore/Commands.DataLakeStore.Test/ScenarioTests/AdlsTests.ps1

@@ -423,38 +423,46 @@ function Test-DataLakeStoreFileSystem
 		$result = Get-AzureRMDataLakeStoreItem -Account $accountName -path $folderToCreate
 		Assert-NotNull $result "No value was returned on folder get"
 		Assert-AreEqual "Directory" $result.Type
+
 		# Create and get Empty File
 		$result = New-AzureRMDataLakeStoreItem -Account $accountName -path $emptyFilePath
 		Assert-NotNull $result "No value was returned on empty file creation"
 		$result = Get-AzureRMDataLakeStoreItem -Account $accountName -path $emptyFilePath
+		$emptyFileCreationDate=$result.LastWriteTime # To be used later
 		Assert-NotNull $result "No value was returned on empty file get"
 		Assert-AreEqual "File" $result.Type
 		Assert-AreEqual 0 $result.Length
+
 		# Create and get file with content
 		$result = New-AzureRMDataLakeStoreItem -Account $accountName -path $contentFilePath -Value $content
 		Assert-NotNull $result "No value was returned on content file creation"
 		$result = Get-AzureRMDataLakeStoreItem -Account $accountName -path $contentFilePath
 		Assert-NotNull $result "No value was returned on content file get"
 		Assert-AreEqual "File" $result.Type
 		Assert-AreEqual $content.length $result.Length
-		# set and validate expiration for a file
+
+		# set absolute expiration for content file
 		Assert-True {253402300800000 -ge $result.ExpirationTime -or 0 -le $result.ExpirationTime} # validate that expiration is currently max value
 		[DateTimeOffset]$timeToUse = [Microsoft.Azure.Test.HttpRecorder.HttpMockServer]::GetVariable("absoluteTime", [DateTimeOffset]::UtcNow.AddSeconds(120))
-		$result = Set-AzureRmDataLakeStoreItemExpiry -Account $accountName -path $contentFilePath -Expiration $timeToUse
-		Assert-AreEqual $timeToUse.UtcTicks $result.Expiration.UtcTicks
+		$result = Set-AdlStoreItemExpiry -Account $accountName -path $contentFilePath -Expiration $timeToUse
+		Assert-NumAreInRange $timeToUse.UtcTicks $result.Expiration.UtcTicks 500000 # range of 50 milliseconds
+
 		# set it back to "never expire"
-		$result = Set-AzureRmDataLakeStoreItemExpiry -Account $accountName -path $contentFilePath
+		$result = Set-AdlStoreItemExpiry -Account $accountName -path $contentFilePath
 		Assert-True {253402300800000 -ge $result.ExpirationTime -or 0 -le $result.ExpirationTime} # validate that expiration is currently max value
+
 		# list files
 		$result = Get-AzureRMDataLakeStoreChildItem -Account $accountName -path $folderToCreate
 		Assert-NotNull $result "No value was returned on folder list"
 		Assert-AreEqual 2 $result.length
+
 		# add content to empty file
 		Add-AzureRMDataLakeStoreItemContent -Account $accountName -Path $emptyFilePath -Value $content
 		$result = Get-AzureRMDataLakeStoreItem -Account $accountName -path $emptyFilePath
 		Assert-NotNull $result "No value was returned on empty file get with content added"
 		Assert-AreEqual "File" $result.Type
 		Assert-AreEqual $content.length $result.Length
+
 		# concat files
 		$result = Join-AzureRMDataLakeStoreItem -Account $accountName -Paths $emptyFilePath,$contentFilePath -Destination $concatFile
 		Assert-NotNull $result "No value was returned on concat file"
@@ -505,6 +513,7 @@ function Test-DataLakeStoreFileSystem
 		Assert-NotNull $result "No value was returned on import file get"
 		Assert-AreEqual "File" $result.Type
 		Assert-AreEqual $localFileInfo.length $result.Length
+
 		# download file
 		$currentDir = Split-Path $fileToCopy
 		$targetFile = Join-Path $currentDir "adlspstestdownload.txt"
@@ -517,7 +526,7 @@ function Test-DataLakeStoreFileSystem
 		$downloadedFileInfo = Get-ChildItem $targetFile
 		Assert-AreEqual $($content.length*2) $downloadedFileInfo.length
 		Remove-Item -path $targetFile -force -confirm:$false
-
+		
 		# move a file
 		$result = Move-AzureRMDataLakeStoreItem -Account $accountName -Path $concatFile -Destination $moveFile
 		Assert-NotNull $result "No value was returned on move file"
@@ -526,6 +535,7 @@ function Test-DataLakeStoreFileSystem
 		Assert-AreEqual "File" $result.Type
 		Assert-AreEqual $($content.length*2) $result.Length
 		Assert-Throws {Get-AzureRMDataLakeStoreItem -Account $accountName -path $concatFile}
+
 		# move a folder
 		$result = Move-AzureRMDataLakeStoreItem -Account $accountName -Path $folderToCreate -Destination $moveFolder
 		Assert-NotNull $result "No value was returned on move folder"
@@ -534,9 +544,11 @@ function Test-DataLakeStoreFileSystem
 		Assert-AreEqual "Directory" $result.Type
 		Assert-AreEqual 0 $result.Length
 		Assert-Throws {Get-AzureRMDataLakeStoreItem -Account $accountName -path $folderToCreate}
+
 		# delete a file
 		Assert-True {Remove-AzureRMDataLakeStoreItem -Account $accountName -paths "$moveFolder/movefile.txt" -force -passthru } "Remove File Failed"
 		Assert-Throws {Get-AzureRMDataLakeStoreItem -Account $accountName -path $moveFile}
+
 		# delete a folder
 		Assert-True {Remove-AzureRMDataLakeStoreItem -Account $accountName -paths $moveFolder -force -recurse -passthru} "Remove folder failed"
 		Assert-Throws {Get-AzureRMDataLakeStoreItem -Account $accountName -path $moveFolder}
@@ -600,6 +612,17 @@ function Test-DataLakeStoreFileSystemPermissions
 		# define the permissions to add/remove
 		$aceUserId = "027c28d5-c91d-49f0-98c5-d10134b169b3"
 
+		#set owner
+        New-AdlStoreItem -Account $accountName -Path "/temp"
+        $prevOwner=Get-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type User
+        $prevGroup=Get-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type Group
+        $currentOwner=Set-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type User -Id $aceUserId -PassThru
+        $currentGroup=Get-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type Group
+        Assert-AreEqual $aceUserId $currentOwner
+        Assert-AreNotEqual $prevOwner $currentOwner
+        Assert-AreEqual $prevGroup $currentGroup
+        Remove-AdlStoreItem -Account $accountName -paths "/temp" -force
+
 		# Set and get all the permissions
 		$result = Get-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/"
 		Assert-NotNull $result "Did not get any result from ACL get" 
@@ -636,14 +659,17 @@ function Test-DataLakeStoreFileSystemPermissions
 		Set-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/" -AceType User -Id $aceUserId -Permissions All
 		$result = Get-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/"
 		Assert-AreEqual $($currentCount+1) $result.Count
+
 		# remove a specific permission with friendly remove
 		Remove-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/" -AceType User -Id $aceUserId
 		$result = Get-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/"
 		Assert-AreEqual $($currentCount) $result.Count
+
 		# set and get a specific permission with the ACE string
 		Set-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/" -Acl $([string]::Format("user:{0}:rwx", $aceUserId))
 		$result = Get-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/"
 		Assert-AreEqual $($currentCount+1) $result.Count
+
 		# remove a specific permission with the ACE string
 		Remove-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/" -Acl $([string]::Format("user:{0}:---", $aceUserId))
 		$result = Get-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/"