Skip to content

Commit 6215676

Browse files
haitsongmsfthaitsongmsft
committed
Mongodb migration changes
1 parent d7b06d4 commit 6215676

15 files changed

+11327
-2914
lines changed

‎src/ResourceManager/DataMigration/Commands.DataMigration.Test/Commands.DataMigration.Test.csproj

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -86,14 +86,18 @@
8686
<None Include="ScenarioTests\ServiceTests.ps1">
8787
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
8888
</None>
89-
<None Include="SessionRecords\Microsoft.Azure.Commands.ScenarioTest.DmsTest.ServiceTests\TestConnectToSourceMongoDb.json" />
89+
<None Include="SessionRecords\Microsoft.Azure.Commands.ScenarioTest.DmsTest.ServiceTests\TestConnectToSourceMongoDb.json" >
90+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
91+
</None>
9092
<None Include="SessionRecords\Microsoft.Azure.Commands.ScenarioTest.DmsTest.ServiceTests\TestConnectToSourceSqlServer.json">
9193
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
9294
</None>
9395
<None Include="SessionRecords\Microsoft.Azure.Commands.ScenarioTest.DmsTest.ServiceTests\TestConnectToSourceSqlServerSync.json">
9496
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
9597
</None>
96-
<None Include="SessionRecords\Microsoft.Azure.Commands.ScenarioTest.DmsTest.ServiceTests\TestConnectToTargetCosmosDb.json" />
98+
<None Include="SessionRecords\Microsoft.Azure.Commands.ScenarioTest.DmsTest.ServiceTests\TestConnectToTargetCosmosDb.json" >
99+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
100+
</None>
97101
<None Include="SessionRecords\Microsoft.Azure.Commands.ScenarioTest.DmsTest.ServiceTests\TestConnectToTargetSqlDb.json">
98102
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
99103
</None>
@@ -115,7 +119,9 @@
115119
<None Include="SessionRecords\Microsoft.Azure.Commands.ScenarioTest.DmsTest.ServiceTests\TestGetUserTableTask.json">
116120
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
117121
</None>
118-
<None Include="SessionRecords\Microsoft.Azure.Commands.ScenarioTest.DmsTest.ServiceTests\TestMigrateMongoDb.json" />
122+
<None Include="SessionRecords\Microsoft.Azure.Commands.ScenarioTest.DmsTest.ServiceTests\TestMigrateMongoDb.json" >
123+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
124+
</None>
119125
<None Include="SessionRecords\Microsoft.Azure.Commands.ScenarioTest.DmsTest.ServiceTests\TestMigrateSqlSqlDb.json">
120126
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
121127
</None>

‎src/ResourceManager/DataMigration/Commands.DataMigration.Test/ScenarioTests/ServiceTests.ps1

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -881,9 +881,9 @@ function Test-MigrateMongoDb
881881
#----------------------
882882
# MIGRATION SETUP
883883
#----------------------
884-
$testColSettingA = New-AzureRmDataMigrationMongoDbCollectionSetting -Name large -RU 1000 -CanDelete -ShardKeyFields "_id"
885-
$testColSettingB = New-AzureRmDataMigrationMongoDbCollectionSetting -Name many -RU 1000 -CanDelete -ShardKeyFields "_id"
886-
$testDbSetting = New-AzureRmDataMigrationMongoDbDatabaseSetting -Name test -collections @($testColSettingA, $testColSettingB)
884+
$testColSettingA = New-AzureRmDataMigrationMongoDbCollectionSetting -Name large -RU 1000 -CanDelete -ShardKey "_id"
885+
$testColSettingB = New-AzureRmDataMigrationMongoDbCollectionSetting -Name many -RU 1000 -CanDelete -ShardKey "_id"
886+
$testDbSetting = New-AzureRmDataMigrationMongoDbDatabaseSetting -Name test -CollectionSetting @($testColSettingA, $testColSettingB)
887887

888888
#----------------------
889889
# MIGRATION VALIDATION

‎src/ResourceManager/DataMigration/Commands.DataMigration.Test/SessionRecords/Microsoft.Azure.Commands.ScenarioTest.DmsTest.ServiceTests/TestConnectToSourceMongoDb.json

Lines changed: 3104 additions & 123 deletions
Large diffs are not rendered by default.

‎src/ResourceManager/DataMigration/Commands.DataMigration.Test/SessionRecords/Microsoft.Azure.Commands.ScenarioTest.DmsTest.ServiceTests/TestConnectToTargetCosmosDb.json

Lines changed: 2825 additions & 123 deletions
Large diffs are not rendered by default.

‎src/ResourceManager/DataMigration/Commands.DataMigration.Test/SessionRecords/Microsoft.Azure.Commands.ScenarioTest.DmsTest.ServiceTests/TestMigrateMongoDb.json

Lines changed: 3401 additions & 558 deletions
Large diffs are not rendered by default.

‎src/ResourceManager/DataMigration/Commands.DataMigration.Test/SessionRecords/Microsoft.Azure.Commands.ScenarioTest.DmsTest.ServiceTests/TestMigrateSqlSqlDBSync.json

Lines changed: 1931 additions & 2061 deletions
Large diffs are not rendered by default.

‎src/ResourceManager/DataMigration/Commands.DataMigration.Test/appsettings.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,6 @@
2020
"FILESHARE_PASSWORD": "Add-fileshare-password-Here",
2121
"MI_LOGINS": "user1,user2,user3",
2222
"MI_AGENTJOBS": "testAgentJob1,testAgentJob2",
23-
"MONGODB_SOURCE_CONNECTIONSTRING": "",
24-
"COSMOSDB_TARGET_CONNECTIONSTRING": ""
23+
"MONGODB_SOURCE_CONNECTIONSTRING": "Add-MongoDb-Connection-Here",
24+
"COSMOSDB_TARGET_CONNECTIONSTRING": "Add-CosmosDb-Connection-Here"
2525
}

‎src/ResourceManager/DataMigration/Commands.DataMigration/Cmdlets/NewMongoDbCollectionSettingCmdlet.cs

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -15,16 +15,14 @@
1515
using System.Management.Automation;
1616
using System.Text.RegularExpressions;
1717
using Microsoft.Azure.Management.DataMigration.Models;
18+
using Microsoft.Azure.Commands.DataMigration.Models.MongoDb;
1819

1920
namespace Microsoft.Azure.Commands.DataMigration.Cmdlets
2021
{
21-
22-
using MongoDbCollectionSettingItem = System.Collections.Generic.KeyValuePair<string, MongoDbCollectionSettings>;
23-
2422
/// <summary>
2523
/// Class that creates a new instance of the mongo db collection info
2624
/// </summary>
27-
[Cmdlet(VerbsCommon.New, "AzureRmDataMigrationMongoDbCollectionSetting", SupportsShouldProcess = true), OutputType(typeof(MongoDbCollectionSettingItem))]
25+
[Cmdlet(VerbsCommon.New, "AzureRmDataMigrationMongoDbCollectionSetting", SupportsShouldProcess = true), OutputType(typeof(MongoDbCollectionSetting))]
2826
[Alias("New-AzureRmDmsMongoDbCollectionSetting")]
2927
public class NewMongoDbCollectionSettingCmdlet : DataMigrationCmdlet
3028
{
@@ -40,7 +38,7 @@ public class NewMongoDbCollectionSettingCmdlet : DataMigrationCmdlet
4038
HelpMessage = "The database level shared RU at the target CosmosDb"
4139
)]
4240
[Alias("RU")]
43-
public int? TargetRU { get; set; }
41+
public int? TargetRequestUnit { get; set; }
4442

4543
[Parameter(
4644
Mandatory = false,
@@ -60,8 +58,7 @@ public class NewMongoDbCollectionSettingCmdlet : DataMigrationCmdlet
6058
Mandatory = false,
6159
HelpMessage = "Comma seperated field names to represent shard key to be created, with format of 'a:-1,b:1,c' where -1/1 is for order"
6260
)]
63-
[Alias("ShardFields")]
64-
public string ShardKeyFields { get; set; }
61+
public string ShardKey { get; set; }
6562

6663
public override void ExecuteCmdlet()
6764
{
@@ -70,15 +67,15 @@ public override void ExecuteCmdlet()
7067
base.ExecuteCmdlet();
7168
var setting = new MongoDbCollectionSettings()
7269
{
73-
TargetRUs = this.TargetRU,
70+
TargetRUs = this.TargetRequestUnit,
7471
CanDelete = this.CanDelete.IsPresent,
75-
ShardKey = new MongoDbShardKeySetting(this._parseShardKeyFields(this.ShardKeyFields), this.UniqueShard.IsPresent)
72+
ShardKey = new MongoDbShardKeySetting(this._parseShardKey(this.ShardKey), this.UniqueShard.IsPresent)
7673
};
77-
WriteObject(new MongoDbCollectionSettingItem(this.CollectionName, setting));
74+
WriteObject(new MongoDbCollectionSetting { Name = this.CollectionName, Setting = setting });
7875
}
7976
}
8077

81-
private System.Collections.Generic.List<MongoDbShardKeyField> _parseShardKeyFields(string shardKeyStr)
78+
private System.Collections.Generic.List<MongoDbShardKeyField> _parseShardKey(string shardKeyStr)
8279
{
8380
var res = new System.Collections.Generic.List<MongoDbShardKeyField>();
8481
if (!string.IsNullOrWhiteSpace(shardKeyStr))

‎src/ResourceManager/DataMigration/Commands.DataMigration/Cmdlets/NewMongoDbDatabaseSettingCmdlet.cs

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -14,16 +14,14 @@
1414

1515
using System.Management.Automation;
1616
using Microsoft.Azure.Management.DataMigration.Models;
17+
using Microsoft.Azure.Commands.DataMigration.Models.MongoDb;
1718

1819
namespace Microsoft.Azure.Commands.DataMigration.Cmdlets
1920
{
20-
using MongoDbCollectionSettingItem = System.Collections.Generic.KeyValuePair<string, MongoDbCollectionSettings>;
21-
using MongoDbDatabaseSettingItem = System.Collections.Generic.KeyValuePair<string, MongoDbDatabaseSettings>;
22-
2321
/// <summary>
2422
/// Class that creates a new instance of the mongo db target database setting
2523
/// </summary>
26-
[Cmdlet(VerbsCommon.New, "AzureRmDataMigrationMongoDbDatabaseSetting", SupportsShouldProcess = true), OutputType(typeof(MongoDbDatabaseSettingItem))]
24+
[Cmdlet(VerbsCommon.New, "AzureRmDataMigrationMongoDbDatabaseSetting", SupportsShouldProcess = true), OutputType(typeof(MongoDbDatabaseSetting))]
2725
[Alias("New-AzureRmDmsMongoDbDatabaseSetting")]
2826
public class NewMongoDbDatabaseSettingCmdlet : DataMigrationCmdlet
2927
{
@@ -39,14 +37,14 @@ public class NewMongoDbDatabaseSettingCmdlet : DataMigrationCmdlet
3937
HelpMessage = "The database level shared RU at the target CosmosDb"
4038
)]
4139
[Alias("RU")]
42-
public int? TargetRU { get; set; }
40+
public int? TargetRequestUnit { get; set; }
4341

4442
[Parameter(
4543
Mandatory = false,
4644
HelpMessage = "The collection settings for the databases, refer to: New-AzureRmDmsMongoCollectionSetting"
4745
)]
4846
[Alias("Coll")]
49-
public MongoDbCollectionSettingItem[] Collections { get; set; }
47+
public MongoDbCollectionSetting[] CollectionSetting { get; set; }
5048

5149
public override void ExecuteCmdlet()
5250
{
@@ -55,20 +53,19 @@ public override void ExecuteCmdlet()
5553
base.ExecuteCmdlet();
5654
var dbInfo = new MongoDbDatabaseSettings()
5755
{
58-
TargetRUs = this.TargetRU,
56+
TargetRUs = this.TargetRequestUnit,
5957
Collections = new System.Collections.Generic.Dictionary<string, MongoDbCollectionSettings>()
6058
};
6159

62-
var dbInfoItem = new System.Collections.Generic.KeyValuePair<string, MongoDbDatabaseSettings>(this.DatabaseName, dbInfo);
63-
if (this.Collections != null && this.Collections.Length > 0)
60+
if (this.CollectionSetting != null && this.CollectionSetting.Length > 0)
6461
{
65-
foreach(var col in this.Collections)
62+
foreach(var col in this.CollectionSetting)
6663
{
67-
dbInfoItem.Value.Collections.Add(col);
64+
dbInfo.Collections.Add(col.Name, col.Setting);
6865
}
6966
}
7067

71-
WriteObject(dbInfoItem);
68+
WriteObject(new MongoDbDatabaseSetting { Name = this.DatabaseName, Setting = dbInfo });
7269
}
7370
}
7471
}

‎src/ResourceManager/DataMigration/Commands.DataMigration/Cmdlets/ValidateMongoDbMigrationTaskCmdlet.cs

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,12 @@
55
// --------------------------------------------------------------------------------------------------------------------
66

77
using System.Management.Automation;
8+
using Microsoft.Azure.Commands.DataMigration.Models.MongoDb;
89
using Microsoft.Azure.Management.DataMigration.Models;
910
using PSModels = Microsoft.Azure.Commands.DataMigration.Models;
1011

1112
namespace Microsoft.Azure.Commands.DataMigration.Cmdlets
1213
{
13-
using MongoDbDatabaseDictionary = System.Collections.Generic.Dictionary<string, MongoDbDatabaseSettings>;
14-
using MongoDbDatabaseSettingItem = System.Collections.Generic.KeyValuePair<string, MongoDbDatabaseSettings>;
15-
1614
public class ValidateMongoDbMigrationTaskCmdlet : TaskCmdlet<PSModels.MongoDbConnectionInfo>
1715
{
1816
private readonly string Replication = "Replication";
@@ -27,7 +25,7 @@ public override void CustomInit()
2725
this.SimpleParam(SourceConnection, typeof(PSModels.MongoDbConnectionInfo), "Source Connection Info Detail", true);
2826
this.SimpleParam(TargetConnection, typeof(PSModels.MongoDbConnectionInfo), "Target Connection Info Detail", true);
2927
this.SimpleParam(Replication, typeof(string), "type of migration, valid value: OneTime, Continous, or Disabled, default is OneTime");
30-
this.SimpleParam(SelectedDatabase, typeof(MongoDbDatabaseSettingItem[]), "Selected database to migrate", true);
28+
this.SimpleParam(SelectedDatabase, typeof(MongoDbDatabaseSetting[]), "Selected database to migrate", true);
3129
}
3230

3331
protected virtual ProjectTaskProperties CreateTaskProperties(MongoDbMigrationSettings input)
@@ -45,15 +43,15 @@ public override ProjectTaskProperties ProcessTaskCmdlet()
4543
Source = new MongoDbConnectionInfo { ConnectionString = source.ConnectionString },
4644
Target = new MongoDbConnectionInfo { ConnectionString = target.ConnectionString },
4745
Replication = "OneTime",
48-
Databases = new MongoDbDatabaseDictionary()
46+
Databases = new System.Collections.Generic.Dictionary<string, MongoDbDatabaseSettings>()
4947
};
5048

5149
if (MyInvocation.BoundParameters.ContainsKey(SelectedDatabase))
5250
{
53-
var items = MyInvocation.BoundParameters[SelectedDatabase] as MongoDbDatabaseSettingItem[];
51+
var items = MyInvocation.BoundParameters[SelectedDatabase] as MongoDbDatabaseSetting[];
5452
foreach (var i in items)
5553
{
56-
input.Databases.Add(i);
54+
input.Databases.Add(i.Name, i.Setting);
5755
}
5856
}
5957

‎src/ResourceManager/DataMigration/Commands.DataMigration/Commands.DataMigration.csproj

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,8 @@
9595
<Compile Include="Models\Exceptions\InvalidConnectionTypeException.cs" />
9696
<Compile Include="Models\Exceptions\PreConditionFailedException.cs" />
9797
<Compile Include="Models\DatabaseInfo\PSDatabaseInfo.cs" />
98+
<Compile Include="Models\MongoDb\MongoDbDatabaseSetting.cs" />
99+
<Compile Include="Models\MongoDb\MongoDbCollectionSetting.cs" />
98100
<Compile Include="Models\PSProject.cs" />
99101
<Compile Include="Models\PSDataMigrationService.cs" />
100102
<Compile Include="Models\PSProjectTask.cs" />
@@ -122,8 +124,8 @@
122124
</ItemGroup>
123125
<ItemGroup>
124126
<None Include="help\Invoke-AzureRmDataMigrationCommand.md" />
125-
<None Include="help\New-AzureRmDataMigrationMongoDbDatabaseSetting .md" />
126-
<None Include="help\New-AzureRmDataMigrationMongoDbCollectionSetting .md" />
127+
<None Include="help\New-AzureRmDataMigrationMongoDbDatabaseSetting.md" />
128+
<None Include="help\New-AzureRmDataMigrationMongoDbCollectionSetting.md" />
127129
<None Include="help\New-AzureRmDataMigrationSelectedDBObject.md" />
128130
<None Include="help\New-AzureRmDataMigrationSyncSelectedDBObject.md" />
129131
<None Include="MSSharedLibKey.snk" />

‎src/ResourceManager/DataMigration/Commands.DataMigration/Models/MongoDb/MongoDbCollectionSetting.cs

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
namespace Microsoft.Azure.Commands.DataMigration.Models.MongoDb
2+
{
3+
public class MongoDbCollectionSetting
4+
{
5+
public string Name { get; set; }
6+
7+
public Microsoft.Azure.Management.DataMigration.Models.MongoDbCollectionSettings Setting { get; set; }
8+
}
9+
}

‎src/ResourceManager/DataMigration/Commands.DataMigration/Models/MongoDb/MongoDbDatabaseSetting.cs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
namespace Microsoft.Azure.Commands.DataMigration.Models.MongoDb
2+
{
3+
public class MongoDbDatabaseSetting
4+
{
5+
public string Name { get; set; }
6+
public Microsoft.Azure.Management.DataMigration.Models.MongoDbDatabaseSettings Setting { get; set; }
7+
}
8+
}

‎src/ResourceManager/DataMigration/Commands.DataMigration/help/New-AzureRmDataMigrationMongoDbCollectionSetting .md renamed to ‎src/ResourceManager/DataMigration/Commands.DataMigration/help/New-AzureRmDataMigrationMongoDbCollectionSetting.md

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ Creates collection setting for migration according for the mongoDb migration
1313
## SYNTAX
1414

1515
```
16-
New-AzureRmDataMigrationMongoDbCollectionSetting -Name <Name> [-TargetRU <TargetRU>] [-CanDelete] [-ShardKeyFields <ShardKeyFields>]
16+
New-AzureRmDataMigrationMongoDbCollectionSetting -Name <Name> [-TargetRequestUnit <TargetRequestUnit>] [-CanDelete] [-ShardKey <ShardKey>]
1717
```
1818

1919
## DESCRIPTION
@@ -25,7 +25,7 @@ the database level settings for migration.
2525

2626
### Example 1
2727
```
28-
PS C:\> New-AzureRmDataMigrationMongoDbCollectionSetting -Name myCollection -TargetRU 1000 -CanDelete -ShardKeyFields "_id:-1,age:1,name"
28+
PS C:\> New-AzureRmDataMigrationMongoDbCollectionSetting -Name myCollection -TargetRequestUnit 1000 -CanDelete -ShardKey "_id:-1,age:1,name"
2929
3030
```
3131

@@ -46,7 +46,7 @@ Accept pipeline input: False
4646
Accept wildcard characters: False
4747
```
4848
49-
### -ShardKeyFields
49+
### -ShardKey
5050
The comma seperated list of the shard keys. For mongoDb target,
5151
you can specify shard key order of "ShardKeyName:Order",
5252
where order is 1, -1 or empty for hashed, for example "_id,email:-1".
@@ -63,7 +63,7 @@ Accept pipeline input: False
6363
Accept wildcard characters: False
6464
```
6565
66-
### -TargetRU
66+
### -TargetRequestUnit
6767
The dedicated collection request unit value. If not set, that collection uses shared database RU.
6868
6969
```yaml
@@ -103,7 +103,7 @@ This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable
103103
104104
## OUTPUTS
105105
106-
### System.Collections.Generic.KeyValuePair<string, MongoDbCollectionSettings>
106+
### Microsoft.Azure.Commands.DataMigration.Models.MongoDbCollectionSetting>
107107
108108
## NOTES
109109

‎src/ResourceManager/DataMigration/Commands.DataMigration/help/New-AzureRmDataMigrationMongoDbDatabaseSetting .md renamed to ‎src/ResourceManager/DataMigration/Commands.DataMigration/help/New-AzureRmDataMigrationMongoDbDatabaseSetting.md

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ Creates database setting for migration for the mongoDb migration
1313
## SYNTAX
1414

1515
```
16-
New-AzureRmDataMigrationMongoDbDatabaseSetting -Name <Name> [-RU <RU>] -Collections <Collections>
16+
New-AzureRmDataMigrationMongoDbDatabaseSetting -Name <Name> [-RU <RU>] -CollectionSetting <Collections>
1717
```
1818

1919
## DESCRIPTION
@@ -24,7 +24,7 @@ The output is a key value pair with name of collection and value of the setting,
2424

2525
### Example 1
2626
```
27-
PS C:\> New-AzureRmDataMigrationMongoDbDatabaseSetting -Name mycollection -RU 1000 -Collections @(coll1, coll2)
27+
PS C:\> New-AzureRmDataMigrationMongoDbDatabaseSetting -Name mycollection -RU 1000 -CollectionSetting @(coll1, coll2)
2828
2929
```
3030

@@ -44,7 +44,7 @@ Default value: None
4444
Accept pipeline input: False
4545
Accept wildcard characters: False
4646
```
47-
### -TargetRU
47+
### -TargetRequestUnit
4848
The dedicated database level request unit value. If not set, that collection uses shared database RU.
4949
5050
```yaml
@@ -83,7 +83,7 @@ This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable
8383
8484
## OUTPUTS
8585
86-
### System.Collections.Generic.KeyValuePair<string, MongoDbDatabaseSettings>
86+
### Microsoft.Azure.Commands.DataMigration.Models.MongoDbDatabaseSetting
8787
8888
## NOTES
8989

0 commit comments

Comments
 (0)