Skip to content
This repository was archived by the owner on Oct 1, 2021. It is now read-only.

Commit 540a077

Browse files
achingbrainachingbrain
authored
feat: migrate mfs root to datastore (#126)
We've been storing the mfs root in the repo's 'root' datastore since forever which means it's a block on the filesystem. It should have been stored in the datastore instead so add migration 11 which moves the key to the datastore. BREAKING CHANGE: adds a new migration, should go out as a major
1 parent d3c0056 commit 540a077

File tree

13 files changed

+142
-18
lines changed

13 files changed

+142
-18
lines changed

‎migrations/index.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,5 +22,6 @@ module.exports = [
2222
Object.assign({version: 7}, emptyMigration),
2323
require('./migration-8'),
2424
require('./migration-9'),
25-
require('./migration-10')
25+
require('./migration-10'),
26+
require('./migration-11')
2627
]

‎migrations/migration-10/index.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33
const {
44
findLevelJs
55
} = require('../../src/utils')
6-
const fromString = require('uint8arrays/from-string')
7-
const toString = require('uint8arrays/to-string')
6+
const { fromString } = require('uint8arrays/from-string')
7+
const { toString } = require('uint8arrays/to-string')
88

99
/**
1010
* @typedef {import('../../src/types').Migration} Migration

‎migrations/migration-11/index.js

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
'use strict'
2+
3+
const { Key } = require('interface-datastore')
4+
5+
const MFS_ROOT_KEY = new Key('/local/filesroot')
6+
7+
/**
8+
* @param {import('../../src/types').Backends} backends
9+
* @param {import('../../src/types').MigrationProgressCallback} onProgress
10+
*/
11+
async function storeMfsRootInDatastore (backends, onProgress = () => {}) {
12+
onProgress(100, 'Migrating MFS root to repo datastore')
13+
14+
await backends.root.open()
15+
await backends.datastore.open()
16+
17+
const root = await backends.root.get(MFS_ROOT_KEY)
18+
await backends.datastore.put(MFS_ROOT_KEY, root)
19+
await backends.root.delete(MFS_ROOT_KEY)
20+
21+
await backends.datastore.close()
22+
await backends.root.close()
23+
24+
onProgress(100, 'Stored MFS root in repo datastore')
25+
}
26+
27+
/**
28+
* @param {import('../../src/types').Backends} backends
29+
* @param {import('../../src/types').MigrationProgressCallback} onProgress
30+
*/
31+
async function storeMfsRootInRoot (backends, onProgress = () => {}) {
32+
onProgress(100, 'Migrating MFS root to repo root datastore')
33+
34+
await backends.root.open()
35+
await backends.datastore.open()
36+
37+
const root = await backends.datastore.get(MFS_ROOT_KEY)
38+
await backends.root.put(MFS_ROOT_KEY, root)
39+
await backends.datastore.delete(MFS_ROOT_KEY)
40+
41+
await backends.datastore.close()
42+
await backends.root.close()
43+
44+
onProgress(100, 'Stored MFS root in repo root datastore')
45+
}
46+
47+
/** @type {import('../../src/types').Migration} */
48+
module.exports = {
49+
version: 11,
50+
description: 'Store mfs root in the datastore',
51+
migrate: storeMfsRootInDatastore,
52+
revert: storeMfsRootInRoot
53+
}

‎migrations/migration-9/pin-set.js

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,10 @@ const fnv1a = require('fnv1a')
1414
const varint = require('varint')
1515
const dagPb = require('@ipld/dag-pb')
1616
const { DEFAULT_FANOUT, MAX_ITEMS, EMPTY_KEY } = require('./utils')
17-
const uint8ArrayConcat = require('uint8arrays/concat')
18-
const uint8ArrayCompare = require('uint8arrays/compare')
19-
const uint8ArrayToString = require('uint8arrays/to-string')
20-
const uint8ArrayFromString = require('uint8arrays/from-string')
17+
const { concat: uint8ArrayConcat } = require('uint8arrays/concat')
18+
const { compare: uint8ArrayCompare } = require('uint8arrays/compare')
19+
const { toString: uint8ArrayToString } = require('uint8arrays/to-string')
20+
const { fromString: uint8ArrayFromString } = require('uint8arrays/from-string')
2121
const { sha256 } = require('multiformats/hashes/sha2')
2222

2323
/**

‎package.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,14 +56,14 @@
5656
"multiformats": "^9.0.0",
5757
"proper-lockfile": "^4.1.1",
5858
"protobufjs": "^6.10.2",
59-
"uint8arrays": "^2.0.5",
59+
"uint8arrays": "^3.0.0",
6060
"varint": "^6.0.0"
6161
},
6262
"devDependencies": {
6363
"@ipld/car": "^3.0.0",
6464
"@types/debug": "^4.1.5",
6565
"@types/varint": "^6.0.0",
66-
"aegir": "^34.1.0",
66+
"aegir": "^35.0.1",
6767
"assert": "^2.0.0",
6868
"aws-sdk": "^2.884.0",
6969
"blockstore-datastore-adapter": "1.0.0",

‎src/repo/version.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33
const repoInit = require('./init')
44
const { MissingRepoOptionsError, NotInitializedRepoError } = require('../errors')
55
const { VERSION_KEY } = require('../utils')
6-
const uint8ArrayFromString = require('uint8arrays/from-string')
7-
const uint8ArrayToString = require('uint8arrays/to-string')
6+
const { fromString: uint8ArrayFromString } = require('uint8arrays/from-string')
7+
const { toString: uint8ArrayToString } = require('uint8arrays/to-string')
88

99
/**
1010
* Function that has responsibility to retrieve version of repo from its root datastore's instance.

‎test/init-test.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
const { expect } = require('aegir/utils/chai')
55
const { CONFIG_KEY, VERSION_KEY } = require('../src/utils')
66
const repoInit = require('../src/repo/init')
7-
const uint8ArrayFromString = require('uint8arrays/from-string')
7+
const { fromString: uint8ArrayFromString } = require('uint8arrays/from-string')
88

99
module.exports = (setup, cleanup) => {
1010
let dir

‎test/migrations/index.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,5 @@ module.exports = (setup, cleanup) => {
44
require('./migration-8-test')(setup, cleanup)
55
require('./migration-9-test')(setup, cleanup)
66
require('./migration-10-test')(setup, cleanup)
7+
require('./migration-11-test')(setup, cleanup)
78
}

‎test/migrations/migration-10-test.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@ const { BlockstoreAdapter } = require('interface-blockstore')
88

99
const migration = require('../../migrations/migration-10')
1010
const Key = require('interface-datastore').Key
11-
const fromString = require('uint8arrays/from-string')
12-
const equals = require('uint8arrays/equals')
11+
const { fromString } = require('uint8arrays/from-string')
12+
const { equals } = require('uint8arrays/equals')
1313
const Level5 = require('level-5')
1414
const Level6 = require('level-6')
1515

‎test/migrations/migration-11-test.js

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
/* eslint-env mocha */
2+
/* eslint-disable max-nested-callbacks */
3+
'use strict'
4+
5+
const { expect } = require('aegir/utils/chai')
6+
const { CID } = require('multiformats/cid')
7+
const migration = require('../../migrations/migration-11')
8+
const { Key } = require('interface-datastore')
9+
10+
const MFS_ROOT_KEY = new Key('/local/filesroot')
11+
const MFS_ROOT = CID.parse('Qmc42sn2WBHYeAShU3nx8mYkhKVq4sRLapawTaGh4XH4iE')
12+
13+
module.exports = (setup, cleanup) => {
14+
describe('migration 11', function () {
15+
this.timeout(240 * 1000)
16+
let dir
17+
let backends
18+
19+
beforeEach(async () => {
20+
({ dir, backends } = await setup())
21+
})
22+
23+
afterEach(async () => {
24+
await cleanup(dir)
25+
})
26+
27+
describe('forwards', () => {
28+
beforeEach(async () => {
29+
await backends.root.open()
30+
await backends.root.put(MFS_ROOT_KEY, MFS_ROOT.bytes)
31+
await backends.root.close()
32+
})
33+
34+
it('should migrate MFS root forward', async () => {
35+
await migration.migrate(backends, () => {})
36+
37+
await backends.root.open()
38+
await backends.datastore.open()
39+
40+
await expect(backends.root.has(MFS_ROOT_KEY)).to.eventually.be.false()
41+
await expect(backends.datastore.has(MFS_ROOT_KEY)).to.eventually.be.true()
42+
43+
await backends.datastore.close()
44+
await backends.root.close()
45+
})
46+
})
47+
48+
describe('backwards', () => {
49+
beforeEach(async () => {
50+
await backends.datastore.open()
51+
await backends.datastore.put(MFS_ROOT_KEY, MFS_ROOT.bytes)
52+
await backends.datastore.close()
53+
})
54+
55+
it('should migrate MFS root backward', async () => {
56+
await migration.revert(backends, () => {})
57+
58+
await backends.root.open()
59+
await backends.datastore.open()
60+
61+
await expect(backends.root.has(MFS_ROOT_KEY)).to.eventually.be.true()
62+
await expect(backends.datastore.has(MFS_ROOT_KEY)).to.eventually.be.false()
63+
64+
await backends.datastore.close()
65+
await backends.root.close()
66+
})
67+
})
68+
})
69+
}

‎test/node.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ const CONFIGURATIONS = [{
3434
new NextToLast(2))
3535
),
3636
datastore: new DatastoreLevel(`${prefix}/datastore`),
37-
keys: new DatastoreLevel(`${prefix}/keys`),
37+
keys: new DatastoreFS(`${prefix}/keys`),
3838
pins: new DatastoreLevel(`${prefix}/pins`)
3939
}
4040
}
@@ -50,7 +50,7 @@ const CONFIGURATIONS = [{
5050
})
5151
),
5252
datastore: new DatastoreLevel(`${prefix}/datastore`),
53-
keys: new DatastoreLevel(`${prefix}/keys`),
53+
keys: new DatastoreFS(`${prefix}/keys`),
5454
pins: new DatastoreLevel(`${prefix}/pins`)
5555
}
5656
}

‎test/test-migrations/migration-2/index.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
const Key = require('interface-datastore').Key
44
const _set = require('just-safe-set')
5-
const uint8ArrayFromString = require('uint8arrays/from-string')
5+
const { fromString: uint8ArrayFromString } = require('uint8arrays/from-string')
66

77
const CONFIG_KEY = new Key('config')
88
const NEW_API_ADDRESS = '/ip6/::/tcp/5001'

‎test/version-test.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
const { expect } = require('aegir/utils/chai')
55
const { VERSION_KEY, CONFIG_KEY } = require('../src/utils')
66
const version = require('../src/repo/version')
7-
const uint8ArrayFromString = require('uint8arrays/from-string')
7+
const { fromString: uint8ArrayFromString } = require('uint8arrays/from-string')
88
const errors = require('../src/errors')
99

1010
// When new versioning mechanism is introduced in new version don't forget to update

0 commit comments

Comments
 (0)