Skip to content

Commit accf53b

Browse files
dirkmcdirkmc
authored
refactor: callbacks -> async / await (#202)
BREAKING CHANGE: All places in the API that used callbacks are now replaced with async/await * feat: make `get()` a generator * make `getMany()` AsyncIterable * feat: make `put()` a generator * make `putMany()` AsyncIterable * remove check in `_findAndConnect()` * feat: make `start()` and `stop()` async/await * refactor: make `connectTo()` async/await * refactor: make `findProviders()` and `findAndConnect()` async/await * refactor: cb => async * refactor: async/await * chore: update travis * refactor: update benchmark tests and allow streaming to putMany * chore: address pr comments * chore: remove callback hell eslint disables * chore: wrap list of tasks in promise.all * chore: callbackify methods inside pull stream * chore: accept PR suggestions * chore: fix typo
1 parent 2e4d69a commit accf53b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+1547
-2189
lines changed

‎.aegir.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
'use strict'
22

33
module.exports = {
4+
bundlesize: { maxSize: '210kB' },
45
karma: {
56
files: [{
67
pattern: 'test/test-data/**/*',

‎.travis.yml

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
language: node_js
22
cache: npm
3+
34
stages:
45
- check
56
- test
67
- cov
78

89
node_js:
10+
- '12'
911
- '10'
1012

1113
os:
@@ -20,7 +22,7 @@ jobs:
2022
include:
2123
- stage: check
2224
script:
23-
- npx aegir commitlint --travis
25+
- npx aegir build --bundlesize
2426
- npx aegir dep-check
2527
- npm run lint
2628

@@ -36,5 +38,17 @@ jobs:
3638
firefox: latest
3739
script: npx aegir test -t browser -t webworker -- --browsers FirefoxHeadless
3840

41+
- stage: test
42+
name: electron-main
43+
os: osx
44+
script:
45+
- npx aegir test -t electron-main --bail
46+
47+
- stage: test
48+
name: electron-renderer
49+
os: osx
50+
script:
51+
- npx aegir test -t electron-renderer --bail
52+
3953
notifications:
4054
email: false

‎benchmarks/index.js

Lines changed: 55 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -1,110 +1,76 @@
1-
/* eslint max-nested-callbacks: ["error", 8] */
21
/* eslint-disable no-console */
32
'use strict'
43

5-
const series = require('async/series')
6-
const parallel = require('async/parallel')
7-
const map = require('async/map')
8-
const mapSeries = require('async/mapSeries')
9-
const each = require('async/each')
10-
const _ = require('lodash')
11-
const Block = require('ipfs-block')
124
const assert = require('assert')
13-
const crypto = require('crypto')
14-
const CID = require('cids')
15-
const multihashing = require('multihashing-async')
5+
const range = require('lodash.range')
166

17-
const utils = require('../test/utils')
7+
const makeBlock = require('../test/utils/make-block')
8+
const genBitswapNetwork = require('../test/utils/mocks').genBitswapNetwork
189

1910
const nodes = [2, 5, 10, 20]
2011
const blockFactors = [1, 10, 100]
2112

22-
console.log('-- start')
23-
mapSeries(nodes, (n, cb) => {
24-
mapSeries(blockFactors, (blockFactor, cb) => {
25-
utils.genBitswapNetwork(n, (err, nodeArr) => {
26-
if (err) {
27-
return cb(err)
28-
}
29-
30-
round(nodeArr, blockFactor, n, (err) => {
31-
if (err) {
32-
return cb(err)
33-
}
34-
35-
shutdown(nodeArr, cb)
36-
})
13+
;(async function () {
14+
console.log('-- start')
15+
await Promise.all(
16+
nodes.map(async nodeCount => {
17+
await Promise.all(
18+
blockFactors.map(async blockFactor => {
19+
const nodeArr = await genBitswapNetwork(nodeCount)
20+
await round(nodeArr, blockFactor, nodeCount)
21+
await shutdown(nodeArr)
22+
})
23+
)
3724
})
38-
}, cb)
39-
}, (err) => {
40-
if (err) {
41-
throw err
42-
}
25+
)
26+
4327
console.log('-- finished')
44-
})
28+
})()
4529

46-
function shutdown (nodeArr, cb) {
47-
each(nodeArr, (node, cb) => {
48-
node.bitswap.stop()
49-
node.libp2p.stop(cb)
50-
}, cb)
30+
async function shutdown (nodeArr) {
31+
await Promise.all(
32+
nodeArr.map(async node => {
33+
await node.bitswap.stop()
34+
await node.libp2p.stop()
35+
})
36+
)
5137
}
5238

53-
function round (nodeArr, blockFactor, n, cb) {
54-
createBlocks(n, blockFactor, (err, blocks) => {
55-
if (err) {
56-
return cb(err)
57-
}
58-
const cids = blocks.map((b) => b.cid)
59-
let d
60-
series([
61-
// put blockFactor amount of blocks per node
62-
(cb) => parallel(_.map(nodeArr, (node, i) => (callback) => {
63-
node.bitswap.start()
64-
65-
const data = _.map(_.range(blockFactor), (j) => {
39+
async function round (nodeArr, blockFactor, n) {
40+
const blocks = await makeBlock(n * blockFactor)
41+
const cids = blocks.map((b) => b.cid)
42+
43+
console.info('put blockFactor amount of blocks per node')
44+
45+
await Promise.all(
46+
nodeArr.map(async (node, i) => {
47+
await node.bitswap.start()
48+
49+
await Promise.all(
50+
range(blockFactor).map(async j => {
6651
const index = i * blockFactor + j
67-
return blocks[index]
68-
})
69-
each(
70-
data,
71-
(d, cb) => node.bitswap.put(d, cb),
72-
callback
73-
)
74-
}), cb),
75-
(cb) => {
76-
d = (new Date()).getTime()
77-
cb()
78-
},
79-
// fetch all blocks on every node
80-
(cb) => parallel(_.map(nodeArr, (node, i) => (callback) => {
81-
map(cids, (cid, cb) => node.bitswap.get(cid, cb), (err, res) => {
82-
if (err) {
83-
return callback(err)
84-
}
85-
86-
assert(res.length === blocks.length)
87-
callback()
52+
53+
await node.bitswap.put(blocks[index])
8854
})
89-
}), cb)
90-
], (err) => {
91-
if (err) {
92-
return cb(err)
93-
}
94-
console.log(' %s nodes - %s blocks/node - %sms', n, blockFactor, (new Date()).getTime() - d)
95-
cb()
55+
)
9656
})
97-
})
98-
}
57+
)
58+
59+
console.info('fetch all blocks on every node')
9960

100-
function createBlocks (n, blockFactor, callback) {
101-
map(_.range(n * blockFactor), (i, cb) => {
102-
const data = crypto.randomBytes(n * blockFactor)
103-
multihashing(data, 'sha2-256', (err, hash) => {
104-
if (err) {
105-
return cb(err)
61+
const d = Date.now()
62+
63+
await Promise.all(
64+
nodeArr.map(async node => {
65+
let count = 0
66+
67+
for await (const _ of node.bitswap.getMany(cids)) { // eslint-disable-line no-unused-vars
68+
count++
10669
}
107-
cb(null, new Block(data, new CID(hash)))
70+
71+
assert(count === blocks.length)
10872
})
109-
}, callback)
73+
)
74+
75+
console.log(' %s nodes - %s blocks/node - %sms', n, blockFactor, Date.now() - d)
11076
}

‎benchmarks/put-get.js

Lines changed: 20 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -1,47 +1,36 @@
1-
/* eslint max-nested-callbacks: ["error", 8] */
1+
/* eslint max-nested-callbacks: ["error", 5] */
22
/* eslint-disable no-console */
33
'use strict'
44

55
const Benchmark = require('benchmark')
6-
const _ = require('lodash')
7-
const Block = require('ipfs-block')
86
const assert = require('assert')
9-
const series = require('async/series')
10-
const map = require('async/map')
11-
const crypto = require('crypto')
12-
const CID = require('cids')
13-
const multihashing = require('multihashing-async')
14-
15-
const utils = require('../test/utils')
7+
const all = require('async-iterator-all')
8+
const makeBlock = require('../test/utils/make-block')
9+
const genBitswapNetwork = require('../test/utils/mocks').genBitswapNetwork
1610

1711
const suite = new Benchmark.Suite('put-get')
1812

1913
const blockCounts = [1, 10, 1000]
2014
const blockSizes = [10, 1024, 10 * 1024]
2115

22-
utils.genBitswapNetwork(1, (err, nodes) => {
23-
if (err) {
24-
throw err
25-
}
26-
const node = nodes[0]
16+
;(async function () {
17+
const [
18+
node
19+
] = await genBitswapNetwork(1)
20+
2721
const bitswap = node.bitswap
2822

2923
blockCounts.forEach((n) => blockSizes.forEach((k) => {
30-
suite.add(`put-get ${n} blocks of size ${k}`, (defer) => {
31-
createBlocks(n, k, (err, blocks) => {
32-
if (err) {
33-
throw err
34-
}
35-
series([
36-
(cb) => bitswap.putMany(blocks, cb),
37-
(cb) => get(blocks, bitswap, cb)
38-
], (err) => {
39-
if (err) {
40-
throw err
41-
}
42-
defer.resolve()
43-
})
44-
})
24+
suite.add(`put-get ${n} blocks of size ${k}`, async (defer) => {
25+
const blocks = await makeBlock(n, k)
26+
27+
await bitswap.putMany(blocks)
28+
29+
const res = await all(bitswap.getMany(blocks.map(block => block.cid)))
30+
31+
assert(res.length === blocks.length)
32+
33+
defer.resolve()
4534
}, {
4635
defer: true
4736
})
@@ -57,29 +46,4 @@ utils.genBitswapNetwork(1, (err, nodes) => {
5746
.run({
5847
async: true
5948
})
60-
})
61-
62-
function createBlocks (n, k, callback) {
63-
map(_.range(n), (i, cb) => {
64-
const data = crypto.randomBytes(k)
65-
multihashing(data, 'sha2-256', (err, hash) => {
66-
if (err) {
67-
return cb(err)
68-
}
69-
cb(null, new Block(data, new CID(hash)))
70-
})
71-
}, callback)
72-
}
73-
74-
function get (blocks, bs, callback) {
75-
map(blocks, (b, cb) => {
76-
bs.get(b.cid, cb)
77-
}, (err, res) => {
78-
if (err) {
79-
return callback(err)
80-
}
81-
82-
assert(res.length === blocks.length)
83-
callback()
84-
})
85-
}
49+
})()

‎package.json

Lines changed: 17 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -43,39 +43,43 @@
4343
"homepage": "https://github.com/ipfs/js-ipfs-bitswap#readme",
4444
"devDependencies": {
4545
"@nodeutils/defaults-deep": "^1.1.0",
46-
"aegir": "^18.2.1",
46+
"aegir": "^20.3.1",
47+
"async-iterator-all": "^1.0.0",
4748
"benchmark": "^2.1.4",
4849
"chai": "^4.2.0",
4950
"dirty-chai": "^2.0.1",
50-
"ipfs-repo": "~0.26.3",
51-
"libp2p": "~0.24.2",
52-
"libp2p-kad-dht": "~0.15.0",
53-
"libp2p-mplex": "~0.8.4",
51+
"ipfs-repo": "^0.28.0",
52+
"libp2p": "^0.26.1",
53+
"libp2p-kad-dht": "^0.16.0",
54+
"libp2p-mplex": "^0.8.0",
5455
"libp2p-secio": "~0.11.1",
55-
"libp2p-tcp": "~0.13.0",
56-
"lodash": "^4.17.11",
56+
"libp2p-tcp": "^0.13.0",
57+
"lodash.difference": "^4.5.0",
58+
"lodash.flatten": "^4.4.0",
5759
"lodash.range": "^3.2.0",
5860
"lodash.without": "^4.4.0",
5961
"ncp": "^2.0.0",
62+
"p-event": "^4.1.0",
6063
"peer-book": "~0.9.0",
61-
"peer-id": "~0.12.0",
62-
"peer-info": "~0.15.0",
63-
"rimraf": "^2.6.2",
64+
"peer-id": "^0.12.2",
65+
"peer-info": "~0.15.1",
66+
"promisify-es6": "^1.0.3",
67+
"rimraf": "^3.0.0",
6468
"safe-buffer": "^5.1.2",
6569
"stats-lite": "^2.2.0",
6670
"uuid": "^3.3.2"
6771
},
6872
"dependencies": {
69-
"async": "^2.6.1",
70-
"bignumber.js": "^8.0.1",
73+
"bignumber.js": "^9.0.0",
74+
"callbackify": "^1.1.0",
7175
"cids": "~0.7.0",
7276
"debug": "^4.1.0",
7377
"ipfs-block": "~0.8.0",
7478
"just-debounce-it": "^1.1.0",
7579
"lodash.isequalwith": "^4.4.0",
7680
"moving-average": "^1.0.0",
7781
"multicodec": "~0.5.0",
78-
"multihashing-async": "~0.5.1",
82+
"multihashing-async": "^0.8.0",
7983
"protons": "^1.0.1",
8084
"pull-length-prefixed": "^1.3.1",
8185
"pull-stream": "^3.6.9",

0 commit comments

Comments
 (0)