fix: replace node buffers with uint8arrays (#251)

BREAKING CHANGES:

- All use of node Buffers have been replaced with Uint8Arrays
- All deps now use Uint8Arrays in place of node Buffers

Author: achingbrain

.travis.yml

@@ -1,15 +1,20 @@
 language: node_js
 cache: npm
 
+branches:
+  only:
+  - master
+  - /^release\/.*$/
+
+node_js:
+  - 'lts/*'
+  - 'node'
+
 stages:
   - check
   - test
   - cov
 
-node_js:
-  - '12'
-  - '10'
-
 os:
   - linux
   - osx

package.json

@@ -43,20 +43,19 @@
   "homepage": "https://github.com/ipfs/js-ipfs-bitswap#readme",
   "devDependencies": {
     "@nodeutils/defaults-deep": "^1.1.0",
-    "aegir": "^25.0.0",
+    "aegir": "^26.0.0",
     "benchmark": "^2.1.4",
-    "buffer": "^5.6.0",
     "delay": "^4.3.0",
-    "ipfs-repo": "^4.0.0",
-    "ipfs-utils": "^2.2.0",
+    "ipfs-repo": "^6.0.1",
+    "ipfs-utils": "^3.0.0",
     "iso-random-stream": "^1.1.1",
     "it-all": "^1.0.2",
     "it-drain": "^1.0.1",
-    "libp2p": "^0.28.0",
-    "libp2p-kad-dht": "^0.19.1",
-    "libp2p-mplex": "^0.9.2",
-    "libp2p-secio": "^0.12.1",
-    "libp2p-tcp": "^0.14.2",
+    "libp2p": "libp2p/js-libp2p#0.29.x",
+    "libp2p-kad-dht": "^0.20.0",
+    "libp2p-mplex": "^0.10.0",
+    "libp2p-secio": "^0.13.0",
+    "libp2p-tcp": "^0.15.0",
     "lodash.difference": "^4.5.0",
     "lodash.flatten": "^4.4.0",
     "lodash.range": "^3.2.0",
@@ -65,7 +64,7 @@
     "p-defer": "^3.0.0",
     "p-event": "^4.1.0",
     "p-wait-for": "^3.1.0",
-    "peer-id": "^0.13.5",
+    "peer-id": "^0.14.0",
     "promisify-es6": "^1.0.3",
     "rimraf": "^3.0.0",
     "sinon": "^9.0.0",
@@ -76,19 +75,20 @@
     "abort-controller": "^3.0.0",
     "any-signal": "^1.1.0",
     "bignumber.js": "^9.0.0",
-    "cids": "^0.8.3",
+    "cids": "^1.0.0",
     "debug": "^4.1.0",
-    "ipld-block": "^0.9.1",
+    "ipld-block": "^0.10.0",
     "it-length-prefixed": "^3.0.0",
     "it-pipe": "^1.1.0",
     "just-debounce-it": "^1.1.0",
-    "libp2p-interfaces": "^0.3.0",
+    "libp2p-interfaces": "^0.4.1",
     "moving-average": "^1.0.0",
-    "multicodec": "^1.0.3",
-    "multihashing-async": "^1.0.0",
-    "protons": "^1.0.1",
+    "multicodec": "^2.0.0",
+    "multihashing-async": "^2.0.1",
+    "protons": "^2.0.0",
     "streaming-iterables": "^5.0.2",
-    "varint-decoder": "^0.4.0"
+    "uint8arrays": "^1.1.0",
+    "varint-decoder": "^1.0.0"
   },
   "pre-push": [
     "lint",

src/notifications.js

@@ -2,13 +2,14 @@
 
 const EventEmitter = require('events').EventEmitter
 const Block = require('ipld-block')
+const uint8ArrayEquals = require('uint8arrays/equals')
+const uint8ArrayToString = require('uint8arrays/to-string')
 
 const CONSTANTS = require('./constants')
 const logger = require('./utils').logger
 
-const cidToMultihashString = (cid) => cid.multihash.toString('base64')
-const unwantEvent = (cid) => `unwant:${cidToMultihashString(cid)}`
-const blockEvent = (cid) => `block:${cidToMultihashString(cid)}`
+const unwantEvent = (cid) => `unwant:${uint8ArrayToString(cid.multihash, 'base64')}`
+const blockEvent = (cid) => `block:${uint8ArrayToString(cid.multihash, 'base64')}`
 
 /**
  * Internal module used to track events about incoming blocks,
@@ -67,7 +68,7 @@ class Notifications extends EventEmitter {
       const onBlock = (block) => {
         this.removeListener(unwantEvt, onUnwant)
 
-        if (!cid.multihash.equals(block.cid.multihash)) {
+        if (!uint8ArrayEquals(cid.multihash, block.cid.multihash)) {
           // wrong block
           return reject(new Error(`Incorrect block received for ${cid}`))
         } else if (cid.version !== block.cid.version || cid.codec !== block.cid.codec) {

src/stats/stat.js

@@ -91,9 +91,11 @@ class Stats extends EventEmitter {
   _updateFrequency (latestTime) {
     const timeDiff = latestTime - this._frequencyLastTime
 
-    Object.keys(this._stats).forEach((key) => {
-      this._updateFrequencyFor(key, timeDiff, latestTime)
-    })
+    if (timeDiff) {
+      Object.keys(this._stats).forEach((key) => {
+        this._updateFrequencyFor(key, timeDiff, latestTime)
+      })
+    }
 
     this._frequencyLastTime = latestTime
   }

src/types/message/index.js

@@ -91,7 +91,7 @@ class BitswapMessage {
       wantlist: {
         entries: Array.from(this.wantlist.values()).map((entry) => {
           return {
-            block: entry.cid.buffer, // cid
+            block: entry.cid.bytes, // cid
             priority: Number(entry.priority),
             cancel: Boolean(entry.cancel)
           }
@@ -117,7 +117,7 @@ class BitswapMessage {
       wantlist: {
         entries: Array.from(this.wantlist.values()).map((entry) => {
           return {
-            block: entry.cid.buffer, // cid
+            block: entry.cid.bytes, // cid
             priority: Number(entry.priority),
             wantType: entry.wantType,
             cancel: Boolean(entry.cancel),
@@ -142,7 +142,7 @@ class BitswapMessage {
 
     for (const [cidStr, bpType] of this.blockPresences) {
       msg.blockPresences.push({
-        cid: new CID(cidStr).buffer,
+        cid: new CID(cidStr).bytes,
         type: bpType
       })
     }
@@ -237,7 +237,7 @@ BitswapMessage.blockPresenceSize = (cid) => {
   // the HAVE / DONT_HAVE on the wire, but when doing that calculation we leave
   // plenty of padding under the maximum message size.
   // (It's more important for this to be fast).
-  return cid.buffer.length + 1
+  return cid.bytes.length + 1
 }
 
 BitswapMessage.Entry = Entry

src/utils/index.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const debug = require('debug')
+const uint8ArrayEquals = require('uint8arrays/equals')
 
 /**
  * Creates a logger for the given subsystem
@@ -103,7 +104,7 @@ const isMapEqual = (a, b) => {
       return false
     }
     // Support Blocks
-    if (valueA._data && !valueA._data.equals(valueB._data)) {
+    if (valueA._data && !uint8ArrayEquals(valueA._data, valueB._data)) {
       return false
     }
   }

test/bitswap-mock-internals.js

@@ -332,7 +332,6 @@ describe('bitswap with mocks', function () {
       setTimeout(() => {
         bs2.put(block, () => {})
       }, 1000)
-
       const b1 = await p1
       expect(b1).to.eql(block)
 

test/bitswap-stats.js

@@ -73,10 +73,10 @@ describe('bitswap stats', () => {
       bitswaps.map((bs) => bs.stop())
     )
     await Promise.all(
-      repos.map(repo => repo.teardown())
+      libp2pNodes.map((n) => n.stop())
     )
     await Promise.all(
-      libp2pNodes.map((n) => n.stop())
+      repos.map(repo => repo.teardown())
     )
   })
 
@@ -216,10 +216,13 @@ describe('bitswap stats', () => {
       const peerStats = bs2.stat().forPeer(libp2pNodes[0].peerId)
       expect(peerStats).to.exist()
 
+      // trigger an update
+      peerStats.push('dataReceived', 1)
+
       const stats = await pEvent(peerStats, 'update')
 
       expect(stats.blocksReceived.toNumber()).to.equal(1)
-      expect(stats.dataReceived.toNumber()).to.equal(48)
+      expect(stats.dataReceived.toNumber()).to.equal(49)
       expect(stats.dupBlksReceived.toNumber()).to.equal(0)
       expect(stats.dupDataReceived.toNumber()).to.equal(0)
       expect(stats.blocksSent.toNumber()).to.equal(0)

test/decision-engine/decision-engine.js

@@ -9,7 +9,8 @@ const flatten = require('lodash.flatten')
 const Block = require('ipld-block')
 const CID = require('cids')
 const multihashing = require('multihashing-async')
-const { Buffer } = require('buffer')
+const uint8ArrayFromString = require('uint8arrays/from-string')
+const uint8ArrayToString = require('uint8arrays/to-string')
 const drain = require('it-drain')
 
 const Message = require('../../src/types/message')
@@ -23,7 +24,7 @@ const sum = (nums) => nums.reduce((a, b) => a + b, 0)
 
 function messageToString (m) {
   return Array.from(m[1].blocks.values())
-    .map((b) => b.data.toString())
+    .map((b) => uint8ArrayToString(b.data))
 }
 
 function stringifyMessages (messages) {
@@ -53,7 +54,7 @@ describe('Engine', () => {
     const receiver = res[1]
 
     await Promise.all(range(1000).map(async (i) => {
-      const data = Buffer.from(`this is message ${i}`)
+      const data = uint8ArrayFromString(`this is message ${i}`)
       const hash = await multihashing(data, 'sha2-256')
 
       const m = new Message(false)
@@ -110,7 +111,7 @@ describe('Engine', () => {
     async function partnerWants (dEngine, values, partner) {
       const message = new Message(false)
 
-      const hashes = await Promise.all(values.map((v) => multihashing(Buffer.from(v), 'sha2-256')))
+      const hashes = await Promise.all(values.map((v) => multihashing(uint8ArrayFromString(v), 'sha2-256')))
       hashes.forEach((hash, i) => {
         message.addEntry(new CID(hash), Math.pow(2, 32) - 1 - i)
       })
@@ -120,7 +121,7 @@ describe('Engine', () => {
     async function partnerCancels (dEngine, values, partner) {
       const message = new Message(false)
 
-      const hashes = await Promise.all(values.map((v) => multihashing(Buffer.from(v), 'sha2-256')))
+      const hashes = await Promise.all(values.map((v) => multihashing(uint8ArrayFromString(v), 'sha2-256')))
       hashes.forEach((hash) => {
         message.cancel(new CID(hash))
       })
@@ -134,8 +135,8 @@ describe('Engine', () => {
       await dEngine.receivedBlocks(blocks)
     }
 
-    const hashes = await Promise.all(alphabet.map(v => multihashing(Buffer.from(v), 'sha2-256')))
-    const blocks = hashes.map((h, i) => new Block(Buffer.from(alphabet[i]), new CID(h)))
+    const hashes = await Promise.all(alphabet.map(v => multihashing(uint8ArrayFromString(v), 'sha2-256')))
+    const blocks = hashes.map((h, i) => new Block(uint8ArrayFromString(alphabet[i]), new CID(h)))
     const partner = await PeerId.create({ bits: 512 })
     const somePeer = await PeerId.create({ bits: 512 })
 
@@ -349,8 +350,8 @@ describe('Engine', () => {
     const vowels = 'aeiou'
 
     const alphabetLs = alphabet.split('')
-    const hashes = await Promise.all(alphabetLs.map(v => multihashing(Buffer.from(v), 'sha2-256')))
-    const blocks = hashes.map((h, i) => new Block(Buffer.from(alphabetLs[i]), new CID(h)))
+    const hashes = await Promise.all(alphabetLs.map(v => multihashing(uint8ArrayFromString(v), 'sha2-256')))
+    const blocks = hashes.map((h, i) => new Block(uint8ArrayFromString(alphabetLs[i]), new CID(h)))
 
     let testCases = [
       // Just send want-blocks
@@ -603,7 +604,7 @@ describe('Engine', () => {
       let i = wantBlks.length + wantHaves.length
       const message = new Message(false)
       for (const [wants, type] of wantTypes) {
-        const hashes = await Promise.all(wants.map((v) => multihashing(Buffer.from(v), 'sha2-256')))
+        const hashes = await Promise.all(wants.map((v) => multihashing(uint8ArrayFromString(v), 'sha2-256')))
         for (const hash of hashes) {
           message.addEntry(new CID(hash), i--, type, false, sendDontHave)
         }
@@ -661,21 +662,21 @@ describe('Engine', () => {
 
       // Expect the correct block contents
       for (const expBlk of expBlks) {
-        const hash = await multihashing(Buffer.from(expBlk), 'sha2-256')
+        const hash = await multihashing(uint8ArrayFromString(expBlk), 'sha2-256')
         expect(msg.blocks.has(new CID(hash).toString()))
       }
 
       // Expect the correct HAVEs
       for (const expHave of expHaves) {
-        const hash = await multihashing(Buffer.from(expHave), 'sha2-256')
+        const hash = await multihashing(uint8ArrayFromString(expHave), 'sha2-256')
         const cid = new CID(hash).toString()
         expect(msg.blockPresences.has(cid)).to.eql(true)
         expect(msg.blockPresences.get(cid)).to.eql(Message.BlockPresenceType.Have)
       }
 
       // Expect the correct DONT_HAVEs
       for (const expDontHave of expDontHaves) {
-        const hash = await multihashing(Buffer.from(expDontHave), 'sha2-256')
+        const hash = await multihashing(uint8ArrayFromString(expDontHave), 'sha2-256')
         const cid = new CID(hash).toString()
         expect(msg.blockPresences.has(cid)).to.eql(true)
         expect(msg.blockPresences.get(cid)).to.eql(Message.BlockPresenceType.DontHave)
@@ -700,7 +701,7 @@ describe('Engine', () => {
     const them = await newEngine()
 
     // add a block to our blockstore
-    const data = Buffer.from(`this is message ${Date.now()}`)
+    const data = uint8ArrayFromString(`this is message ${Date.now()}`)
     const hash = await multihashing(data, 'sha2-256')
     const cid = new CID(hash)
     const block = new Block(data, cid)

test/network/gen-bitswap-network.node.js

@@ -4,7 +4,6 @@
 
 const { expect } = require('aegir/utils/chai')
 const Block = require('ipld-block')
-const { Buffer } = require('buffer')
 const crypto = require('crypto')
 const CID = require('cids')
 const multihashing = require('multihashing-async')
@@ -21,7 +20,7 @@ describe('gen Bitswap network', function () {
 
     const node = nodes[0]
     const blocks = await Promise.all(range(100).map(async (k) => {
-      const b = Buffer.alloc(1024)
+      const b = new Uint8Array(1024)
       b.fill(k)
       const hash = await multihashing(b, 'sha2-256')
       const cid = new CID(hash)