Merge pull request #97 from Permissionless-Software-Foundation/ct-unstable

Syncing with upstream ipfs-service-provider

README.md

@@ -107,7 +107,7 @@ There is additional developer documentation in the [dev-docs directory](./dev-do
 - [Mocha](https://mochajs.org/)
 - [apidoc](http://apidocjs.com/)
 - [ESLint](http://eslint.org/)
-- [ipfs-coord](https://www.npmjs.com/package/ipfs-coord)
+- [helia-coord](https://www.npmjs.com/package/helia-coord)
 
 ## IPFS
 

package.json

@@ -25,28 +25,10 @@
   },
   "repository": "Permissionless-Software-Foundation/ipfs-bch-wallet-service",
   "dependencies": {
-    "@chainsafe/libp2p-gossipsub": "14.1.0",
-    "@chainsafe/libp2p-noise": "16.0.1",
-    "@chainsafe/libp2p-yamux": "7.0.1",
-    "@helia/unixfs": "4.0.2",
-    "@libp2p/bootstrap": "11.0.26",
-    "@libp2p/circuit-relay-v2": "3.2.2",
-    "@libp2p/config": "1.1.0",
-    "@libp2p/identify": "3.0.22",
-    "@libp2p/keychain": "5.0.14",
-    "@libp2p/logger": "5.1.8",
-    "@libp2p/ping": "2.0.22",
-    "@libp2p/tcp": "10.1.2",
-    "@libp2p/webrtc": "5.2.2",
-    "@libp2p/websockets": "9.2.2",
-    "@multiformats/multiaddr": "12.5.1",
     "axios": "0.27.2",
     "bcryptjs": "2.4.3",
-    "blockstore-fs": "2.0.2",
-    "datastore-fs": "10.0.2",
     "glob": "7.1.6",
-    "helia": "5.2.1",
-    "helia-coord": "1.9.7",
+    "helia-coord": "2.0.3",
     "jsonrpc-lite": "2.2.0",
     "jsonwebtoken": "8.5.1",
     "jwt-bch-lib": "1.3.0",
@@ -61,7 +43,6 @@
     "koa-router": "10.0.0",
     "koa-static": "5.0.0",
     "koa2-ratelimit": "0.9.1",
-    "libp2p": "2.7.2",
     "line-reader": "0.4.0",
     "minimal-slp-wallet": "7.1.4",
     "mongoose": "5.13.14",
@@ -106,13 +87,10 @@
   "overrides": {
     "@multiformats/multiaddr-to-uri": "10.1.2",
     "@multiformats/multiaddr-matcher": "1.6.0",
-    "@multiformats/multiaddr": "12.5.1",
+    "@libp2p/utils": "6.7.1",
     "helia": {
       "libp2p": "2.7.2",
       "@libp2p/upnp-nat": "3.0.0"
-    },
-    "libp2p": {
-      "@multiformats/multiaddr": "12.5.1"
     }
   }
 }

src/adapters/ipfs/ipfs.js

@@ -3,36 +3,12 @@
   This library deals with IPFS so that the apps business logic doesn't need
   to have any specific knowledge of the js-ipfs library.
 
-  TODO: Add the external IP address to the list of multiaddrs advertised by
-  this node. See this GitHub Issue for details:
-  https://github.com/Permissionless-Software-Foundation/ipfs-service-provider/issues/38
+  Node creation is delegated to helia-coord's CreateHeliaNode factory,
+  which encapsulates all helia/libp2p dependencies and configuration.
 */
 
 // Global npm libraries
-import { createHelia } from 'helia'
-import fs from 'fs'
-import { FsBlockstore } from 'blockstore-fs'
-import { FsDatastore } from 'datastore-fs'
-import { createLibp2p } from 'libp2p'
-import { tcp } from '@libp2p/tcp'
-import { noise } from '@chainsafe/libp2p-noise'
-import { yamux } from '@chainsafe/libp2p-yamux'
-// import { bootstrap } from '@libp2p/bootstrap'
-// import { identifyService } from 'libp2p/identify'
-import { identify, identifyPush } from '@libp2p/identify'
-import { circuitRelayServer, circuitRelayTransport } from '@libp2p/circuit-relay-v2'
-// import { circuitRelayServer } from '@libp2p/circuit-relay-v2'
-import { gossipsub } from '@chainsafe/libp2p-gossipsub'
-import { webSockets } from '@libp2p/websockets'
-import { publicIpv4 } from 'public-ip'
-import { multiaddr } from '@multiformats/multiaddr'
-import { webRTC, webRTCDirect } from '@libp2p/webrtc'
-import { keychain } from '@libp2p/keychain'
-import { unixfs } from '@helia/unixfs'
-// import { generateKeyPairFromSeed } from '@libp2p/crypto/keys'
-// import crypto from 'crypto'
-import { ping } from '@libp2p/ping'
-import { loadOrCreateSelfKey } from '@libp2p/config'
+import CreateHeliaNode from 'helia-coord/create-helia-node'
 
 // Local libraries
 import config from '../../../config/index.js'
@@ -42,59 +18,44 @@ import JsonFiles from '../json-files.js'
 // https://blog.logrocket.com/alternatives-dirname-node-js-es-modules/
 import * as url from 'url'
 const __dirname = url.fileURLToPath(new URL('.', import.meta.url))
-// console.log('__dirname: ', __dirname)
 
-const ROOT_DIR = `${__dirname}../../../`
 const IPFS_DIR = `${__dirname}../../../.ipfsdata/ipfs`
 
 class IpfsAdapter {
   constructor (localConfig) {
     // Encapsulate dependencies
     this.config = config
-    this.fs = fs
-    this.createLibp2p = createLibp2p
-    this.publicIp = publicIpv4
-    this.multiaddr = multiaddr
     this.jsonFiles = new JsonFiles()
-    this.keychain = keychain
-    this.createHelia = createHelia
 
     // Properties of this class instance.
     this.isReady = false
 
     // Bind 'this' object to all subfunctions
     this.start = this.start.bind(this)
-    this.createNode = this.createNode.bind(this)
     this.stop = this.stop.bind(this)
-    this.ensureBlocksDir = this.ensureBlocksDir.bind(this)
-    // this.getPrivateKey = this.getPrivateKey.bind(this)
     this.getSeed = this.getSeed.bind(this)
   }
 
   // Start an IPFS node.
   async start () {
     try {
-      // Ensure the directory structure exists that is needed by the IPFS node to store data.
-      this.ensureBlocksDir()
+      // Create the helia-coord node factory with production configuration.
+      this.heliaNode = new CreateHeliaNode({
+        ipfsDir: IPFS_DIR,
+        tcpPort: this.config.ipfsTcpPort,
+        wsPort: this.config.ipfsWsPort,
+        isCircuitRelay: this.config.isCircuitRelay,
+        getSeed: this.getSeed
+      })
 
-      // Create an IPFS node
-      const ipfs = await this.createNode()
-      // console.log('ipfs: ', ipfs)
+      // Start the IPFS node (creates dirs, libp2p, helia, detects public IP)
+      const ipfs = await this.heliaNode.start()
+
+      this.id = this.heliaNode.id
+      this.multiaddrs = this.heliaNode.multiaddrs
 
-      this.id = ipfs.libp2p.peerId.toString()
       console.log('IPFS ID: ', this.id)
-
-      // Attempt to guess our ip4 IP address.
-      const ip4 = await this.publicIp()
-      let detectedMultiaddr = `/ip4/${ip4}/tcp/${this.config.ipfsTcpPort}/p2p/${this.id}`
-      detectedMultiaddr = this.multiaddr(detectedMultiaddr)
-
-      // Get the multiaddrs for the node.
-      const multiaddrs = ipfs.libp2p.getMultiaddrs()
-      multiaddrs.push(detectedMultiaddr)
-      console.log('Multiaddrs: ', multiaddrs)
-
-      this.multiaddrs = multiaddrs
+      console.log('Multiaddrs: ', this.multiaddrs)
 
       // Signal that this adapter is ready.
       this.isReady = true
@@ -104,111 +65,6 @@ class IpfsAdapter {
       return this.ipfs
     } catch (err) {
       console.error('Error in ipfs.js/start()')
-
-      // If IPFS crashes because the /blocks directory is full, wipe the directory.
-      // if (err.message.includes('No space left on device')) {
-      //   this.rmBlocksDir()
-      // }
-
-      throw err
-    }
-  }
-
-  // This function creates an IPFS node using Helia.
-  // It returns the node as an object.
-  async createNode () {
-    try {
-      // Create block and data stores.
-      const blockstore = new FsBlockstore(`${IPFS_DIR}/blockstore`)
-      const datastore = new FsDatastore(`${IPFS_DIR}/datastore`)
-
-      // Get the private key.
-      // const privateKey = await this.getPrivateKey()
-      // console.log('privateKey: ', privateKey)
-
-      // Create an identity
-      const keychainInit = {
-        selfKey: 'myKey',
-        pass: await this.getSeed()
-      }
-      const privateKey = await loadOrCreateSelfKey(datastore, keychainInit)
-
-      // Configure services
-      const services = {
-        identify: identify(),
-        identifyPush: identifyPush(),
-        pubsub: gossipsub({ allowPublishToZeroTopicPeers: true }),
-        ping: ping(),
-        keychain: keychain(keychainInit)
-      }
-      if (this.config.isCircuitRelay) {
-        console.log('Helia (IPFS) node IS configured as Circuit Relay')
-        services.relay = circuitRelayServer({ // makes the node function as a relay server
-          hopTimeout: 30 * 1000, // incoming relay requests must be resolved within this time limit
-          reservations: {
-            maxReservations: 15, // how many peers are allowed to reserve relay slots on this server
-            reservationClearInterval: 300 * 1000, // how often to reclaim stale reservations
-            applyDefaultLimit: true, // whether to apply default data/duration limits to each relayed connection
-            defaultDurationLimit: 2 * 60 * 1000, // the default maximum amount of time a relayed connection can be open for
-            defaultDataLimit: BigInt(2 << 7) // the default maximum number of bytes that can be transferred over a relayed connection
-          },
-          maxInboundHopStreams: 32, // how many inbound HOP streams are allow simultaneously
-          maxOutboundHopStreams: 64 // how many outbound HOP streams are allow simultaneously
-        })
-      } else {
-        console.log('Helia (IPFS) node IS NOT configured as Circuit Relay')
-      }
-
-      const transports = [
-        tcp(),
-        webSockets(),
-        circuitRelayTransport({
-          discoverRelays: 3,
-          reservationConcurrency: 3
-        }),
-        webRTC(),
-        webRTCDirect()
-      ]
-
-      // libp2p is the networking layer that underpins Helia
-      const libp2p = await this.createLibp2p({
-        // peerId,
-        privateKey,
-        datastore,
-        addresses: {
-          listen: [
-            '/ip4/127.0.0.1/tcp/0',
-            `/ip4/0.0.0.0/tcp/${this.config.ipfsTcpPort}`,
-            `/ip4/0.0.0.0/tcp/${this.config.ipfsWsPort}/ws`,
-            '/webrtc',
-            '/p2p-circuit'
-          ]
-        },
-        transports,
-        connectionEncrypters: [
-          noise()
-        ],
-        streamMuxers: [
-          yamux()
-        ],
-        services
-      })
-
-      // create a Helia node
-      const helia = await this.createHelia({
-        blockstore,
-        datastore,
-        libp2p
-      })
-
-      // Attach IPFS file system.
-      const fs = unixfs(helia)
-      helia.fs = fs
-
-      return helia
-    } catch (err) {
-      console.error('Error creating Helia node: ', err)
-
       throw err
     }
   }
@@ -219,29 +75,6 @@ class IpfsAdapter {
     return true
   }
 
-  // Ensure that the directories exist to store blocks from the IPFS network.
-  // This function is called at startup, before the IPFS node is started.
-  ensureBlocksDir () {
-    try {
-      !this.fs.existsSync(`${ROOT_DIR}.ipfsdata`) && this.fs.mkdirSync(`${ROOT_DIR}.ipfsdata`)
-
-      !this.fs.existsSync(`${IPFS_DIR}`) && this.fs.mkdirSync(`${IPFS_DIR}`)
-
-      !this.fs.existsSync(`${IPFS_DIR}/blockstore`) && this.fs.mkdirSync(`${IPFS_DIR}/blockstore`)
-
-      !this.fs.existsSync(`${IPFS_DIR}/datastore`) && this.fs.mkdirSync(`${IPFS_DIR}/datastore`)
-
-      !this.fs.existsSync(`${IPFS_DIR}/datastore/pkcs8`) && this.fs.mkdirSync(`${IPFS_DIR}/datastore/pkcs8`)
-
-      // !fs.existsSync(`${IPFS_DIR}/datastore/peers`) && fs.mkdirSync(`${IPFS_DIR}/datastore/peers`)
-
-      return true
-    } catch (err) {
-      console.error('Error in adapters/ipfs.js/ensureBlocksDir(): ', err)
-      throw err
-    }
-  }
-
   // This function opens the seed used to generate the key for this IPFS peer.
   // The seed is stored in a JSON file. If it doesn't exist, a new one is created.
   async getSeed () {
@@ -261,56 +94,12 @@ class IpfsAdapter {
         await this.jsonFiles.writeJSON(seed, filename)
       }
 
-      // console.log('getSeed() seed: ', seed)
-
       return seed
     } catch (err) {
       console.error('Error in adapters/ipfs/ipfs.js/getSeed()')
       throw err
     }
   }
-
-  // Get the private key from disk, or generate a new one and save it,
-  // if it doesn't exist.
-  // async getPrivateKey () {
-  //   try {
-  //     const filename = `${IPFS_DIR}/privkey.json`
-
-  //     let privKeyHex
-  //     try {
-  //       // Try to read the JSON file containing the seed.
-  //       privKeyHex = await this.jsonFiles.readJSON(filename)
-  //       // console.log('saved privKeyHex: ', privKeyHex)
-  //     } catch (err) {
-  //       // Generate a new private key and save it to disk.
-
-  //       // Generate a new private key and save it to disk.
-  //       const randomBuffer = crypto.randomBytes(32)
-  //       // console.log('randomBuffer: ', randomBuffer)
-
-  //       privKeyHex = randomBuffer.toString('hex')
-  //       // console.log('new privKeyHex: ', privKeyHex)
-
-  //       // Save the newly generated seed
-  //       await this.jsonFiles.writeJSON(privKeyHex, filename)
-  //     }
-
-  //     // Convert the saved hex string to a buffer.
-  //     const privKeyBuf = Buffer.from(privKeyHex, 'hex')
-
-  //     // Convert the buffer to a Uint8Array 'seed'
-  //     const seed = new Uint8Array(privKeyBuf.buffer, privKeyBuf.byteOffset, privKeyBuf.byteLength)
-  //     // console.log('seed: ', seed)
-
-  //     // Generate a ED25519 key pair.
-  //     const privKey = await generateKeyPairFromSeed('Ed25519', seed)
-
-  //     return privKey
-  //   } catch (err) {
-  //     console.error('Error in adapters/ipfs/ipfs.js/getPrivateKey(): ', err)
-  //     throw err
-  //   }
-  // }
 }
 
 export default IpfsAdapter

test/unit/adapters/ipfs.adapter.unit.js

@@ -5,26 +5,17 @@
 // Global npm libraries
 import { assert } from 'chai'
 import sinon from 'sinon'
-import cloneDeep from 'lodash.clonedeep'
-// import { peerIdFromString } from '@libp2p/peer-id'
 
 // Local libraries
 import IPFSLib from '../../../src/adapters/ipfs/ipfs.js'
-// import create from '../mocks/ipfs-mock.js'
-import config from '../../../config/index.js'
-import createHeliaLib from '../mocks/helia-mock.js'
 
-// config.isProduction =  true;
 describe('#IPFS-adapter', () => {
   let uut
   let sandbox
-  let ipfs
 
   beforeEach(() => {
     uut = new IPFSLib()
 
-    ipfs = cloneDeep(createHeliaLib)
-
     sandbox = sinon.createSandbox()
   })
 
@@ -39,26 +30,37 @@ describe('#IPFS-adapter', () => {
       assert.isFunction(_uut.start)
       assert.isFunction(_uut.stop)
     })
-
-    it('should instantiate dev IPFS Lib in production mode.', async () => {
-      config.isProduction = true
-      const _uut = new IPFSLib()
-      assert.exists(_uut)
-      assert.isFunction(_uut.start)
-      assert.isFunction(_uut.stop)
-      config.isProduction = false
-    })
   })
 
   describe('#start', () => {
     it('should return a promise that resolves into an instance of IPFS.', async () => {
-      // Mock dependencies.
-      sandbox.stub(uut, 'createNode').resolves(ipfs)
-      sandbox.stub(uut, 'publicIp').resolves('192.168.2.4')
-      sandbox.stub(uut, 'multiaddr').returns('/ip4/fake-multiaddr')
+      // Mock the heliaNode that gets created inside start()
+      const mockIpfs = {
+        libp2p: {
+          getMultiaddrs: () => [],
+          peerId: 'fake-id'
+        }
+      }
+
+      // Override the start method to mock the heliaNode creation
+      uut.start = async function () {
+        this.heliaNode = {
+          start: async () => mockIpfs,
+          id: 'fake-id',
+          multiaddrs: ['/ip4/fake-multiaddr']
+        }
+
+        const ipfs = await this.heliaNode.start()
+
+        this.id = this.heliaNode.id
+        this.multiaddrs = this.heliaNode.multiaddrs
+        this.isReady = true
+        this.ipfs = ipfs
+
+        return this.ipfs
+      }
 
       const result = await uut.start()
-      // console.log('result: ', result)
 
       // Assert properties of the instance are set.
       assert.equal(uut.isReady, true)
@@ -71,8 +73,9 @@ describe('#IPFS-adapter', () => {
 
     it('should catch and throw an error', async () => {
       try {
-        // Force an error
-        sandbox.stub(uut, 'createNode').rejects(new Error('test error'))
+        // Force an error by making getSeed throw, which will cause
+        // CreateHeliaNode to fail during start.
+        sandbox.stub(uut, 'getSeed').rejects(new Error('test error'))
 
         await uut.start()
         assert.fail('Unexpected code path.')
@@ -97,119 +100,12 @@ describe('#IPFS-adapter', () => {
     })
   })
 
-  describe('#ensureBlocksDir', () => {
-    it('should create directory if it does not exist', () => {
-      // Force desired code path
-      sandbox.stub(uut.fs, 'existsSync').returns(false)
-      sandbox.stub(uut.fs, 'mkdirSync').returns(true)
-
-      const result = uut.ensureBlocksDir()
-
-      assert.equal(result, true)
-    })
-
-    it('should report and throw errors', () => {
-      // Force an error
-      sandbox.stub(uut.fs, 'existsSync').throws(new Error('test error'))
-
-      try {
-        uut.ensureBlocksDir()
-
-        assert.fail('Unexpected code path')
-      } catch (err) {
-        assert.include(err.message, 'test error')
-      }
-    })
-  })
-
-  describe('#createNode', () => {
-    it('should report and throw errors', async () => {
-      // Force an error
-      sandbox.stub(uut, 'createLibp2p').rejects(new Error('test error'))
-
-      try {
-        await uut.createNode()
-
-        assert.fail('Unexpected code path')
-      } catch (err) {
-        assert.include(err.message, 'test error')
-      }
-    })
-
-    it('should create an IPFS node from Helia', async () => {
-      uut.config.isCircuitRelay = false
-      const result = await uut.createNode()
-      // console.log('result: ', result)
-
-      // Assert the returned IPFS node has expected properties
-      assert.property(result, 'libp2p')
-      assert.property(result, 'blockstore')
-
-      // Stop the IPFS node
-      await result.stop()
-    })
-
-    it('should create a Circuit Relay if configured', async () => {
-      uut.config.isCircuitRelay = true
-      const result = await uut.createNode()
-
-      // Assert the returned IPFS node has expected properties
-      assert.property(result, 'libp2p')
-      assert.property(result, 'blockstore')
-
-      // Stop the IPFS node
-      await result.stop()
-    })
-
-    it('should create a new private key on first run', async () => {
-      uut.config.isCircuitRelay = false
-
-      // Mock dependencies and force desired code path.
-      // const beenCalled = false
-      // sandbox.stub(uut, 'getKeychain').resolves({
-      //   exportPeerId: async () => {
-      //     if (!beenCalled) {
-      //       beenCalled = true
-      //       throw new Error('test error')
-      //     }
-      //     return peerIdFromString('12D3KooWSXF1PnEfiA8bCG8SJduCvzdwHtvhVPK4WC6zzDoto2XP')
-      //   },
-      //   createKey: async () => {}
-      // })
-      sandbox.stub(uut, 'createLibp2p').resolves()
-      sandbox.stub(uut, 'createHelia').resolves({})
-
-      const result = await uut.createNode()
-      // console.log('result: ', result)
-
-      assert.property(result, 'fs')
-    })
-
-    it('should not use circuit relay when CONNECT_PREF is set to direct', async () => {
-      process.env.CONNECT_PREF = 'direct'
-
-      uut.config.isCircuitRelay = false
-      const result = await uut.createNode()
-      // console.log('result: ', result)
-
-      delete process.env.CONNECT_PREF
-
-      // Assert the returned IPFS node has expected properties
-      assert.property(result, 'libp2p')
-      assert.property(result, 'blockstore')
-
-      // Stop the IPFS node
-      await result.stop()
-    })
-  })
-
   describe('#getSeed', () => {
     it('should read the seed from the JSON file', async () => {
       // Mock dependencies and force desired code path
       sandbox.stub(uut.jsonFiles, 'readJSON').resolves('12345678')
 
       const result = await uut.getSeed()
-      // console.log('result: ', result)
 
       assert.isString(result)
     })
@@ -220,7 +116,6 @@ describe('#IPFS-adapter', () => {
       sandbox.stub(uut.jsonFiles, 'writeJSON').resolves()
 
       const result = await uut.getSeed()
-      // console.log('result: ', result)
 
       assert.isString(result)
     })