Merge pull request #11 from Permissionless-Software-Foundation/ct-unstable

Syncing with upstream ipfs-service-provider

.gitignore

@@ -67,5 +67,6 @@ orbitdb
 ipfsdata
 .ipfsdata
 run-dev.sh
+data/
 
 !README.md

bin/server.js

@@ -104,8 +104,18 @@ class Server {
       return app
     } catch (err) {
       console.error('Could not start server. Error: ', err)
+
+      console.log(
+        'Exiting after 5 seconds. Depending on process manager to restart.'
+      )
+      await sleep(5000)
+      process.exit(1)
     }
   }
 }
 
+function sleep (ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
 module.exports = Server

config/env/common.js

@@ -15,7 +15,7 @@ const ipfsCoordName = process.env.COORD_NAME
 
 module.exports = {
   // Configure TCP port.
-  port: process.env.PORT || 5001,
+  port: process.env.PORT || 5005,
 
   // Password for HTML UI that displays logs.
   logPass: 'test',
@@ -72,8 +72,8 @@ module.exports = {
   },
 
   // IPFS Ports
-  ipfsTcpPort: process.env.IPFS_TCP_PORT ? process.env.IPFS_TCP_PORT : 4001,
-  ipfsWsPort: process.env.IPFS_WS_PORT ? process.env.IPFS_WS_PORT : 4003,
+  ipfsTcpPort: process.env.IPFS_TCP_PORT ? process.env.IPFS_TCP_PORT : 4101,
+  ipfsWsPort: process.env.IPFS_WS_PORT ? process.env.IPFS_WS_PORT : 4103,
 
   // BCH Mnemonic for generating encryption keys and payment address
   mnemonic: process.env.MNEMONIC ? process.env.MNEMONIC : '',
@@ -91,5 +91,12 @@ module.exports = {
   // Preferred P2WDB provider
   preferredP2wdbProvider: process.env.PREFERRED_P2WDB_PROVIDER
     ? process.env.PREFERRED_P2WDB_PROVIDER
-    : ''
+    : '',
+
+  // Settings for production, using external go-ipfs node.
+  isProduction: process.env.SVC_ENV === 'production' ? true : false,
+  ipfsHost: process.env.IPFS_HOST ? process.env.IPFS_HOST : 'localhost',
+  ipfsApiPort: process.env.IPFS_API_PORT
+    ? parseInt(process.env.IPFS_API_PORT)
+    : 5001
 }

package.json

@@ -23,12 +23,13 @@
   },
   "repository": "Permissionless-Software-Foundation/ipfs-bch-wallet-consumer",
   "dependencies": {
-    "@chris.troutner/ipfs": "2.0.2",
-    "@psf/bch-js": "4.20.26",
+    "@psf/bch-js": "4.20.28",
     "axios": "0.21.1",
     "bcryptjs": "2.4.3",
     "glob": "7.1.6",
-    "ipfs-coord": "6.7.5",
+    "ipfs": "0.60.0",
+    "ipfs-coord": "6.8.10",
+    "ipfs-http-client": "55.0.0",
     "jsonrpc-lite": "2.2.0",
     "jsonwebtoken": "8.5.1",
     "jwt-bch-lib": "1.3.0",
@@ -42,7 +43,7 @@
     "koa-passport": "4.1.3",
     "koa-router": "10.0.0",
     "koa-static": "5.0.0",
-    "koa2-ratelimit": "0.9.0",
+    "koa2-ratelimit": "0.9.1",
     "line-reader": "0.4.0",
     "mongoose": "5.11.15",
     "node-fetch": "npm:@achingbrain/node-fetch@2.6.7",
@@ -64,6 +65,7 @@
     "eslint-plugin-prettier": "3.3.1",
     "eslint-plugin-standard": "4.0.0",
     "husky": "4.3.8",
+    "lodash.clonedeep": "^4.5.0",
     "mocha": "8.2.1",
     "nyc": "15.1.0",
     "semantic-release": "17.4.4",

production/docker/Dockerfile

@@ -33,8 +33,8 @@ WORKDIR /home/safeuser
 #RUN runuser -l safeuser -c "npm config set prefix '~/.npm-global'"
 
 # Update to the latest version of npm.
-# Working with npm@7.21.1
-RUN npm install -g npm@7.23.0
+#RUN npm install -g npm@7.23.0
+RUN npm install -g npm
 
 # npm mirror to prevent direct dependency on npm.
 RUN npm set registry http://94.130.170.209:4873/
@@ -66,7 +66,7 @@ RUN npm run docs
 VOLUME /home/safeuser/keys
 
 # Expose the port the API will be served on.
-EXPOSE 5001
+EXPOSE 5010
 
 # Start the application.
 COPY start-production.sh start-production.sh

production/docker/README.md

@@ -0,0 +1,61 @@
+# Docker Containers
+
+The 'production' environment is assumed to be a set of Docker containers orchestrated with Docker Compose. The files in this directory will stand up three Docker containers:
+
+1 An instance of go-ipfs.
+2 An instance of MongoDB.
+3 The JavaScript software in this repository.
+
+The software in this repository depends on the first two containers, so if they aren't running correctly, the application won't run correctly either.
+
+## IPFS
+
+IPFS can be a little tricky to set up. By default, the container uses the following ports:
+
+- 4001 for TCP connections, exposed publicly.
+- 5001 for control by the application, exposed privately.
+- 8080 for an IPFS gateway, consumed by the application, exposed privately.
+
+If you already have an IPFS node running on a the computer, you will need to change the ports to avaid a conflict. To change the ports from the default, you'll need to perform a series of steps, and the order of the steps matter.
+
+1. Edit the `docker-compose.yml` file and change the ports. Then save the file. Here is an example:
+
+```
+ports:
+  - 4101:4101
+  - 172.17.0.1:5101:5101
+  - 172.17.0.1:8180:8180
+```
+
+2. Bring the Docker containers up, and then back down. This will allow the IPFS container to create the config file that you'll need to edit.
+
+- `docker-compose up -d`
+- Wait a few seconds.
+- `docker-compose down`
+
+3. Update the generated config file at `../data/go-ipfs/data/config`, to update the ports in the config file, like this:
+
+```
+"Addresses": {
+    "API": "/ip4/0.0.0.0/tcp/5101",
+    "Announce": [],
+    "AppendAnnounce": [],
+    "Gateway": "/ip4/0.0.0.0/tcp/8180",
+    "NoAnnounce": [],
+    "Swarm": [
+      "/ip4/0.0.0.0/tcp/4101",
+      "/ip6/::/tcp/4101",
+      "/ip4/0.0.0.0/udp/4101/quic",
+      "/ip6/::/udp/4101/quic"
+    ]
+  },
+
+```
+
+4. Update the port changes in the `start-production.sh` shell script. This tells the application which ports to use, in order to control the IPFS node, are are used when signaling other nodes.
+
+5. Quickly rebuild the containers, to add the modified `start-production.sh` shell script to the application Docker container:
+
+- `docker-compose build`
+
+6. Now start the containers, and the port changes to IPFS should be complete.

production/docker/docker-compose.yml

@@ -1,6 +1,6 @@
 # Start the service with the command 'docker-compose up -d'
 
-version: '2'
+version: '3.9'
 
 services:
   mongo-ipfs-service:
@@ -13,6 +13,35 @@ services:
     command: mongod --logpath=/dev/null # -- quiet
     restart: always
 
+  ipfs:
+    image: ipfs/go-ipfs:v0.11.0
+    container_name: ipfs
+    environment:
+      MY_ENV_VAR: 'placeholder'
+    logging:
+      driver: 'json-file'
+      options:
+        max-size: '10m'
+        max-file: '10'
+    mem_limit: 2000mb
+    ports:
+      - 4001:4001
+      - 172.17.0.1:5001:5001
+      - 172.17.0.1:8080:8080
+    volumes:
+      - ../data/go-ipfs/data:/data/ipfs
+      - ../data/go-ipfs/export:/export
+    # https://docs.docker.com/compose/compose-file/compose-file-v3/#command
+    # https://github.com/ipfs/go-ipfs/blob/91c52657166bcf86f2476926e4fe56694dc26562/Dockerfile#L115
+    command:
+      [
+        'daemon',
+        '--migrate=true',
+        '--agent-version-suffix=docker',
+        '--enable-pubsub-experiment'
+      ]
+    restart: always
+
   ipfs-service:
     build: .
     container_name: ipfs-service
@@ -24,10 +53,9 @@ services:
     mem_limit: 500mb
     links:
       - mongo-ipfs-service
+      - ipfs
     ports:
-      - '5001:5001' # <host port>:<container port>
-      - '5268:5268' # IPFS TCP
-      - '5269:5269' # IPFS WS
+      - '5010:5010' # <host port>:<container port>
     volumes:
       - ../data/ipfsdata:/home/safeuser/ipfs-service-provider/.ipfsdata
     restart: always

production/docker/start-production.sh

@@ -17,7 +17,7 @@ export COORD_NAME=ipfs-service-provider-generic
 #export CR_DOMAIN=subdomain.yourdomain.com
 
 # Debug level. 0 = minimal info. 2 = max info.
-export DEBUG_LEVEL=1
+export DEBUG_LEVEL=2
 
 # END: Optional configuration settings
 
@@ -25,13 +25,14 @@ export DEBUG_LEVEL=1
 # Production database connection string.
 export DBURL=mongodb://172.17.0.1:5555/ipfs-service-prod
 
-# Configure IPFS ports
-export IPFS_TCP_PORT=5268
-export IPFS_WS_PORT=5269
-
-
 # Configure REST API port
-export PORT=5001
+export PORT=5010
 
+# Production settings using external go-ipfs node.
 export SVC_ENV=production
+export IPFS_HOST=172.17.0.1
+export IPFS_API_PORT=5001
+export IPFS_TCP_PORT=4001
+#export IPFS_WS_PORT=5269
+
 npm start

shell-scripts/ipfs-service-provider-generic.sh

@@ -3,7 +3,7 @@
 # This script is an example for running a generic ipfs-service-provider instance.
 
 # Ports
-export PORT=5001 # REST API port
+export PORT=5005 # REST API port
 export IPFS_TCP_PORT=5268
 export IPFS_WS_PORT=5269
 

shell-scripts/local-external-ipfs-node.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# This script is an example for running a production environment, which is
+# defined by running an external go-ipfs node.
+
+# Ports
+export PORT=5005 # REST API port
+
+# The human-readible name that is used when displaying data about this node.
+export COORD_NAME=trout-dev-bch-wallet-consumer
+
+# This is used for end-to-end encryption (e2ee).
+export MNEMONIC="stomach mean catch march census author cousin size south romance pumpkin trap"
+
+# 0 = less verbose. 3 = most verbose
+export DEBUG_LEVEL=2
+
+# Production settings that use external IPFS node.
+# https://github.com/christroutner/docker-ipfs
+export SVC_ENV=production
+export IPFS_HOST=localhost
+export IPFS_API_PORT=5001
+
+# Configure IPFS ports
+export IPFS_TCP_PORT=4001
+#export IPFS_WS_PORT=5269
+
+# MongoDB connection string.
+export DBURL=mongodb://localhost:27017/ipfs-service-dev
+
+npm start

src/adapters/ipfs/ipfs-coord.js

@@ -51,7 +51,10 @@ class IpfsCoordAdapter {
 
     // Periodically poll services for available wallet service providers.
     this.pollBchServiceInterval = setInterval(this.pollForBchServices, 10000)
-    this.pollP2wdbServiceInterval = setInterval(this.pollForP2wdbServices, 11000)
+    this.pollP2wdbServiceInterval = setInterval(
+      this.pollForP2wdbServices,
+      11000
+    )
 
     // State object. TODO: Make this more robust.
     this.state = {
@@ -75,6 +78,7 @@ class IpfsCoordAdapter {
 
         circuitRelayInfo.ip4 = ip4
         circuitRelayInfo.tcpPort = this.config.ipfsTcpPort
+        circuitRelayInfo.wsPort = this.config.ipfsWsPort
 
         // Domain used by browser-based secure websocket connections.
         circuitRelayInfo.crDomain = this.config.crDomain
@@ -83,7 +87,7 @@ class IpfsCoordAdapter {
       }
     }
 
-    this.ipfsCoord = new this.IpfsCoord({
+    const ipfsCoordOptions = {
       ipfs: this.ipfs,
       type: 'node.js',
       // type: 'browser',
@@ -94,7 +98,14 @@ class IpfsCoordAdapter {
       apiInfo: this.config.apiInfo,
       announceJsonLd: this.config.announceJsonLd,
       debugLevel: this.config.debugLevel
-    })
+    }
+
+    // Production env uses external go-ipfs node.
+    if (this.config.isProduction) {
+      ipfsCoordOptions.nodeType = 'external'
+    }
+
+    this.ipfsCoord = new this.IpfsCoord(ipfsCoordOptions)
 
     // Wait for the ipfs-coord library to signal that it is ready.
     await this.ipfsCoord.start()
@@ -169,7 +180,7 @@ class IpfsCoordAdapter {
           // to the preferred provider when it's discovered.
           if (
             _this.config.preferredP2wdbProvider &&
-              thisPeer === _this.config.preferredP2wdbProvider
+            thisPeer === _this.config.preferredP2wdbProvider
           ) {
             _this.state.selectedP2wdbProvider = thisPeer
           }
@@ -187,8 +198,15 @@ class IpfsCoordAdapter {
         }
       }
     } catch (err) {
+      // catch and handle known failure mode.
+      if (
+        err.message.includes("Cannot read property 'peerList' of undefined")
+      ) {
+        return
+      }
+
       console.error('Error in pollForP2wdbServices()')
-      throw err
+      // Do not throw error. This is a top-level function.
     }
   }
 
@@ -259,6 +277,13 @@ class IpfsCoordAdapter {
         }
       }
     } catch (err) {
+      // catch and handle known failure mode.
+      if (
+        err.message.includes("Cannot read property 'peerList' of undefined")
+      ) {
+        return
+      }
+
       console.error('Error in pollForBchServices(): ', err)
       // Do not throw error. This is a top-level function.
     }

src/adapters/ipfs/ipfs.js

@@ -10,7 +10,9 @@
 
 // Global npm libraries
 // const IPFS = require('ipfs')
-const IPFS = require('@chris.troutner/ipfs')
+// const IPFS = require('@chris.troutner/ipfs')
+const IPFSembedded = require('ipfs')
+const IPFSexternal = require('ipfs-http-client')
 const fs = require('fs')
 
 // Local libraries
@@ -21,11 +23,17 @@ const IPFS_DIR = './.ipfsdata/ipfs'
 class IpfsAdapter {
   constructor (localConfig) {
     // Encapsulate dependencies
-    this.IPFS = IPFS
+    this.config = config
+
+    // Choose the IPFS constructor based on the config settings.
+    this.IPFS = IPFSembedded // default
+    if (this.config.isProduction) {
+      this.IPFS = IPFSexternal
+    }
 
     // Properties of this class instance.
     this.isReady = false
-    this.config = config
+
     this.fs = fs
   }
 
@@ -33,7 +41,7 @@ class IpfsAdapter {
   async start () {
     try {
       // Ipfs Options
-      const ipfsOptions = {
+      const ipfsOptionsEmbedded = {
         repo: IPFS_DIR,
         start: true,
         config: {
@@ -64,6 +72,16 @@ class IpfsAdapter {
         }
       }
 
+      const ipfsOptionsExternal = {
+        host: this.config.ipfsHost,
+        port: this.config.ipfsApiPort
+      }
+
+      let ipfsOptions = ipfsOptionsEmbedded
+      if (this.config.isProduction) {
+        ipfsOptions = ipfsOptionsExternal
+      }
+
       // Create a new IPFS node.
       this.ipfs = await this.IPFS.create(ipfsOptions)
 

src/controllers/index.js

@@ -7,21 +7,18 @@
 // Public npm libraries.
 const EventEmitter = require('events')
 
-// Load the Clean Architecture Adapters library
+// Local libraries
+const config = require('../../config')
 const Adapters = require('../adapters')
-
-// Load the JSON RPC Controller.
 const JSONRPC = require('./json-rpc')
-
-// Load the Clean Architecture Use Case libraries.
 const UseCases = require('../use-cases')
-// const useCases = new UseCases({ adapters })
-
-// Load the REST API Controllers.
 const RESTControllers = require('./rest-api')
 
 class Controllers {
   constructor (localConfig = {}) {
+    // Encapsulate dependencies
+    this.config = config
+
     // Initialize EventEmitters used to pass event-driven data around the app.
     this.eventEmitter = new EventEmitter()
     localConfig.eventEmitter = this.eventEmitter
@@ -33,13 +30,16 @@ class Controllers {
   }
 
   async attachControllers (app) {
-    // Wait for any startup processes to complete for the Adapters libraries.
-    await this.adapters.start()
+    // Skip in a test environment.
+    if (this.config.env !== 'test') {
+      // Wait for any startup processes to complete for the Adapters libraries.
+      await this.adapters.start()
 
-    // Attach the REST controllers to the Koa app.
-    // this.attachRESTControllers(app)
+      // Attach the REST controllers to the Koa app.
+      // this.attachRESTControllers(app)
 
-    this.attachRPCControllers()
+      this.attachRPCControllers()
+    }
   }
 
   // Top-level function for this library.