Merge pull request #59 from Permissionless-Software-Foundation/ct-unstable

fix(bch-js): Updating to bch-js v5

README.md

@@ -24,13 +24,17 @@ This repository is one step in a longer path. Here are the major milestones of t
 
 ## Requirements
 
-- node **^14.17.0**
-- npm **^7.13.0**
+- node **^14.18.2**
+- npm **^8.3.0**
+- Docker **^20.10.8**
+- Docker Compose **^1.27.4**
 
 ## Installation
 
 ### Development Environment
 
+**Note:** This software now uses an external go-ipfs IPFS node. The instructions below have not been updated to reflect this.
+
 A development environment will allow you modify the code on-the-fly and contribute to the code base of this repository. [PM2](https://www.npmjs.com/package/pm2) is recommended for running this code base as an IPFS Circuit Relay.
 
 - [Video: Installing ipfs-service-provider](https://youtu.be/Z0NsboIVN44)
@@ -41,23 +45,33 @@ git clone https://github.com/Permissionless-Software-Foundation/ipfs-bch-wallet-
 cd ipfs-bch-wallet-service
 ./install-mongo.sh
 npm install
-cd shell-scripts 
+cd shell-scripts
 ./ipfs-bch-wallet-service.sh
 ```
 
 ### Production Environment
 
-The [docker](./production/docker) directory contains a Dockerfile for building a production deployment. However, there is currently [a bug](https://github.com/Permissionless-Software-Foundation/ipfs-service-provider/issues/38) preventing the Docker container from being used as a Circuit Relay.
+The [docker](./production/docker) directory contains a Dockerfile for building a production deployment.
 
 ```
 docker-compose build --no-cache
 docker-compose up -d
 ```
 
+- Bring the containers up once, then take them back down with `docker-compose down`.
+- Copy the [swarm.key](./swarm.key) file to `ipfs-service-provider/production/data/go-ipfs/data/swarm.key`.
+- Bring the containers back up with `docker-compose up -d`.
+
 ### Operation Notes
 
 - There is a memory leak in the version of js-ipfs. The app is currently configured to shut down every 8 hours to flush memory. It relies on a process manager like pm2, Docker, or systemd to restart the app after it shuts down, in order to ensure continuous operation.
 
+- The PSF network operates as a private network. It does not connect or interact with the wider PSF network, relying instead on gateways to bridge the two networks, when they need to share content. This improves the performance and experience for everyone in the PSF network. To join the network, you'll need to add the [swarm.key](./swarm.key) file to the IPFS data folder.
+
+- [Instructions on setting up IPFS private networks.](https://github.com/ipfs/go-ipfs/blob/master/docs/experimental-features.md#private-networks)
+- For external installations, the swarm.key file will typically go in `~/.ipfs/swarm.key`
+- For production Docker containers, the key would go in `ipfs-service-provider/production/data/go-ipfs/data/swarm.key`
+
 ## Structure
 
 The file layout of this repository differs from the koa-api-boilerplate. Instead, it follows the file layout of [Clean Architecture](troutsblog.com/blog/clean-architecture).

bin/server.js

@@ -97,8 +97,11 @@ class Server {
         )
       }
 
-      // Attach the other IPFS controllers
-      await controllers.attachControllers(app)
+      // Attach the other IPFS controllers.
+      // Skip if this is a test environment.
+      if (config.env !== 'test') {
+        await controllers.attachControllers(app)
+      }
 
       // ipfs-coord has a memory leak. This app shuts down after 4 hours. It
       // expects to be run by Docker or pm2, which can automatically restart
@@ -110,8 +113,18 @@ class Server {
       return app
     } catch (err) {
       console.error('Could not start server. Error: ', err)
+
+      console.log(
+        'Exiting after 5 seconds. Depending on process manager to restart.'
+      )
+      await sleep(5000)
+      process.exit(1)
     }
   }
 }
 
+function sleep (ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms))
+}
+
 module.exports = Server

config/env/common.js

@@ -82,5 +82,12 @@ module.exports = {
   // BCH Mnemonic for generating encryption keys and payment address
   mnemonic: process.env.MNEMONIC ? process.env.MNEMONIC : '',
 
-  debugLevel: process.env.DEBUG_LEVEL ? parseInt(process.env.DEBUG_LEVEL) : 2
+  debugLevel: process.env.DEBUG_LEVEL ? parseInt(process.env.DEBUG_LEVEL) : 2,
+
+  // Settings for production, using external go-ipfs node.
+  isProduction: process.env.SVC_ENV === 'production' ? true : false,
+  ipfsHost: process.env.IPFS_HOST ? process.env.IPFS_HOST : 'localhost',
+  ipfsApiPort: process.env.IPFS_API_PORT
+    ? parseInt(process.env.IPFS_API_PORT)
+    : 5001
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "ipfs-bch-wallet-service",
-  "version": "1.11.12",
+  "version": "2.0.0",
   "description": "IPFS and REST service providing blockchain access needed by a wallet. ",
   "main": "index.js",
   "scripts": {
@@ -23,12 +23,13 @@
   },
   "repository": "Permissionless-Software-Foundation/ipfs-bch-wallet-service",
   "dependencies": {
-    "@chris.troutner/ipfs": "2.0.2",
-    "@psf/bch-js": "4.20.28",
-    "axios": "0.21.1",
+    "@psf/bch-js": "5.2.0",
+    "axios": "0.25.0",
     "bcryptjs": "2.4.3",
     "glob": "7.1.6",
-    "ipfs-coord": "6.7.5",
+    "ipfs": "0.60.0",
+    "ipfs-coord": "6.8.10",
+    "ipfs-http-client": "55.0.0",
     "jsonrpc-lite": "2.2.0",
     "jsonwebtoken": "8.5.1",
     "jwt-bch-lib": "1.3.0",
@@ -42,7 +43,7 @@
     "koa-passport": "4.1.3",
     "koa-router": "10.0.0",
     "koa-static": "5.0.0",
-    "koa2-ratelimit": "0.9.0",
+    "koa2-ratelimit": "0.9.1",
     "line-reader": "0.4.0",
     "mongoose": "5.11.15",
     "node-fetch": "npm:@achingbrain/node-fetch@2.6.7",

production/docker/Dockerfile

@@ -35,8 +35,8 @@ RUN runuser -l safeuser -c "npm config set prefix '~/.npm-global'"
 RUN npm install -g npm
 
 # Update to the latest version of npm.
-# Working with npm@7.21.1
-RUN npm install -g npm@7.23.0
+#RUN npm install -g npm@7.23.0
+RUN npm install -g npm
 
 # npm mirror to prevent direct dependency on npm.
 RUN npm set registry http://94.130.170.209:4873/
@@ -70,6 +70,8 @@ RUN mkdir /home/safeuser/.ipfsdata
 #VOLUME /home/safeuser/ipfsdata
 #VOLUME /home/safeuser/orbitdb
 
+EXPOSE 5010
+
 # Start the application.
 COPY start-production.sh start-production.sh
 CMD ["./start-production.sh"]

production/docker/README.md

@@ -0,0 +1,61 @@
+# Docker Containers
+
+The 'production' environment is assumed to be a set of Docker containers orchestrated with Docker Compose. The files in this directory will stand up three Docker containers:
+
+1 An instance of go-ipfs.
+2 An instance of MongoDB.
+3 The JavaScript software in this repository.
+
+The software in this repository depends on the first two containers, so if they aren't running correctly, the application won't run correctly either.
+
+## IPFS
+
+IPFS can be a little tricky to set up. By default, the container uses the following ports:
+
+- 4001 for TCP connections, exposed publicly.
+- 5001 for control by the application, exposed privately.
+- 8080 for an IPFS gateway, consumed by the application, exposed privately.
+
+If you already have an IPFS node running on a the computer, you will need to change the ports to avaid a conflict. To change the ports from the default, you'll need to perform a series of steps, and the order of the steps matter.
+
+1. Edit the `docker-compose.yml` file and change the ports. Then save the file. Here is an example:
+
+```
+ports:
+  - 4101:4101
+  - 172.17.0.1:5101:5101
+  - 172.17.0.1:8180:8180
+```
+
+2. Bring the Docker containers up, and then back down. This will allow the IPFS container to create the config file that you'll need to edit.
+
+- `docker-compose up -d`
+- Wait a few seconds.
+- `docker-compose down`
+
+3. Update the generated config file at `../data/go-ipfs/data/config`, to update the ports in the config file, like this:
+
+```
+"Addresses": {
+    "API": "/ip4/0.0.0.0/tcp/5101",
+    "Announce": [],
+    "AppendAnnounce": [],
+    "Gateway": "/ip4/0.0.0.0/tcp/8180",
+    "NoAnnounce": [],
+    "Swarm": [
+      "/ip4/0.0.0.0/tcp/4101",
+      "/ip6/::/tcp/4101",
+      "/ip4/0.0.0.0/udp/4101/quic",
+      "/ip6/::/udp/4101/quic"
+    ]
+  },
+
+```
+
+4. Update the port changes in the `start-production.sh` shell script. This tells the application which ports to use, in order to control the IPFS node, are are used when signaling other nodes.
+
+5. Quickly rebuild the containers, to add the modified `start-production.sh` shell script to the application Docker container:
+
+- `docker-compose build`
+
+6. Now start the containers, and the port changes to IPFS should be complete.

production/docker/docker-compose.yml

@@ -1,8 +1,9 @@
 # Start the service with the command 'docker-compose up -d'
 
-version: '2'
+version: '3.9'
+
 services:
-  bch-wallet-mongodb:
+  mongo-bch-service:
     image: mongo
     container_name: mongo-bch-service
     ports:
@@ -12,6 +13,35 @@ services:
     command: mongod --logpath=/dev/null # -- quiet
     restart: always
 
+  ipfs-bch-wallet:
+    image: ipfs/go-ipfs:v0.11.0
+    container_name: ipfs-bch-wallet
+    environment:
+      MY_ENV_VAR: 'placeholder'
+    logging:
+      driver: 'json-file'
+      options:
+        max-size: '10m'
+        max-file: '10'
+    mem_limit: 2000mb
+    ports:
+      - 4001:4001
+      - 172.17.0.1:5001:5001
+      - 172.17.0.1:8080:8080
+    volumes:
+      - ../data/go-ipfs/data:/data/ipfs
+      - ../data/go-ipfs/export:/export
+    # https://docs.docker.com/compose/compose-file/compose-file-v3/#command
+    # https://github.com/ipfs/go-ipfs/blob/91c52657166bcf86f2476926e4fe56694dc26562/Dockerfile#L115
+    command:
+      [
+        'daemon',
+        '--migrate=true',
+        '--agent-version-suffix=docker',
+        '--enable-pubsub-experiment'
+      ]
+    restart: always
+
   bch-wallet-service:
     build: ./
     container_name: bch-wallet-service
@@ -22,11 +52,10 @@ services:
         max-file: '10'
     mem_limit: 1000mb
     links:
-      - bch-wallet-mongodb
+      - mongo-bch-service
+      - ipfs-bch-wallet
     ports:
-      # <host port>:<container port>
-      - '5001:5001' # REST API
-      - '5668:5668' # IPFS TCP Port
-      - '5669:5669' # IPFS WS Port
+      - '5010:5010' # <host port>:<container port>
     volumes:
       - ../data/ipfsdata:/home/safeuser/ipfs-bch-wallet-service/.ipfsdata
+    restart: always

production/docker/start-production.sh

@@ -11,13 +11,22 @@ export MNEMONIC="olive two muscle bottom coral ancient wait legend bronze useful
 export COORD_NAME=bch-wallet-service-generic
 
 # Allow this node to function as a circuit relay. It must not be behind a firewall.
-#export ENABLE_CIRCUIT_RELAY=true
+#export ENABLE_CIRCUIT_RELAY=1
 # For browsers to use your circuit realy, you must set up a domain, SSL certificate,
 # and you must forward that subdomain to the IPFS_WS_PORT.
 #export CR_DOMAIN=subdomain.yourdomain.com
 
 # Debug level. 0 = minimal info. 2 = max info.
-export DEBUG_LEVEL=1
+export DEBUG_LEVEL=2
+
+# BCH Wallet Service Settings
+# Change APISERVER to point to your own installation of bch-api
+export APISERVER=https://api.fullstack.cash/v5/
+# These settings are used to proxy a subscription to FullStack.cash.
+export GET_JWT_AT_STARTUP=1
+export AUTHSERVER=https://auth.fullstack.cash
+export FULLSTACKLOGIN=demo@demo.com
+export FULLSTACKPASS=demo
 
 # END: Optional configuration settings
 
@@ -25,17 +34,14 @@ export DEBUG_LEVEL=1
 # Production database connection string.
 export DBURL=mongodb://172.17.0.1:5555/ipfs-service-prod
 
-# Customize these environment variables for your own installation.
-export GET_JWT_AT_STARTUP=1
-export AUTHSERVER=https://auth.fullstack.cash
-export APISERVER=https://api.fullstack.cash/v5/
-export FULLSTACKLOGIN=demo@demo.com
-export FULLSTACKPASS=demo
-
-# Ports
-export PORT=5001 # REST API
-export IPFS_TCP_PORT=5668 # IPSF TCP Port
-export IPFS_WS_PORT=5669 # IPFS WS Port
+# Configure REST API port
+export PORT=5010
 
+# Production settings using external go-ipfs node.
 export SVC_ENV=production
+export IPFS_HOST=172.17.0.1
+export IPFS_API_PORT=5001
+export IPFS_TCP_PORT=4001
+#export IPFS_WS_PORT=4002
+
 npm start

shell-scripts/local-external-ipfs-node.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# This script is an example for running a production environment, which is
+# defined by running an external go-ipfs node.
+
+# Ports
+export PORT=5010 # REST API port
+
+# The human-readible name that is used when displaying data about this node.
+export COORD_NAME=ipfs-service-provider-generic
+
+# This is used for end-to-end encryption (e2ee).
+export MNEMONIC="churn aisle shield silver ladder swear hunt slim pen demand spoil veteran"
+
+# 0 = less verbose. 3 = most verbose
+export DEBUG_LEVEL=2
+
+# Production settings that use external IPFS node.
+# https://github.com/christroutner/docker-ipfs
+export SVC_ENV=production
+#export IPFS_HOST=localhost
+export IPFS_HOST=172.17.0.1
+export IPFS_API_PORT=5001
+
+# Configure IPFS ports
+export IPFS_TCP_PORT=4001
+#export IPFS_WS_PORT=5269
+
+# MongoDB connection string.
+export DBURL=mongodb://localhost:27017/ipfs-service-dev
+
+npm start

src/adapters/ipfs/ipfs-coord.js

@@ -63,7 +63,7 @@ class IpfsCoordAdapter {
       }
     }
 
-    this.ipfsCoord = new this.IpfsCoord({
+    const ipfsCoordOptions = {
       ipfs: this.ipfs,
       type: 'node.js',
       // type: 'browser',
@@ -75,7 +75,14 @@ class IpfsCoordAdapter {
       announceJsonLd: this.config.announceJsonLd,
       mnemonic: this.config.mnemonic,
       debugLevel: this.config.debugLevel
-    })
+    }
+
+    // Production env uses external go-ipfs node.
+    if (this.config.isProduction) {
+      ipfsCoordOptions.nodeType = 'external'
+    }
+
+    this.ipfsCoord = new this.IpfsCoord(ipfsCoordOptions)
 
     // Wait for the ipfs-coord library to signal that it is ready.
     await this.ipfsCoord.start({ type: 'node.js' })

src/adapters/ipfs/ipfs.js

@@ -10,8 +10,11 @@
 
 // Global npm libraries
 // const IPFS = require('ipfs')
-const IPFS = require('@chris.troutner/ipfs')
+// const IPFS = require('@chris.troutner/ipfs')
+const IPFSembedded = require('ipfs')
+const IPFSexternal = require('ipfs-http-client')
 const fs = require('fs')
+const http = require('http')
 
 // Local libraries
 const config = require('../../../config')
@@ -21,11 +24,17 @@ const IPFS_DIR = './.ipfsdata/ipfs'
 class IpfsAdapter {
   constructor (localConfig) {
     // Encapsulate dependencies
-    this.IPFS = IPFS
+    this.config = config
+
+    // Choose the IPFS constructor based on the config settings.
+    this.IPFS = IPFSembedded // default
+    if (this.config.isProduction) {
+      this.IPFS = IPFSexternal
+    }
 
     // Properties of this class instance.
     this.isReady = false
-    this.config = config
+
     this.fs = fs
   }
 
@@ -33,7 +42,7 @@ class IpfsAdapter {
   async start () {
     try {
       // Ipfs Options
-      const ipfsOptions = {
+      const ipfsOptionsEmbedded = {
         repo: IPFS_DIR,
         start: true,
         config: {
@@ -64,6 +73,17 @@ class IpfsAdapter {
         }
       }
 
+      const ipfsOptionsExternal = {
+        host: this.config.ipfsHost,
+        port: this.config.ipfsApiPort,
+        agent: http.Agent({ keepAlive: true, maxSockets: 2000 })
+      }
+
+      let ipfsOptions = ipfsOptionsEmbedded
+      if (this.config.isProduction) {
+        ipfsOptions = ipfsOptionsExternal
+      }
+
       // Create a new IPFS node.
       this.ipfs = await this.IPFS.create(ipfsOptions)
 

swarm.key

@@ -0,0 +1,3 @@
+/key/swarm/psk/1.0.0/
+/base16/
+bbd935b70105b03ebd0c6a3c2d2730cd22fc0d18c490ccf689b6c8a22e1bed2a