Merge pull request #15 from Permissionless-Software-Foundation/ct-unstable

Syncing with upstream ipfs-service-provider

README.md

@@ -8,13 +8,17 @@ When started, this web server starts an [IPFS](https://ipfs.io) node and connect
 
 ## Requirements
 
-- node **^14.17.0**
-- npm **^7.13.0**
+- node **^14.18.2**
+- npm **^8.3.0**
+- Docker **^20.10.8**
+- Docker Compose **^1.27.4**
 
 ## Installation
 
 ### Development Environment
 
+**Note:** This software now uses an external go-ipfs IPFS node. The instructions below have not been updated to reflect this.
+
 A development environment will allow you modify the code on-the-fly and contribute to the code base of this repository. [PM2](https://www.npmjs.com/package/pm2) is recommended for running this code base as an IPFS Circuit Relay.
 
 - [Video: Installing ipfs-service-provider](https://youtu.be/Z0NsboIVN44)
@@ -31,17 +35,27 @@ npm install
 
 ### Production Environment
 
-The [docker](./production/docker) directory contains a Dockerfile for building a production deployment. However, there is currently [a bug](https://github.com/Permissionless-Software-Foundation/ipfs-service-provider/issues/38) preventing the Docker container from being used as a Circuit Relay.
+The [docker](./production/docker) directory contains a Dockerfile for building a production deployment.
 
 ```
 docker-compose build --no-cache
 docker-compose up -d
 ```
 
+- Bring the containers up once, then take them back down with `docker-compose down`.
+- Copy the [swarm.key](./swarm.key) file to `ipfs-service-provider/production/data/go-ipfs/data/swarm.key`.
+- Bring the containers back up with `docker-compose up -d`.
+
 ### Operation Notes
 
 - There is a memory leak in the version of js-ipfs. The app is currently configured to shut down every 8 hours to flush memory. It relies on a process manager like pm2, Docker, or systemd to restart the app after it shuts down, in order to ensure continuous operation.
 
+- The PSF network operates as a private network. It does not connect or interact with the wider PSF network, relying instead on gateways to bridge the two networks, when they need to share content. This improves the performance and experience for everyone in the PSF network. To join the network, you'll need to add the [swarm.key](./swarm.key) file to the IPFS data folder.
+
+- [Instructions on setting up IPFS private networks.](https://github.com/ipfs/go-ipfs/blob/master/docs/experimental-features.md#private-networks)
+- For external installations, the swarm.key file will typically go in `~/.ipfs/swarm.key`
+- For production Docker containers, the key would go in `ipfs-service-provider/production/data/go-ipfs/data/swarm.key`
+
 ## Structure
 
 The file layout of this repository differs from the koa-api-boilerplate. Instead, it follows the file layout of [Clean Architecture](troutsblog.com/blog/clean-architecture).

bin/server.js

@@ -68,6 +68,11 @@ class Server {
       app.use(passport.initialize())
       app.use(passport.session())
 
+      // Enable CORS for testing
+      // THIS IS A SECURITY RISK. COMMENT OUT FOR PRODUCTION
+      // Dev Note: This line must come BEFORE controllers.attachRESTControllers()
+      app.use(cors({ origin: '*' }))
+
       // Attach REST API and JSON RPC controllers to the app.
       const Controllers = require('../src/controllers')
       const controllers = new Controllers()
@@ -75,10 +80,6 @@ class Server {
 
       app.controllers = controllers
 
-      // Enable CORS for testing
-      // THIS IS A SECURITY RISK. COMMENT OUT FOR PRODUCTION
-      app.use(cors({ origin: '*' }))
-
       // MIDDLEWARE END
 
       console.log(`Running server in environment: ${config.env}`)
@@ -91,15 +92,11 @@ class Server {
       const success = await this.adminLib.createSystemUser()
       if (success) console.log('System admin user created.')
 
-      // Attach the other IPFS controllers
-      await controllers.attachControllers(app)
-
-      // ipfs-coord has a memory leak. This app shuts down after 4 hours. It
-      // expects to be run by Docker or pm2, which can automatically restart
-      // the app.
-      setTimeout(function () {
-        process.exit(0)
-      }, 60000 * 60 * 2) // 2 hours
+      // Attach the other IPFS controllers.
+      // Skip if this is a test environment.
+      if (config.env !== 'test') {
+        await controllers.attachControllers(app)
+      }
 
       return app
     } catch (err) {

package-lock.json

@@ -9,12 +9,12 @@
       "version": "1.0.0",
       "license": "MIT",
       "dependencies": {
-        "@psf/bch-js": "4.20.28",
-        "axios": "0.21.1",
+        "@psf/bch-js": "5.2.0",
+        "axios": "0.25.0",
         "bcryptjs": "2.4.3",
         "glob": "7.1.6",
         "ipfs": "0.60.0",
-        "ipfs-coord": "6.8.5",
+        "ipfs-coord": "6.8.10",
         "ipfs-http-client": "55.0.0",
         "jsonrpc-lite": "2.2.0",
         "jsonwebtoken": "8.5.1",
@@ -60,6 +60,39 @@
         "uuid": "8.3.2"
       }
     },
+    "../ipfs-coord": {
+      "version": "6.0.0",
+      "extraneous": true,
+      "license": "MIT",
+      "dependencies": {
+        "axios": "0.21.4",
+        "bch-encrypt-lib": "2.0.0",
+        "orbit-db": "0.28.0"
+      },
+      "devDependencies": {
+        "@psf/bch-js": "4.20.10",
+        "apidoc": "0.25.0",
+        "chai": "4.2.0",
+        "coveralls": "3.1.0",
+        "eslint": "7.17.0",
+        "eslint-config-prettier": "7.1.0",
+        "eslint-config-standard": "16.0.2",
+        "eslint-plugin-node": "11.1.0",
+        "eslint-plugin-prettier": "3.3.1",
+        "eslint-plugin-standard": "4.0.0",
+        "husky": "4.3.6",
+        "lodash.clonedeep": "4.5.0",
+        "mocha": "8.4.0",
+        "nyc": "15.1.0",
+        "semantic-release": "17.4.4",
+        "sinon": "9.2.2",
+        "standard": "16.0.3"
+      },
+      "peerDependencies": {
+        "ipfs": "0.60.0",
+        "ipfs-http-client": "55.0.0"
+      }
+    },
     "node_modules/@assemblyscript/loader": {
       "version": "0.9.4",
       "resolved": "http://94.130.170.209:4873/@assemblyscript%2floader/-/loader-0.9.4.tgz",
@@ -2198,9 +2231,9 @@
       "license": "BSD-3-Clause"
     },
     "node_modules/@psf/bch-js": {
-      "version": "4.20.28",
-      "resolved": "http://94.130.170.209:4873/@psf%2fbch-js/-/bch-js-4.20.28.tgz",
-      "integrity": "sha512-tlWfYTUnmAzrFZDHTuP2CccE6fp20j4XcCIJiJYlpaKzxB0ilH8CEVWBQaMY+Fb+eaTrlSdiaDC5MBWV4klpQg==",
+      "version": "5.2.0",
+      "resolved": "http://94.130.170.209:4873/@psf%2fbch-js/-/bch-js-5.2.0.tgz",
+      "integrity": "sha512-1QbKrdtww+nISvuwKe9X8/9GrqZWswjn0AChm0+UqobEV6WqGUXbsYL7621nvUaWzHz040evgOAiM6VaTtfNhg==",
       "license": "MIT",
       "dependencies": {
         "@chris.troutner/bip32-utils": "1.0.5",
@@ -3578,12 +3611,12 @@
       "license": "MIT"
     },
     "node_modules/axios": {
-      "version": "0.21.1",
-      "resolved": "http://94.130.170.209:4873/axios/-/axios-0.21.1.tgz",
-      "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==",
+      "version": "0.25.0",
+      "resolved": "http://94.130.170.209:4873/axios/-/axios-0.25.0.tgz",
+      "integrity": "sha512-cD8FOb0tRH3uuEe6+evtAbgJtfxr7ly3fQjYcMcuPlgkwVS9xboaVIpcDV+cYQe+yGykgwZCs1pzjntcGa6l5g==",
       "license": "MIT",
       "dependencies": {
-        "follow-redirects": "^1.10.0"
+        "follow-redirects": "^1.14.7"
       }
     },
     "node_modules/backo2": {
@@ -7544,9 +7577,9 @@
       "license": "MIT"
     },
     "node_modules/follow-redirects": {
-      "version": "1.14.6",
-      "resolved": "http://94.130.170.209:4873/follow-redirects/-/follow-redirects-1.14.6.tgz",
-      "integrity": "sha512-fhUl5EwSJbbl8AR+uYL2KQDxLkdSjZGR36xy46AO7cOMTrCMON6Sa28FmAnC2tRTDbd/Uuzz3aJBv7EBN7JH8A==",
+      "version": "1.14.7",
+      "resolved": "http://94.130.170.209:4873/follow-redirects/-/follow-redirects-1.14.7.tgz",
+      "integrity": "sha512-+hbxoLbFMbRKDwohX8GkTataGqO6Jb7jGwpAlwgy2bIz25XtRm7KEzJM76R1WiNT5SwZkX4Y75SwBolkpmE7iQ==",
       "funding": [
         {
           "type": "individual",
@@ -9735,9 +9768,9 @@
       }
     },
     "node_modules/ipfs-coord": {
-      "version": "6.8.5",
-      "resolved": "http://94.130.170.209:4873/ipfs-coord/-/ipfs-coord-6.8.5.tgz",
-      "integrity": "sha512-IQR++uPZuHKOxrT49raHho/CwQe1tKe2Q1Ed0hGDfarXJTtfmMmNH3jGSecyd5BYZNeugL67BnGuvoYrZFY7AA==",
+      "version": "6.8.10",
+      "resolved": "http://94.130.170.209:4873/ipfs-coord/-/ipfs-coord-6.8.10.tgz",
+      "integrity": "sha512-04nlX4AkfqLuuLzpMcHnpv++iOETLRkzFA2GfAmY90r99HynfHt3DypP6loVdAnD34iWgD+tLhKNQ7QQM7cgfQ==",
       "license": "MIT",
       "dependencies": {
         "axios": "0.21.4",
@@ -12066,6 +12099,15 @@
         "axios": "^0.21.1"
       }
     },
+    "node_modules/jwt-bch-lib/node_modules/axios": {
+      "version": "0.21.4",
+      "resolved": "http://94.130.170.209:4873/axios/-/axios-0.21.4.tgz",
+      "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==",
+      "license": "MIT",
+      "dependencies": {
+        "follow-redirects": "^1.14.0"
+      }
+    },
     "node_modules/k-bucket": {
       "version": "5.1.0",
       "resolved": "http://94.130.170.209:4873/k-bucket/-/k-bucket-5.1.0.tgz",
@@ -27347,9 +27389,9 @@
       "integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA="
     },
     "@psf/bch-js": {
-      "version": "4.20.28",
-      "resolved": "http://94.130.170.209:4873/@psf%2fbch-js/-/bch-js-4.20.28.tgz",
-      "integrity": "sha512-tlWfYTUnmAzrFZDHTuP2CccE6fp20j4XcCIJiJYlpaKzxB0ilH8CEVWBQaMY+Fb+eaTrlSdiaDC5MBWV4klpQg==",
+      "version": "5.2.0",
+      "resolved": "http://94.130.170.209:4873/@psf%2fbch-js/-/bch-js-5.2.0.tgz",
+      "integrity": "sha512-1QbKrdtww+nISvuwKe9X8/9GrqZWswjn0AChm0+UqobEV6WqGUXbsYL7621nvUaWzHz040evgOAiM6VaTtfNhg==",
       "requires": {
         "@chris.troutner/bip32-utils": "1.0.5",
         "@psf/bip21": "2.0.1",
@@ -28413,11 +28455,11 @@
       "integrity": "sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA=="
     },
     "axios": {
-      "version": "0.21.1",
-      "resolved": "http://94.130.170.209:4873/axios/-/axios-0.21.1.tgz",
-      "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==",
+      "version": "0.25.0",
+      "resolved": "http://94.130.170.209:4873/axios/-/axios-0.25.0.tgz",
+      "integrity": "sha512-cD8FOb0tRH3uuEe6+evtAbgJtfxr7ly3fQjYcMcuPlgkwVS9xboaVIpcDV+cYQe+yGykgwZCs1pzjntcGa6l5g==",
       "requires": {
-        "follow-redirects": "^1.10.0"
+        "follow-redirects": "^1.14.7"
       }
     },
     "backo2": {
@@ -31354,9 +31396,9 @@
       "integrity": "sha1-kV4tbQI8Q9UiStn20qPEFW9XEvU="
     },
     "follow-redirects": {
-      "version": "1.14.6",
-      "resolved": "http://94.130.170.209:4873/follow-redirects/-/follow-redirects-1.14.6.tgz",
-      "integrity": "sha512-fhUl5EwSJbbl8AR+uYL2KQDxLkdSjZGR36xy46AO7cOMTrCMON6Sa28FmAnC2tRTDbd/Uuzz3aJBv7EBN7JH8A=="
+      "version": "1.14.7",
+      "resolved": "http://94.130.170.209:4873/follow-redirects/-/follow-redirects-1.14.7.tgz",
+      "integrity": "sha512-+hbxoLbFMbRKDwohX8GkTataGqO6Jb7jGwpAlwgy2bIz25XtRm7KEzJM76R1WiNT5SwZkX4Y75SwBolkpmE7iQ=="
     },
     "for-each": {
       "version": "0.3.3",
@@ -32906,9 +32948,9 @@
       }
     },
     "ipfs-coord": {
-      "version": "6.8.5",
-      "resolved": "http://94.130.170.209:4873/ipfs-coord/-/ipfs-coord-6.8.5.tgz",
-      "integrity": "sha512-IQR++uPZuHKOxrT49raHho/CwQe1tKe2Q1Ed0hGDfarXJTtfmMmNH3jGSecyd5BYZNeugL67BnGuvoYrZFY7AA==",
+      "version": "6.8.10",
+      "resolved": "http://94.130.170.209:4873/ipfs-coord/-/ipfs-coord-6.8.10.tgz",
+      "integrity": "sha512-04nlX4AkfqLuuLzpMcHnpv++iOETLRkzFA2GfAmY90r99HynfHt3DypP6loVdAnD34iWgD+tLhKNQ7QQM7cgfQ==",
       "requires": {
         "axios": "0.21.4",
         "bch-encrypt-lib": "2.0.0",
@@ -34686,6 +34728,16 @@
       "integrity": "sha512-CZs3jUHJlWvConvoTeLDogDEwUysKeojYJxfUhbwJKJrtDCSfi3ZWbsZZ8WN30Xs28aZPDB+qlHPJlTXQmup0w==",
       "requires": {
         "axios": "^0.21.1"
+      },
+      "dependencies": {
+        "axios": {
+          "version": "0.21.4",
+          "resolved": "http://94.130.170.209:4873/axios/-/axios-0.21.4.tgz",
+          "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==",
+          "requires": {
+            "follow-redirects": "^1.14.0"
+          }
+        }
       }
     },
     "k-bucket": {

package.json

@@ -23,12 +23,12 @@
   },
   "repository": "Permissionless-Software-Foundation/ipfs-bch-wallet-consumer",
   "dependencies": {
-    "@psf/bch-js": "4.20.28",
-    "axios": "0.21.1",
+    "@psf/bch-js": "5.2.0",
+    "axios": "0.25.0",
     "bcryptjs": "2.4.3",
     "glob": "7.1.6",
     "ipfs": "0.60.0",
-    "ipfs-coord": "6.8.5",
+    "ipfs-coord": "6.8.10",
     "ipfs-http-client": "55.0.0",
     "jsonrpc-lite": "2.2.0",
     "jsonwebtoken": "8.5.1",

src/adapters/bch/index.js

@@ -198,8 +198,9 @@ class BchAdapter {
 
       // Wait for data to come back from the wallet service.
       const data = await this.waitForRPCResponse(rpcId)
+      // console.log(`data: ${JSON.stringify(data, null, 2)}`)
 
-      return data[0]
+      return data
     } catch (err) {
       // console.log('createUser() error: ', err)
       wlogger.error('Error in use-cases/bch.js/getUtxos()')
@@ -400,6 +401,9 @@ class BchAdapter {
             dataFound = true
             // console.log('data was found in the queue')
 
+            // console.log(
+            //   `rawData.payload: ${JSON.stringify(rawData.payload, null, 2)}`
+            // )
             data = rawData.payload.result.value
 
             // Remove the data from the queue

src/adapters/fullstack-jwt.js

@@ -52,12 +52,6 @@ class FullStackJWT {
   // Get's a JWT token from FullStack.cash.
   async getJWT () {
     try {
-      // Skip connecting FullStack.cash auth server to the network if this is an E2E test.
-      if (process.env.TEST_TYPE === 'e2e') {
-        this.apiToken = 'faketoken'
-        return this.apiToken
-      }
-
       // Log into the auth server.
       await this.jwtLib.register()
 

src/adapters/ipfs/ipfs.js

@@ -14,6 +14,7 @@
 const IPFSembedded = require('ipfs')
 const IPFSexternal = require('ipfs-http-client')
 const fs = require('fs')
+const http = require('http')
 
 // Local libraries
 const config = require('../../../config')
@@ -74,7 +75,8 @@ class IpfsAdapter {
 
       const ipfsOptionsExternal = {
         host: this.config.ipfsHost,
-        port: this.config.ipfsApiPort
+        port: this.config.ipfsApiPort,
+        agent: http.Agent({ keepAlive: true, maxSockets: 2000 })
       }
 
       let ipfsOptions = ipfsOptionsEmbedded

swarm.key

@@ -0,0 +1,3 @@
+/key/swarm/psk/1.0.0/
+/base16/
+bbd935b70105b03ebd0c6a3c2d2730cd22fc0d18c490ccf689b6c8a22e1bed2a