Syncing with upstream ipfs-service-provider. Fixing merge conflicts

src/adapters/ipfs/ipfs.js

@@ -19,17 +19,20 @@ import { noise } from '@chainsafe/libp2p-noise'
 import { yamux } from '@chainsafe/libp2p-yamux'
 // import { bootstrap } from '@libp2p/bootstrap'
 // import { identifyService } from 'libp2p/identify'
-import { identify } from '@libp2p/identify'
-// import { circuitRelayServer, circuitRelayTransport } from 'libp2p/circuit-relay'
-import { circuitRelayServer } from '@libp2p/circuit-relay-v2'
+import { identify, identifyPush } from '@libp2p/identify'
+import { circuitRelayServer, circuitRelayTransport } from '@libp2p/circuit-relay-v2'
+// import { circuitRelayServer } from '@libp2p/circuit-relay-v2'
 import { gossipsub } from '@chainsafe/libp2p-gossipsub'
 import { webSockets } from '@libp2p/websockets'
 import { publicIpv4 } from 'public-ip'
 import { multiaddr } from '@multiformats/multiaddr'
-// import { webRTC } from '@libp2p/webrtc'
+import { webRTC, webRTCDirect } from '@libp2p/webrtc'
 import { keychain } from '@libp2p/keychain'
-import { defaultLogger } from '@libp2p/logger'
 import { unixfs } from '@helia/unixfs'
+// import { generateKeyPairFromSeed } from '@libp2p/crypto/keys'
+// import crypto from 'crypto'
+import { ping } from '@libp2p/ping'
+import { loadOrCreateSelfKey } from '@libp2p/config'
 
 // Local libraries
 import config from '../../../config/index.js'
@@ -64,8 +67,8 @@ class IpfsAdapter {
     this.createNode = this.createNode.bind(this)
     this.stop = this.stop.bind(this)
     this.ensureBlocksDir = this.ensureBlocksDir.bind(this)
+    // this.getPrivateKey = this.getPrivateKey.bind(this)
     this.getSeed = this.getSeed.bind(this)
-    this.getKeychain = this.getKeychain.bind(this)
   }
 
   // Start an IPFS node.
@@ -111,19 +114,6 @@ class IpfsAdapter {
     }
   }
 
-  async getKeychain (datastore) {
-    const keychainInit = {
-      pass: await this.getSeed()
-    }
-
-    const chain = this.keychain(keychainInit)({
-      datastore,
-      logger: defaultLogger()
-    })
-
-    return chain
-  }
-
   // This function creates an IPFS node using Helia.
   // It returns the node as an object.
   async createNode () {
@@ -132,44 +122,38 @@ class IpfsAdapter {
       const blockstore = new FsBlockstore(`${IPFS_DIR}/blockstore`)
       const datastore = new FsDatastore(`${IPFS_DIR}/datastore`)
 
-      // const keychainInit = {
-      //   pass: await this.getSeed()
-      // }
+      // Get the private key.
+      // const privateKey = await this.getPrivateKey()
+      // console.log('privateKey: ', privateKey)
 
       // Create an identity
-      let peerId
-      // console.log('this.keychain: ', this.keychain)
-      // const chain = this.keychain(keychainInit)({
-      //   datastore,
-      //   logger: defaultLogger()
-      // })
-      const chain = await this.getKeychain(datastore)
-      try {
-        peerId = await chain.exportPeerId('myKey')
-      } catch (err) {
-        await chain.createKey('myKey', 'Ed25519', 4096)
-        peerId = await chain.exportPeerId('myKey')
+      const keychainInit = {
+        selfKey: 'myKey',
+        pass: await this.getSeed()
       }
+      const privateKey = await loadOrCreateSelfKey(datastore, keychainInit)
 
       // Configure services
       const services = {
         identify: identify(),
-        pubsub: gossipsub({ allowPublishToZeroTopicPeers: true })
+        identifyPush: identifyPush(),
+        pubsub: gossipsub({ allowPublishToZeroTopicPeers: true }),
+        ping: ping(),
+        keychain: keychain(keychainInit)
       }
       if (this.config.isCircuitRelay) {
         console.log('Helia (IPFS) node IS configured as Circuit Relay')
         services.relay = circuitRelayServer({ // makes the node function as a relay server
           hopTimeout: 30 * 1000, // incoming relay requests must be resolved within this time limit
-          advertise: true,
           reservations: {
             maxReservations: 15, // how many peers are allowed to reserve relay slots on this server
             reservationClearInterval: 300 * 1000, // how often to reclaim stale reservations
             applyDefaultLimit: true, // whether to apply default data/duration limits to each relayed connection
             defaultDurationLimit: 2 * 60 * 1000, // the default maximum amount of time a relayed connection can be open for
-            defaultDataLimit: BigInt(2 << 7), // the default maximum number of bytes that can be transferred over a relayed connection
-            maxInboundHopStreams: 32, // how many inbound HOP streams are allow simultaneously
-            maxOutboundHopStreams: 64 // how many outbound HOP streams are allow simultaneously
-          }
+            defaultDataLimit: BigInt(2 << 7) // the default maximum number of bytes that can be transferred over a relayed connection
+          },
+          maxInboundHopStreams: 32, // how many inbound HOP streams are allow simultaneously
+          maxOutboundHopStreams: 64 // how many outbound HOP streams are allow simultaneously
         })
       } else {
         console.log('Helia (IPFS) node IS NOT configured as Circuit Relay')
@@ -177,28 +161,31 @@ class IpfsAdapter {
 
       const transports = [
         tcp(),
-        webSockets()
-        // circuitRelayTransport({
-        //   discoverRelays: 3,
-        //   reservationConcurrency: 3
-        // }),
-        // webRTC()
+        webSockets(),
+        circuitRelayTransport({
+          discoverRelays: 3,
+          reservationConcurrency: 3
+        }),
+        webRTC(),
+        webRTCDirect()
       ]
 
       // libp2p is the networking layer that underpins Helia
       const libp2p = await this.createLibp2p({
-        peerId,
+        // peerId,
+        privateKey,
         datastore,
         addresses: {
           listen: [
             '/ip4/127.0.0.1/tcp/0',
             `/ip4/0.0.0.0/tcp/${this.config.ipfsTcpPort}`,
             `/ip4/0.0.0.0/tcp/${this.config.ipfsWsPort}/ws`,
-            '/webrtc'
+            '/webrtc',
+            '/p2p-circuit'
           ]
         },
         transports,
-        connectionEncryption: [
+        connectionEncrypters: [
           noise()
         ],
         streamMuxers: [
@@ -244,6 +231,8 @@ class IpfsAdapter {
 
       !this.fs.existsSync(`${IPFS_DIR}/datastore`) && this.fs.mkdirSync(`${IPFS_DIR}/datastore`)
 
+      !this.fs.existsSync(`${IPFS_DIR}/datastore/pkcs8`) && this.fs.mkdirSync(`${IPFS_DIR}/datastore/pkcs8`)
+
       // !fs.existsSync(`${IPFS_DIR}/datastore/peers`) && fs.mkdirSync(`${IPFS_DIR}/datastore/peers`)
 
       return true
@@ -280,6 +269,48 @@ class IpfsAdapter {
       throw err
     }
   }
+
+  // Get the private key from disk, or generate a new one and save it,
+  // if it doesn't exist.
+  // async getPrivateKey () {
+  //   try {
+  //     const filename = `${IPFS_DIR}/privkey.json`
+
+  //     let privKeyHex
+  //     try {
+  //       // Try to read the JSON file containing the seed.
+  //       privKeyHex = await this.jsonFiles.readJSON(filename)
+  //       // console.log('saved privKeyHex: ', privKeyHex)
+  //     } catch (err) {
+  //       // Generate a new private key and save it to disk.
+
+  //       // Generate a new private key and save it to disk.
+  //       const randomBuffer = crypto.randomBytes(32)
+  //       // console.log('randomBuffer: ', randomBuffer)
+
+  //       privKeyHex = randomBuffer.toString('hex')
+  //       // console.log('new privKeyHex: ', privKeyHex)
+
+  //       // Save the newly generated seed
+  //       await this.jsonFiles.writeJSON(privKeyHex, filename)
+  //     }
+
+  //     // Convert the saved hex string to a buffer.
+  //     const privKeyBuf = Buffer.from(privKeyHex, 'hex')
+
+  //     // Convert the buffer to a Uint8Array 'seed'
+  //     const seed = new Uint8Array(privKeyBuf.buffer, privKeyBuf.byteOffset, privKeyBuf.byteLength)
+  //     // console.log('seed: ', seed)
+
+  //     // Generate a ED25519 key pair.
+  //     const privKey = await generateKeyPairFromSeed('Ed25519', seed)
+
+  //     return privKey
+  //   } catch (err) {
+  //     console.error('Error in adapters/ipfs/ipfs.js/getPrivateKey(): ', err)
+  //     throw err
+  //   }
+  // }
 }
 
 export default IpfsAdapter

src/controllers/index.js

@@ -21,6 +21,13 @@ class Controllers {
     this.useCases = new UseCases({ adapters: this.adapters })
     this.timerControllers = new TimerControllers({ adapters: this.adapters, useCases: this.useCases })
     this.config = config
+
+    // Bind 'this' object to all subfunction
+    this.initAdapters = this.initAdapters.bind(this)
+    this.initUseCases = this.initUseCases.bind(this)
+    this.attachRESTControllers = this.attachRESTControllers.bind(this)
+    this.attachControllers = this.attachControllers.bind(this)
+    this.attachRPCControllers = this.attachRPCControllers.bind(this)
   }
 
   // Spin up any adapter libraries that have async startup needs.

src/controllers/rest-api/index.js

@@ -14,6 +14,7 @@ import LogsRESTController from './logs/index.js'
 import IpfsRESTController from './ipfs/index.js'
 import BCHRESTController from './bch/index.js'
 import config from '../../../config/index.js'
+import UsageRESTController from './usage/index.js'
 
 class RESTControllers {
   constructor (localConfig = {}) {
@@ -31,6 +32,9 @@ class RESTControllers {
       )
     }
 
+    // Bind 'this' object to all subfunctions.
+    this.attachRESTControllers = this.attachRESTControllers.bind(this)
+
     // Encapsulate dependencies
     this.config = config
   }
@@ -66,6 +70,10 @@ class RESTControllers {
     // Attach the REST API Controllers associated with the /fulcrum route
     const bchRESTController = new BCHRESTController(dependencies)
     bchRESTController.attach(app)
+    
+    // Attach the REST API Controllers associated with the /usage route
+    const usageRESTController = new UsageRESTController(dependencies)
+    usageRESTController.attach(app)
   }
 }
 

src/controllers/rest-api/usage/controller.js

@@ -0,0 +1,119 @@
+/*
+  REST API Controller library for the /usage route
+*/
+
+// Global npm libraries
+
+// Local libraries
+import wlogger from '../../../adapters/wlogger.js'
+
+class UsageRESTControllerLib {
+  constructor (localConfig = {}) {
+    // Dependency Injection.
+    this.adapters = localConfig.adapters
+    if (!this.adapters) {
+      throw new Error(
+        'Instance of Adapters library required when instantiating /usage REST Controller.'
+      )
+    }
+    this.useCases = localConfig.useCases
+    if (!this.useCases) {
+      throw new Error(
+        'Instance of Use Cases library required when instantiating /usage REST Controller.'
+      )
+    }
+
+    // Encapsulate dependencies
+
+    // Bind 'this' object to all subfunctions
+    this.getStatus = this.getStatus.bind(this)
+    this.getTopIps = this.getTopIps.bind(this)
+    this.getTopEndpoints = this.getTopEndpoints.bind(this)
+    this.handleError = this.handleError.bind(this)
+  }
+
+  /**
+   * @api {get} /usage Get status on IPFS infrastructure
+   * @apiPermission public
+   * @apiName GetUsageStatus
+   * @apiGroup REST Usage
+   *
+   * @apiExample Example usage:
+   * curl -H "Content-Type: application/json" -X GET localhost:5020/usage
+   *
+   */
+  getStatus (ctx) {
+    try {
+      // const status = await this.adapters.ipfs.getStatus()
+      const status = this.useCases.usage.getRestSummary()
+
+      ctx.body = { status }
+    } catch (err) {
+      wlogger.error('Error in usage/controller.js/getStatus(): ')
+      // ctx.throw(422, err.message)
+      this.handleError(ctx, err)
+    }
+  }
+
+  /**
+   * @api {get} /usage/ips Get top IP addresses consuming the REST API
+   * @apiPermission public
+   * @apiName GetUsageIPs
+   * @apiGroup REST Usage
+   *
+   * @apiExample Example usage:
+   * curl -H "Content-Type: application/json" -X GET localhost:5020/usage/ips
+   *
+   */
+  getTopIps (ctx) {
+    try {
+      const ips = this.useCases.usage.getTopIps()
+
+      ctx.body = { ips }
+    } catch (err) {
+      wlogger.error('Error in usage/controller.js/getTopIps(): ')
+      // ctx.throw(422, err.message)
+      this.handleError(ctx, err)
+    }
+  }
+
+  /**
+   * @api {get} /usage/endpoints Get top endpoints consumed from the REST API
+   * @apiPermission public
+   * @apiName GetUsageEndpoints
+   * @apiGroup REST Usage
+   *
+   * @apiExample Example usage:
+   * curl -H "Content-Type: application/json" -X GET localhost:5020/usage/endpoints
+   *
+   */
+  getTopEndpoints (ctx) {
+    try {
+      const endpoints = this.useCases.usage.getTopEndpoints()
+
+      ctx.body = { endpoints }
+    } catch (err) {
+      wlogger.error('Error in usage/controller.js/getTopEndpoints(): ')
+      // ctx.throw(422, err.message)
+      this.handleError(ctx, err)
+    }
+  }
+
+  // DRY error handler
+  handleError (ctx, err) {
+    // If an HTTP status is specified by the buisiness logic, use that.
+    if (err.status) {
+      if (err.message) {
+        ctx.throw(err.status, err.message)
+      } else {
+        ctx.throw(err.status)
+      }
+    } else {
+      // By default use a 422 error if the HTTP status is not specified.
+      ctx.throw(422, err.message)
+    }
+  }
+}
+
+// module.exports = IpfsRESTControllerLib
+export default UsageRESTControllerLib

src/controllers/rest-api/usage/index.js

@@ -0,0 +1,65 @@
+/*
+  REST API library for the /usage route.
+*/
+
+// Public npm libraries.
+import Router from 'koa-router'
+
+// Local libraries.
+import UsageRESTControllerLib from './controller.js'
+import Validators from '../middleware/validators.js'
+
+// let _this
+
+class UsageRouter {
+  constructor (localConfig = {}) {
+    // Dependency Injection.
+    this.adapters = localConfig.adapters
+    if (!this.adapters) {
+      throw new Error(
+        'Instance of Adapters library required when instantiating IPFS REST Controller.'
+      )
+    }
+    this.useCases = localConfig.useCases
+    if (!this.useCases) {
+      throw new Error(
+        'Instance of Use Cases library required when instantiating IPFS REST Controller.'
+      )
+    }
+
+    const dependencies = {
+      adapters: this.adapters,
+      useCases: this.useCases
+    }
+
+    // Encapsulate dependencies.
+    this.usageRESTController = new UsageRESTControllerLib(dependencies)
+    this.validators = new Validators()
+
+    // Instantiate the router and set the base route.
+    const baseUrl = '/usage'
+    this.router = new Router({ prefix: baseUrl })
+
+    // _this = this
+  }
+
+  attach (app) {
+    if (!app) {
+      throw new Error(
+        'Must pass app object when attaching REST API controllers.'
+      )
+    }
+
+    // Define the routes and attach the controller.
+    this.router.get('/', this.usageRESTController.getStatus)
+    this.router.get('/ips', this.usageRESTController.getTopIps)
+    this.router.get('/endpoints', this.usageRESTController.getTopEndpoints)
+
+    // Attach the Controller routes to the Koa app.
+    app.use(this.router.routes())
+    app.use(this.router.allowedMethods())
+  }
+}
+
+// module.exports = BchRouter
+export default UsageRouter

src/controllers/timer-controllers.js

@@ -28,6 +28,7 @@ class TimerControllers {
 
     // Bind 'this' object to all subfunctions.
     this.exampleTimerFunc = this.exampleTimerFunc.bind(this)
+    this.cleanUsage = this.cleanUsage.bind(this)
 
     // this.startTimers()
   }
@@ -36,13 +37,15 @@ class TimerControllers {
   startTimers () {
     // Any new timer control functions can be added here. They will be started
     // when the server starts.
-    this.optimizeWalletHandle = setInterval(this.exampleTimerFunc, 60000 * 10)
+    this.optimizeWalletHandle = setInterval(this.exampleTimerFunc, 60000 * 60)
+    this.cleanUsageHandle = setInterval(this.cleanUsage, 60000 * 60) // 1 hour
 
     return true
   }
 
   stopTimers () {
     clearInterval(this.optimizeWalletHandle)
+    clearInterval(this.cleanusageHandle)
   }
 
   // Replace this example function with your own timer handler.
@@ -60,6 +63,20 @@ class TimerControllers {
       return false
     }
   }
+
+  // Clean the usage state so that stats reflect the last 24 hours.
+  cleanUsage () {
+    try {
+      this.useCases.usage.cleanUsage()
+
+      return true
+    } catch (err) {
+      console.error('Error in time-controller.js/cleanUsage(): ', err)
+
+      // Note: Do not throw an error. This is a top-level function.
+      return false
+    }
+  }
 }
 
 export default TimerControllers

src/use-cases/index.js

@@ -7,6 +7,7 @@
 // Local libraries
 import UserUseCases from './user.js'
 import BCHUseCases from './bch/index.js'
+import { UsageUseCases } from './usage-use-cases.js'
 
 class UseCases {
   constructor (localConfig = {}) {
@@ -20,6 +21,7 @@ class UseCases {
     // console.log('use-cases/index.js localConfig: ', localConfig)
     this.user = new UserUseCases(localConfig)
     this.bch = new BCHUseCases(localConfig)
+    this.usage = new UsageUseCases(localConfig)
   }
 
   // Run any startup Use Cases at the start of the app.

src/use-cases/usage-use-cases.js

@@ -0,0 +1,136 @@
+/*
+  Use Case library for tracking usage. This library contains business logic
+  for tracking the usage of REST API and JSON RPC calls. This library is used
+  by admins to keep an eye on how many API calls were made in a 24-hour and
+  1-hour time period.
+*/
+
+// This global variable is used to share data between the REST middleware and
+// the Usage Use Case class instance.
+let restCalls = []
+
+class UsageUseCases {
+  constructor (localConfig = {}) {
+    // console.log('User localConfig: ', localConfig)
+    this.adapters = localConfig.adapters
+    if (!this.adapters) {
+      throw new Error(
+        'Instance of adapters must be passed in when instantiating Usage Use Cases library.'
+      )
+    }
+
+    // Bind 'this' object to all subfunctions
+    this.cleanUsage = this.cleanUsage.bind(this)
+    this.getRestSummary = this.getRestSummary.bind(this)
+    this.getTopIps = this.getTopIps.bind(this)
+    this.getTopEndpoints = this.getTopEndpoints.bind(this)
+
+    // State
+  }
+
+  // Clean up the state by removing entries that are older than 24 hours. This
+  // ensures stats reflect only the last 24 hours.
+  // This function is called by a Timer Controller.
+  cleanUsage () {
+    try {
+      const now = new Date()
+      const twentyFourHoursAgo = now.getTime() - (60000 * 60 * 24)
+
+      restCalls = restCalls.filter(x => x.timestamp > twentyFourHoursAgo)
+      return restCalls
+    } catch (err) {
+      console.error('Error in usage-use-cases.js/cleanUsage()')
+      throw err
+    }
+  }
+
+  // Track the calls to a REST API
+  getRestSummary (inObj = {}) {
+    try {
+      console.log(`getRestSummary(): There have been ${restCalls.length} REST calls`)
+
+      return restCalls.length
+    } catch (err) {
+      console.error('Error in usage-use-cases.js/getRestSummary()')
+      throw err
+    }
+  }
+
+  // Get the top 20 IP addresses from the stats.
+  getTopIps () {
+    try {
+      const ips = restCalls.map(x => x.ip)
+      // Create a Map to count occurrences of each IP address string
+      const countMap = new Map()
+      ips.forEach(ip => {
+        countMap.set(ip, (countMap.get(ip) || 0) + 1)
+      })
+
+      // Convert the Map into an array of objects with `str` and `cnt` properties
+      const result = Array.from(countMap, ([ip, cnt]) => ({ ip, cnt }))
+
+      // Sort the results by the `cnt` property in descending order
+      result.sort((a, b) => b.cnt - a.cnt)
+
+      // Ensure the result has at most 20 elements
+      return result.slice(0, 20)
+    } catch (err) {
+      console.error('Error in usage-use-cases.js/getTopIps()')
+      throw err
+    }
+  }
+
+  // Get the top 20 most consumed endpoints.
+  getTopEndpoints () {
+    try {
+      const endpoints = restCalls.map(x => `${x.method} ${x.url}`)
+
+      // Create a Map to count occurrences of each IP address string
+      const countMap = new Map()
+      endpoints.forEach(endpoint => {
+        countMap.set(endpoint, (countMap.get(endpoint) || 0) + 1)
+      })
+
+      // Convert the Map into an array of objects with `str` and `cnt` properties
+      const result = Array.from(countMap, ([endpoint, cnt]) => ({ endpoint, cnt }))
+
+      // Sort the results by the `cnt` property in descending order
+      result.sort((a, b) => b.cnt - a.cnt)
+
+      // Ensure the result has at most 20 elements
+      return result.slice(0, 20)
+    } catch (err) {
+      console.error('Error in usage-use-cases.js/getTopEndpoints()')
+      throw err
+    }
+  }
+}
+
+// This Koa middleware is called any time there is a REST API. It logs the
+// details from the request object.
+function usageMiddleware () {
+  return async (ctx, next) => {
+    try {
+      await next()
+
+      // console.log('ctx.request: ', ctx.request)
+      const now = new Date()
+
+      const reqObj = {
+        ip: ctx.request.ip,
+        url: ctx.request.url,
+        method: ctx.request.method,
+        timestamp: now.getTime()
+      }
+      // console.log('reqObj: ', reqObj)
+
+      restCalls.push(reqObj)
+    } catch (err) {
+      ctx.status = err.status || 500
+      ctx.body = err.message
+      ctx.app.emit('error', err, ctx)
+    }
+  }
+};
+
+export { UsageUseCases, usageMiddleware, restCalls }