Merge pull request #115 from Permissionless-Software-Foundation/ct-unstable

Improved error handling of hydrateUtxos

config/index.js

@@ -10,13 +10,18 @@ const config = {
     : 'secret-jwt-token',
 
   // Rate Limits
-  anonRateLimit: process.env.ANON_RATE_LIMIT ? Number(process.env.ANON_RATE_LIMIT) : 50,
+  anonRateLimit: process.env.ANON_RATE_LIMIT
+    ? Number(process.env.ANON_RATE_LIMIT)
+    : 500,
   whitelistRateLimit: process.env.WHITELIST_RATE_LIMIT
     ? Number(process.env.WHITELIST_RATE_LIMIT)
     : 10,
+  pointsPerMinute: process.env.POINTS_PER_MINUTE
+    ? Number(process.env.POINTS_PER_MINUTE)
+    : 10000,
   whitelistDomains: process.env.WHITELIST_DOMAINS
     ? process.env.WHITELIST_DOMAINS.split(',')
-    : ['fullstack.cash', 'psfoundation.cash']
+    : ['fullstack.cash', 'psfoundation.cash', '10.0.']
 }
 
 module.exports = config

package.json

@@ -22,14 +22,14 @@
     "coverage": "nyc report --reporter=text-lcov | coveralls",
     "coverage:report": "export NETWORK=mainnet && nyc --reporter=html mocha --timeout 25000 test/v4/",
     "docs": "./node_modules/.bin/apidoc -i src/routes/v4 -o docs",
-    "test:temp1": "export NETWORK=mainnet && export TEST=integration && mocha --exit --timeout 25000 -g '#hydrateUtxosWL' test/v4/integration/",
-    "test:temp2": "mocha test/v4/rate-limits.js"
+    "test:temp1": "export NETWORK=mainnet && export TEST=integration && mocha --exit --timeout 25000 -g '#hydrateUtxos-' test/v4/integration/",
+    "test:temp2": "mocha test/v4/rate-limit2-unit.js"
   },
   "engines": {
     "node": ">=10.15.1"
   },
   "dependencies": {
-    "@psf/bch-js": "^4.15.19",
+    "@psf/bch-js": "^4.16.1",
     "apidoc": "^0.26.0",
     "axios": "^0.21.1",
     "bitcore-lib-cash": "^8.23.1",
@@ -66,6 +66,7 @@
     "eslint-plugin-prettier": "^3.1.0",
     "eslint-plugin-standard": "^4.0.0",
     "fs-extra": "^9.0.0",
+    "lodash.clonedeep": "^4.5.0",
     "nock": "^13.0.5",
     "nyc": "^15.0.0",
     "prettier": "^2.0.0",

src/app.js

@@ -3,7 +3,6 @@
 const express = require('express')
 
 // Middleware
-// const { routeRateLimit } = require("./middleware/route-ratelimit")
 const RateLimits = require('./middleware/route-ratelimit')
 const rateLimits = new RateLimits()
 
@@ -96,17 +95,36 @@ app.use('/', logReqInfo)
 
 const v4prefix = 'v4'
 
-// Inspect the header for a JWT token.
-app.use(`/${v4prefix}/`, jwtAuth.getTokenFromHeaders)
-
-// Instantiate the authorization middleware, used to implement pro-tier rate limiting.
-// Handles Anonymous and Basic Authorization schemes used by passport.js
+// START Rate Limits
 const auth = new AuthMW()
-app.use(`/${v4prefix}/`, auth.mw())
 
-// Rate limit on all v4 routes
-// Establish and enforce rate limits.
-app.use(`/${v4prefix}/`, rateLimits.rateLimitByResource)
+// Ensure req.locals and res.locals objects exist.
+app.use(`/${v4prefix}/`, rateLimits.populateLocals)
+
+// Allow users to turn off rate limits with an environment variable.
+const DO_NOT_USE_RATE_LIMITS = process.env.DO_NOT_USE_RATE_LIMITS || false
+
+console.log(`DO_NOT_USE_RATE_LIMITS: ${DO_NOT_USE_RATE_LIMITS}`)
+
+if (!DO_NOT_USE_RATE_LIMITS) {
+  console.log('Rate limits are being used')
+  // Inspect the header for a JWT token.
+  app.use(`/${v4prefix}/`, jwtAuth.getTokenFromHeaders)
+
+  // Instantiate the authorization middleware, used to implement pro-tier rate limiting.
+  // Handles Anonymous and Basic Authorization schemes used by passport.js
+  app.use(`/${v4prefix}/`, auth.mw())
+
+  // Experimental rate limits
+  app.use(`/${v4prefix}/`, rateLimits.applyRateLimits)
+
+  // Rate limit on all v4 routes
+  // Establish and enforce rate limits.
+  // app.use(`/${v4prefix}/`, rateLimits.rateLimitByResource)
+} else {
+  console.log('Rate limits are NOT being used')
+}
+// END Rate Limits
 
 // Connect v4 routes
 app.use(`/${v4prefix}/` + 'health-check', healthCheckV4)

src/middleware/auth.js

@@ -1,22 +1,25 @@
 /*
-  CT 2/4/20 Note: This library handles anonymous and Basic auth. This library
-  can be phased out with the chage to JWT tokens and the new rate-limit library.
-
-  Handle authorization for bypassing rate limits.
+  This library handles anonymous and Basic Authentication.
 
   1) Default is 'Anonymous Authentication', which unlocks the freemimum tier by
   default.
+
   2) Hard-coded 'Basic Authentication' is a token that does not expire and is
-  provided to buisiness partners.
+  provided for clients who run their own isolated infrastructure without rate
+  limits, but still need a way from preventing the random public from using
+  their API.
+
   3) JWT-based 'Local Authentication' is used for normal users that pay to
   access the premium pro-tier services.
 
   This file uses the passport npm library to check the header of each REST API
-  call for the prescence of a Basic authorization header:
+  call for the prescence of a Basic Authentication header:
   https://en.wikipedia.org/wiki/Basic_access_authentication
 
   If the header is found and validated, the req.locals.proLimit Boolean value
-  is set and passed to the route-ratelimits.ts middleware.
+  is set and passed to the route-ratelimit.js middleware. route-ratelimit.js
+  is for fine-grain JWT-based rate limits. If req.locals.proLimit is set to
+  true, then those rate limits will be skipped.
 */
 
 'use strict'
@@ -76,8 +79,9 @@ class AuthMW {
         req.locals.proLimit = false
 
         // Evaluate the username and password and set the rate limit accordingly.
-        // if (username === "BITBOX" && password === PRO_PASS) {
         if (username === 'fullstackcash') {
+          // Can set several different passwords in the environment variable.
+          // Loop through each one to see if one matches.
           for (let i = 0; i < PRO_PASS.length; i++) {
             const thisPass = PRO_PASS[i]
 

src/middleware/route-ratelimit.js

@@ -1,49 +1,62 @@
 /*
-  Sets the rate limits for the anonymous and paid tiers. Current rate limits:
-  - 1000 points in 60 seconds
-  - 10 points per call for paid tier (100 RPM)
-  - 50 points per call for anonymous tier (20 RPM)
+This file will replace the original rate-limit.js file.
 
-  Background:
-  The rate limits below were originially coded with the idea of charging on a
-  per-resource basis. However, that was confusing to end users trying to purchase
-  a subscription. So everything was simplied to two tiers: paid and anonymous
+Sets the rate limits for the anonymous and paid tiers. Current rate limits:
+- 10000 points in 60 seconds
+- 500 points per call for anonymous tier (20 RPM)
+- 100 points per call for tier 40 (100 RPM)
+- 40 points per call for tier 50 (250 RPM)
+- 16 points per call for tier 60 (625 RPM)
+
+The rate limit handling is designed for these four use cases:
+- Users who want to buy a JWT token for 24 hour access.
+- Users who want to buy different RPM tiers: 100, 250, 600
+- Basic Authentication which should not have any rate limits applied.
+- Local installations that do not want any authentication or rate limits at all.
+
+The Basic Auth use cases is considered when determining internal rate limits.
+The internal rate limits should not be applied to calls from those users.
+
+A lot of attention has been paid to passing rate-limit information for the user
+when they trigger an endpoint that makes a lot of internal API calls. Examples
+are hydrateUtxos() and getPublicKey(). These keeps things fair by charging the
+same for 'light' API calls and 'heavy' API calls.
+
+TODO:
+- Add code for applying rate limits to whitelist domains.
 */
 
-'use strict'
-
 // Public npm libraries.
 const jwt = require('jsonwebtoken')
+const Redis = require('ioredis')
+const { RateLimiterRedis } = require('rate-limiter-flexible')
 
 // local libraries.
 const wlogger = require('../util/winston-logging')
 const config = require('../../config')
 
-const ANON_LIMITS = config.anonRateLimit
-const WHITELIST_RATE_LIMIT = config.whitelistRateLimit
-const WHITELIST_DOMAINS = config.whitelistDomains
-const INTERNAL_RATE_LIMIT = 1
+let _this // Global pointer to instance of class, when 'this' context is lost.
 
-// Redis
+// Setup Redis to track rate limits for each user.
 const redisOptions = {
   enableOfflineQueue: false,
   port: process.env.REDIS_PORT ? process.env.REDIS_PORT : 6379,
   host: process.env.REDIS_HOST ? process.env.REDIS_HOST : '127.0.0.1'
 }
-console.log(`redisOptions: ${JSON.stringify(redisOptions, null, 2)}`)
-
-const Redis = require('ioredis')
 const redisClient = new Redis(redisOptions)
-
-// Rate limiter middleware lib.
-const { RateLimiterRedis } = require('rate-limiter-flexible')
 const rateLimitOptions = {
   storeClient: redisClient,
-  points: 1000, // Number of points
+  points: config.pointsPerMinute, // Number of points
   duration: 60 // Per minute (per 60 seconds)
 }
 
-let _this
+// Constants
+const ANON_LIMITS = config.anonRateLimit
+// const WHITELIST_RATE_LIMIT = config.whitelistRateLimit
+const WHITELIST_DOMAINS = config.whitelistDomains
+const WHITELIST_POINTS_TO_CONSUME = config.whitelistRateLimit
+const POINTS_PER_MINUTE = config.pointsPerMinute
+const INTERNAL_POINTS_TO_CONSUME = 10
 
 class RateLimits {
   constructor () {
@@ -54,248 +67,249 @@ class RateLimits {
     this.config = config
   }
 
-  // Used to disconnect from the Redis DB.
-  // Called by unit tests so that node.js thread doesn't live forever.
-  closeRedis () {
-    redisClient.disconnect()
-  }
-
-  async wipeRedis () {
-    await redisClient.flushdb()
-  }
-
-  // This is the new rate limit function that uses the rate-limiter-flexible npm
-  // library. It uses fine-grain rate limiting based on the resources being
-  // consumed.
-  async rateLimitByResource (req, res, next) {
+  // This is the main middleware funciton of this library. All other functions
+  // support this function.
+  async applyRateLimits (req, res, next) {
     try {
-      let userId
-      let decoded = {}
-
-      // Create a req.locals object if not passed in.
-      if (!req.locals) {
-        req.locals = {
-          // default values
-          jwtToken: '',
-          proLimit: false,
-          apiLevel: 0
-        }
+      // Exit if the user has already authenticated with Basic Authentication.
+      if (req.locals.proLimit) {
+        console.log('External call, basic auth, skipping rate limits.')
+        wlogger.debug(
+          'req.locals.proLimit = true; Using Basic Authentication instead of rate limits'
+        )
+        return next()
       }
 
-      // Create a res.locals object if it does not exist. This is used for
-      // debugging.
-      if (!res.locals) {
-        res.locals = {
-          rateLimitTriggered: false
-        }
-      }
+      // Determine if the call is an external or internal API call.
+      const isInternal = _this.checkInternalIp(req)
+      console.log(`isInternal: ${isInternal}`)
 
-      // Decode the JWT token if one exists.
-      if (req.locals.jwtToken) {
-        try {
-          decoded = _this.jwt.verify(
-            req.locals.jwtToken,
-            _this.config.apiTokenSecret
-          )
-          // console.log(`decoded: ${JSON.stringify(decoded, null, 2)}`)
+      // Determine if the call originates from another computer on the intranet.
+      const isWhitelistOrigin = _this.isInWhitelist(req)
+      console.log('isWhitelistOrigin: ', isWhitelistOrigin)
 
-          userId = decoded.id
-        } catch (err) {
-          // This handler will be triggered if the JWT token does not match the
-          // token secret.
-          wlogger.error(
-            `Last three letters of token secret: ${_this.config.apiTokenSecret.slice(
-              -3
-            )}`
-          )
-          wlogger.error(
-            'Error trying to decode JWT token in route-ratelimit.js/newRateLimit(): ',
-            err
+      // Handle the use case of internally-generated requests.
+      if (isInternal) {
+        // Internal API calls should pass the authentication data in through the
+        // the usrObj in the body.
+        if (req.body && req.body.usrObj) {
+          if (req.body.usrObj.proLimit) {
+            console.log('Internal call, basic auth, skipping rate limits.')
+
+            // If this is an internal call that originated from a user using
+            // Basic Authentication, then skip rate-limits.
+            return next()
+          } else {
+            console.log(
+              'Internal call, applying rate limits. Using JWT if available.'
+            )
+
+            // Determine if user has exceeded their rate limits. Pass in the
+            // JWT token if one exists.
+            const hasExceededRateLimit = await _this.trackRateLimits(
+              req,
+              res,
+              req.body.usrObj.jwtToken
+            )
+
+            if (!hasExceededRateLimit) {
+              // Rate limits have not been exceeded. Processing can continue.
+              return next()
+            } else {
+              // trackRateLimits() returns the 'res' object with an error message
+              // and status code.
+              return hasExceededRateLimit
+            }
+          }
+        } else {
+          // This should be a corner case. Calls should not be going into this
+          // code path, so the system should throw up big warning signs when they
+          // do.
+          // This code path happens when an internal call is made but does not
+          // pass the usrObj. Legacy code needs to be refactored to use the usrObj
+          // and avoid this code path. This code path is 'pooled': all users
+          // share the same rate limits. Even at 1000 RPM, this pool will get
+          // exhausted easily.
+          const warnMsg =
+            'Internal call. req.body.usrObj does not exist. Applying high-speed internal rate limits.'
+          console.log(warnMsg)
+          wlogger.info(warnMsg)
+
+          const defaultPayload = {
+            id: '98.76.54.32',
+            email: 'internal@bchtest.net',
+            apiLevel: 40,
+            rateLimit: 100,
+            pointsToConsume: INTERNAL_POINTS_TO_CONSUME,
+            duration: 30
+          }
+
+          // Default values, in case there is an error.
+          const defaultJwt = _this.generateJwtToken(defaultPayload)
+
+          // Track the rate limit for this user. Pass in the JWT token, if one
+          // is available.
+          const hasExceededRateLimit = await _this.trackRateLimits(
+            req,
+            res,
+            defaultJwt
           )
+
+          if (!hasExceededRateLimit) {
+            // Rate limits have not been exceeded. Processing can continue.
+            return next()
+          } else {
+            // trackRateLimits() returns the 'res' object with an error message
+            // and status code.
+            return hasExceededRateLimit
+          }
         }
         //
-      } else if (req.body && req.body.usrObj) {
-        // Same as above, but this code path is activated from internal calls to
-        // bch-js, like hydrateUtxo(), which passes the user object from the
-        // original API call.
-
-        try {
-          decoded = _this.jwt.verify(
-            req.body.usrObj.jwtToken,
-            _this.config.apiTokenSecret
-          )
-          // console.log(`decoded: ${JSON.stringify(decoded, null, 2)}`)
-
-          userId = decoded.id
-        } catch (err) {
-          // This handler will be triggered if the JWT token does not match the
-          // token secret.
-          wlogger.error(
-            'Error in route-ratelimit.js trying to decode JWT token in usrObj'
-          )
-        }
+        //
       } else {
-        wlogger.debug('No JWT token found!')
-      }
+        // Handle the normal use-case of external requests
+        console.log(
+          'External call, applying rate limits. Using JWT if available.'
+        )
 
-      // Default value is 50 points per request = 20 RPM
-      let rateLimit = ANON_LIMITS
-
-      // Only evaluate the JWT token if the user is not using Basic Authentication.
-      if (!req.locals.proLimit) {
-        // Code here for the rate limiter is adapted from this example:
-        // https://github.com/animir/node-rate-limiter-flexible/wiki/Overall-example#authorized-and-not-authorized-users
-        try {
-          // The resource being consumed: full node, indexer, SLPDB, etc.
-          const resource = _this.getResource(req.url)
-          wlogger.debug(`resource: ${resource}`)
-
-          // Key will be the JWT ID if it exists, otherwise the IP address of the caller.
-          let key = userId || req.ip
-          res.locals.key = key // Feedback for tests.
-          // console.log(`key: ${key}`)
-
-          // const pointsToConsume = userId ? 1 : 30
-          decoded.resource = resource
-          let pointsToConsume = _this.calcPoints(decoded)
-          res.locals.pointsToConsume = pointsToConsume // Feedback for tests.
-
-          // Retrieve the origin.
-          let origin = req.get('origin')
-
-          // Handle calls coming from the intranet.
-          if (origin === undefined && key.indexOf('10.0.0.5') > -1) {
-            origin = 'slp-api'
+        // For calls originating from a whitelist domain, apply a high-RPM
+        // JWT token to the call.
+        if (isWhitelistOrigin) {
+          const defaultPayload = {
+            id: '77.77.77.77',
+            email: 'whitelist@bchtest.net',
+            apiLevel: 40,
+            rateLimit: 100,
+            pointsToConsume: WHITELIST_POINTS_TO_CONSUME,
+            duration: 30
           }
 
-          wlogger.info(`origin: ${origin}`)
+          // Inject the high-RPM JWT token into the call.
+          req.locals.jwtToken = _this.generateJwtToken(defaultPayload)
+        }
 
-          // If the request originates from one of the approved wallet apps, then
-          // apply paid-access rate limits.
-          // console.log(`origin: ${JSON.stringify(origin, null, 2)}`)
-          // console.log(`whitelist: ${JSON.stringify(WHITELIST_DOMAINS, null, 2)}`)
-          const isInWhitelist = _this.isInWhitelist(origin)
-          if (isInWhitelist) {
-            pointsToConsume = WHITELIST_RATE_LIMIT
-            res.locals.pointsToConsume = pointsToConsume // Feedback for tests.
-          }
+        // Track the rate limit for this user. Pass in the JWT token, if one
+        // is available.
+        const hasExceededRateLimit = await _this.trackRateLimits(
+          req,
+          res,
+          req.locals.jwtToken
+        )
 
-          // For internal calls, increase rate limits to as fast as possible.
-          if (
-            // Comment out the line below when running bch-js e2e rate limit tests.
-            key.toString().indexOf('::ffff:127.0.0.1') > -1 ||
-            // Do not comment out this line.
-            key.toString().indexOf('172.17.') > -1
-          ) {
-            pointsToConsume = INTERNAL_RATE_LIMIT
-            res.locals.pointsToConsume = pointsToConsume // Feedback for tests.
-          }
-
-          wlogger.info(
-            `User ${key} consuming ${pointsToConsume} point for resource ${resource}.`
-          )
-
-          rateLimit = Math.floor(1000 / pointsToConsume)
-
-          // Update the key so that rate limits track both the user and the resource.
-          key = `${key}-${resource}`
-
-          await _this.rateLimiter.consume(key, pointsToConsume)
-        } catch (err) {
-          // console.log('err: ', err)
-
-          // Used for returning data for tests.
-          res.locals.rateLimitTriggered = true
-          // console.log('res.locals: ', res.locals)
-
-          // Rate limited was triggered
-          res.status(429) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
-          return res.json({
-            error: `Too many requests. Your limits are currently ${rateLimit} requests per minute. Increase rate limits at https://fullstack.cash`
-          })
+        if (!hasExceededRateLimit) {
+          // Rate limits have not been exceeded. Processing can continue.
+          return next()
+        } else {
+          // trackRateLimits() returns the 'res' object with an error message
+          // and status code.
+          return hasExceededRateLimit
         }
       }
     } catch (err) {
-      wlogger.error('Error in route-ratelimit.js/newRateLimit(): ', err)
-      // throw err
+      wlogger.error('Error in route-ratelimit2.js/applyRateLimits(): ', err)
     }
 
+    // By default, move to the next middleware.
     next()
   }
 
-  // Calculates the points consumed, based on the jwt information and the route
-  // requested.
-  calcPoints (jwtInfo) {
-    let retVal = ANON_LIMITS // By default, use anonymous tier.
+  // A wrapper for Redis-based rate limiter.
+  // Will return false if the user has not exceeded the rate limit. Otherwise
+  // it will return the 'res' object with an error status and message, which
+  // should be returned by the middleware.
+  async trackRateLimits (req, res, jwtToken) {
+    // Anonymous rate limits are used by default.
+    let pointsToConsume = ANON_LIMITS
+    let key = req.ip // Use the IP address as the key, by default.
 
     try {
-      // console.log(`jwtInfo: ${JSON.stringify(jwtInfo, null, 2)}`)
+      // Decode the JWT token if it exists
+      if (jwtToken) {
+        const decoded = _this.decodeJwtToken(jwtToken)
+        // console.log(`decoded: ${JSON.stringify(decoded, null, 2)}`)
 
-      const apiLevel = jwtInfo.apiLevel
-      const resource = jwtInfo.resource
+        // Preferentially use the decoded ID in the JWT payload, as the key.
+        key = decoded.id
 
-      const level30Routes = ['insight', 'bitcore', 'blockbook', 'electrumx']
-      const level40Routes = ['slp']
-
-      wlogger.debug(`apiLevel: ${apiLevel}`)
-
-      // Only evaluate if user is using a JWT token.
-      if (jwtInfo.id) {
-        // SLP indexer routes
-        if (level40Routes.includes(resource)) {
-          if (apiLevel >= 40) retVal = 10
-          // else if (apiLevel >= 10) retVal = 10
-          else retVal = ANON_LIMITS
-
-          // Normal indexer routes
-        } else if (level30Routes.includes(resource)) {
-          if (apiLevel >= 30) retVal = 10
-          else retVal = ANON_LIMITS
-
-          // Full node tier
-        } else if (apiLevel >= 20) {
-          retVal = 10
-
-          // Free tier, full node only.
-        } else {
-          retVal = ANON_LIMITS
-        }
+        pointsToConsume = decoded.pointsToConsume
       }
+      console.log(`rate limit key: ${key}`)
 
-      return retVal
+      // This function will throw an error if the user exceeds the rate limit.
+      // The 429 error response is handled by the catch().
+      await _this.rateLimiter.consume(key, pointsToConsume)
+
+      res.locals.pointsToConsume = pointsToConsume // Feedback for tests.
+
+      // Signal that the user has not exceeded their rate limits.
+      return false
     } catch (err) {
-      wlogger.error('Error in route-ratelimit.js/calcPoints()')
-      // throw err
-      retVal = ANON_LIMITS
-    }
+      console.log('err: ', err)
 
-    return retVal
+      const rateLimit = Math.floor(POINTS_PER_MINUTE / pointsToConsume)
+
+      res.locals.rateLimitTriggered = true
+      // console.log('res.locals: ', res.locals)
+
+      // Rate limited was triggered
+      res.status(429) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
+      return res.json({
+        error: `Too many requests. Your limits are currently ${rateLimit} requests per minute. Increase rate limits at https://fullstack.cash`
+      })
+    }
   }
 
-  // This function parses the req.url property to identify what resource
-  // the user is requesting.
-  // This was created as a function so that it can be unit tested. Not sure
-  // what kind of variations will be seen in production.
-  getResource (url) {
+  // Attempts to decode a JWT token. Returns default values if it fails.
+  decodeJwtToken (jwtToken) {
+    const defaultPayload = {
+      id: '123.456.789.10',
+      email: 'test@bchtest.net',
+      apiLevel: 10,
+      rateLimit: 3,
+      pointsToConsume: ANON_LIMITS,
+      duration: 30
+    }
+
     try {
-      wlogger.debug(`url: ${JSON.stringify(url, null, 2)}`)
+      // Default values, in case there is an error.
+      const defaultJwt = _this.generateJwtToken(defaultPayload)
 
-      const splitUrl = url.split('/')
-      const resource = splitUrl[1]
+      // Generate a default payload to use, if the decoding of the user-provided
+      // jwt fails.
+      let decoded = _this.jwt.verify(defaultJwt, _this.config.apiTokenSecret)
 
-      return resource
+      try {
+        decoded = _this.jwt.verify(jwtToken, _this.config.apiTokenSecret)
+      } catch (err) {
+        wlogger.error('Error in route-ratelimit2.js/decodeJwtTokens(): ', err)
+      }
+
+      return decoded
     } catch (err) {
-      wlogger.error('Error in getResource().')
-      throw err
+      wlogger.error(
+        'Unhandled error in route-ratelimit2.js/deocdeJwtToken: ',
+        err
+      )
+
+      // Making sure there is an exp property. Not sure if this will cause an
+      // issue, using a hard-coded value.
+      defaultPayload.exp = 1574269450
+
+      return defaultPayload
     }
   }
 
   // Returns a boolean if the origin of the request matches a domain in the
   // whitelist.
-  isInWhitelist (origin) {
+  isInWhitelist (req) {
     try {
       const retVal = false // Default value.
 
+      // Retrieve the origin.
+      const origin = req.get('origin')
+      console.log(`origin: ${origin}`)
+
+      // If the origin is not determinable, return false.
       if (!origin) return false
 
       // console.log(`WHITELIST_DOMAINS: ${JSON.stringify(WHITELIST_DOMAINS, null, 2)}`)
@@ -303,9 +317,7 @@ class RateLimits {
       for (let i = 0; i < WHITELIST_DOMAINS.length; i++) {
         const thisDomain = WHITELIST_DOMAINS[i]
 
-        if (origin.toString().indexOf(thisDomain) > -1) {
-          return true
-        }
+        if (origin.includes(thisDomain)) return true
       }
 
       return retVal
@@ -316,6 +328,93 @@ class RateLimits {
       return false
     }
   }
+
+  // Checks the request object to see if it's IP address matches an internal
+  // IP address. That means the call is an internal API call and should be
+  // treated differently than an external API call.
+  checkInternalIp (req) {
+    try {
+      // Default value
+      let isInternal = false
+
+      const ip = req.ip
+
+      if (ip.includes('127.0.0.1')) isInternal = true
+
+      if (ip.includes('172.17.')) isInternal = true
+
+      // TODO: Add 192.168.
+
+      return isInternal
+    } catch (err) {
+      wlogger.error(
+        'Error in checkInternalIp(). Returning false be default. Err: ',
+        err
+      )
+      return false
+    }
+  }
+
+  // Used to disconnect from the Redis DB.
+  // Called by unit tests so that node.js thread doesn't live forever.
+  closeRedis () {
+    redisClient.disconnect()
+  }
+
+  // Clear the redis database. Used by unit tests.
+  async wipeRedis () {
+    await redisClient.flushdb()
+  }
+
+  // Generates a JWT token for testing purposes. This is not used in production.
+  // This function mirrors the kind of JWT token that would be generated by
+  // jwt-bch-api.
+  generateJwtToken (payload) {
+    try {
+      const jwtOptions = {
+        expiresIn: '30 days'
+      }
+
+      const token = _this.jwt.sign(
+        payload,
+        _this.config.apiTokenSecret,
+        jwtOptions
+      )
+
+      return token
+    } catch (err) {
+      console.error('Error in generateJwtToken()')
+      throw err
+    }
+  }
+
+  // Called when rate limits are not used.
+  populateLocals (req, res, next) {
+    try {
+      // Create a re*Q*.locals object if not passed in.
+      // req.locals.proLimit will be true if the user is using Basic Authentication.
+      if (!req.locals) {
+        req.locals = {
+          // default values
+          jwtToken: '',
+          proLimit: false,
+          apiLevel: 0
+        }
+      }
+
+      // Create a re*S*.locals object if it does not exist.
+      if (!res.locals) {
+        res.locals = {
+          rateLimitTriggered: false
+        }
+      }
+
+      next()
+    } catch (err) {
+      console.error('Error in populateLocals(): ', err)
+      throw err
+    }
+  }
 }
 
 module.exports = RateLimits

src/routes/v4/blockbook.js

@@ -169,7 +169,7 @@ class Blockbook {
 
       // Enforce array size rate limits
       if (!_this.routeUtils.validateArraySize(req, addresses)) {
-        res.status(429) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
+        res.status(400) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
         return res.json({
           error: 'Array too large.'
         })
@@ -333,7 +333,7 @@ class Blockbook {
 
       // Enforce array size rate limits
       if (!_this.routeUtils.validateArraySize(req, addresses)) {
-        res.status(429) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
+        res.status(400) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
         return res.json({
           error: 'Array too large.'
         })
@@ -472,7 +472,7 @@ class Blockbook {
 
       // Enforce array size rate limits
       if (!_this.routeUtils.validateArraySize(req, txids)) {
-        res.status(429) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
+        res.status(400) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
         return res.json({
           error: 'Array too large.'
         })

src/routes/v4/electrumx.js

@@ -284,7 +284,7 @@ class Electrum {
 
       // Enforce array size rate limits
       if (!_this.routeUtils.validateArraySize(req, addresses)) {
-        res.status(429) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
+        res.status(400) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
         return res.json({
           error: 'Array too large.'
         })
@@ -470,7 +470,7 @@ class Electrum {
 
       // Enforce array size rate limits
       if (!_this.routeUtils.validateArraySize(req, txids)) {
-        res.status(429) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
+        res.status(400) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
         return res.json({
           success: false,
           error: 'Array too large.'
@@ -726,7 +726,7 @@ class Electrum {
 
       // Enforce array size rate limits
       if (!_this.routeUtils.validateArraySize(req, heights)) {
-        res.status(429) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
+        res.status(400) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
         return res.json({
           success: false,
           error: 'Array too large.'
@@ -895,7 +895,7 @@ class Electrum {
 
       // Enforce array size rate limits
       if (!_this.routeUtils.validateArraySize(req, addresses)) {
-        res.status(429) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
+        res.status(400) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
         return res.json({
           error: 'Array too large.'
         })
@@ -1088,7 +1088,7 @@ class Electrum {
 
       // Enforce array size rate limits
       if (!_this.routeUtils.validateArraySize(req, addresses)) {
-        res.status(429) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
+        res.status(400) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
         return res.json({
           error: 'Array too large.'
         })
@@ -1281,7 +1281,7 @@ class Electrum {
 
       // Enforce array size rate limits
       if (!_this.routeUtils.validateArraySize(req, addresses)) {
-        res.status(429) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
+        res.status(400) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
         return res.json({
           error: 'Array too large.'
         })

src/routes/v4/encryption.js

@@ -114,16 +114,17 @@ class Encryption {
         })
       }
 
+      // console.log(
       wlogger.debug(
         'Executing encryption/getPublicKey with this address: ',
         cashAddr
       )
 
-      const rawTxData = await _this.bchjs.Electrumx.transactions(cashAddr, usrObj)
+      const rawTxData = await _this.bchjs.Electrumx.transactions([cashAddr], usrObj)
       // console.log(`rawTxData: ${JSON.stringify(rawTxData, null, 2)}`)
 
       // Extract just the TXIDs
-      const txids = rawTxData.transactions.map((elem) => elem.tx_hash)
+      const txids = rawTxData.transactions[0].transactions.map((elem) => elem.tx_hash)
       // console.log(`txids: ${JSON.stringify(txids, null, 2)}`)
 
       // throw error if there is no transaction history.
@@ -135,16 +136,14 @@ class Encryption {
       for (let i = 0; i < txids.length; i++) {
         const thisTx = txids[i]
 
-        // CT 2/24/21: I might want to convert this to the POST call, to take
-        // advantage of the usrObj. It does not get passed in a GET call.
         const txDetails = await _this.bchjs.RawTransactions.getRawTransaction(
-          thisTx,
+          [thisTx],
           true,
           usrObj
         )
         // console.log(`txDetails: ${JSON.stringify(txDetails, null, 2)}`)
 
-        const vin = txDetails.vin
+        const vin = txDetails[0].vin
 
         // Loop through each input.
         for (let j = 0; j < vin.length; j++) {
@@ -182,6 +181,7 @@ class Encryption {
         publicKey: 'not found'
       })
     } catch (err) {
+      console.log('Error in encryption.js/getPublicKey().', err)
       wlogger.error('Error in encryption.js/getPublicKey().', err)
 
       return _this.errorHandler(err, res)

src/routes/v4/full-node/blockchain.js

@@ -284,7 +284,7 @@ class Blockchain {
 
       // Enforce array size rate limits
       if (!routeUtils.validateArraySize(req, hashes)) {
-        res.status(429) // https://github.com/Bitcoin-com/api.fullstack.cash/issues/330
+        res.status(400) // https://github.com/Bitcoin-com/api.fullstack.cash/issues/330
         return res.json({
           error: 'Array too large.'
         })
@@ -457,7 +457,7 @@ class Blockchain {
 
       // Enforce array size rate limits
       if (!routeUtils.validateArraySize(req, txids)) {
-        res.status(429) // https://github.com/Bitcoin-com/api.fullstack.cash/issues/330
+        res.status(400) // https://github.com/Bitcoin-com/api.fullstack.cash/issues/330
         return res.json({
           error: 'Array too large.'
         })
@@ -794,7 +794,7 @@ class Blockchain {
 
       // Enforce array size rate limits
       if (!routeUtils.validateArraySize(req, txids)) {
-        res.status(429) // https://github.com/Bitcoin-com/api.fullstack.cash/issues/330
+        res.status(400) // https://github.com/Bitcoin-com/api.fullstack.cash/issues/330
         return res.json({
           error: 'Array too large.'
         })
@@ -888,7 +888,7 @@ class Blockchain {
 
       // Enforce array size rate limits
       if (!routeUtils.validateArraySize(req, proofs)) {
-        res.status(429) // https://github.com/Bitcoin-com/api.fullstack.cash/issues/330
+        res.status(400) // https://github.com/Bitcoin-com/api.fullstack.cash/issues/330
         return res.json({
           error: 'Array too large.'
         })

src/routes/v4/full-node/rawtransactions.js

@@ -118,7 +118,7 @@ class RawTransactions {
 
       // Enforce array size rate limits
       if (!_this.routeUtils.validateArraySize(req, hexes)) {
-        res.status(429) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
+        res.status(400) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
         return res.json({
           error: 'Array too large.'
         })
@@ -234,7 +234,7 @@ class RawTransactions {
 
       // Enforce array size rate limits
       if (!_this.routeUtils.validateArraySize(req, hexes)) {
-        res.status(429) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
+        res.status(400) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
         return res.json({
           error: 'Array too large.'
         })
@@ -324,20 +324,12 @@ class RawTransactions {
 
       // Enforce array size rate limits
       if (!_this.routeUtils.validateArraySize(req, txids)) {
-        res.status(429) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
+        res.status(400) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
         return res.json({
           error: 'Array too large.'
         })
       }
 
-      // stub response object
-      // const returnResponse = {
-      //   status: 100,
-      //   json: {
-      //     error: ''
-      //   }
-      // }
-
       // Validate each txid in the array.
       for (let i = 0; i < txids.length; i++) {
         const txid = txids[i]
@@ -448,7 +440,7 @@ class RawTransactions {
 
       // Enforce array size rate limits
       if (!_this.routeUtils.validateArraySize(req, hexes)) {
-        res.status(429) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
+        res.status(400) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
         return res.json({
           error: 'Array too large.'
         })

src/routes/v4/price.js

@@ -27,11 +27,15 @@ class Price {
     this.coinexPriceUrl =
       'https://api.coinex.com/v1/market/ticker?market=bchausdt'
 
+    this.bchCoinexPriceUrl =
+      'https://api.coinex.com/v1/market/ticker?market=bchusdt'
+
     this.router = express.Router()
     this.router.get('/', _this.root)
     this.router.get('/usd', _this.getUSD)
     this.router.get('/rates', _this.getBCHRate)
     this.router.get('/bchausd', _this.getBCHAUSD)
+    this.router.get('/bchusd', _this.getBCHUSD)
   }
 
   // DRY error handler.
@@ -150,6 +154,40 @@ class Price {
       return _this.errorHandler(err, res)
     }
   }
+
+  /**
+   * @api {get} /price/bchusd Get the USD price of BCH
+   * @apiName Get the USD price of BCH
+   * @apiGroup Price
+   * @apiDescription Get the USD price of BCH from Coinex.
+   *
+   *
+   * @apiExample Example usage:
+   * curl -X GET "https://api.fullstack.cash/v4/price/bchusd" -H "accept: application/json"
+   *
+   */
+  async getBCHUSD (req, res, next) {
+    try {
+      // Request options
+      const opt = {
+        method: 'get',
+        baseURL: _this.bchCoinexPriceUrl,
+        timeout: 15000
+      }
+
+      const response = await axios.request(opt)
+      // console.log(`response.data: ${JSON.stringify(response.data, null, 2)}`)
+
+      const price = Number(response.data.data.ticker.last)
+
+      return res.json({ usd: price })
+    } catch (err) {
+      // Write out error to error log.
+      wlogger.error('Error in price.js/getBCHUSD().', err)
+
+      return _this.errorHandler(err, res)
+    }
+  }
 }
 
 module.exports = Price

src/routes/v4/route-utils.js

@@ -1,192 +0,0 @@
-/*
-  A private library of utility functions used by several different routes.
-*/
-
-'use strict'
-
-const axios = require('axios')
-const wlogger = require('../../util/winston-logging')
-
-const util = require('util')
-util.inspect.defaultOptions = { depth: 1 }
-
-const BCHJS = require('@psf/bch-js')
-const bchjs = new BCHJS()
-
-module.exports = {
-  validateNetwork, // Prevents a common user error
-  setEnvVars, // Allows RPC variables to be set dynamically based on changing env vars.
-  decodeError, // Extract and interpret error messages.
-  validateArraySize, // Ensure the passed array meets rate limiting requirements.
-  getAxiosOptions
-}
-
-// This function expects the Request Express.js object and an array as input.
-// The array is then validated against freemium and pro-tier rate limiting
-// requirements. A boolean is returned to indicate if the array size if valid
-// or not.
-function validateArraySize (req, array) {
-  const FREEMIUM_INPUT_SIZE = 20
-  const PRO_INPUT_SIZE = 20
-
-  if (req.locals && req.locals.proLimit) {
-    if (array.length <= PRO_INPUT_SIZE) return true
-  } else if (array.length <= FREEMIUM_INPUT_SIZE) {
-    return true
-  }
-
-  return false
-}
-
-// Returns true if user-provided cash address matches the correct network,
-// mainnet or testnet. If NETWORK env var is not defined, it returns false.
-// This prevent a common user-error issue that is easy to make: passing a
-// testnet address into rest.bitcoin.com or passing a mainnet address into
-// trest.bitcoin.com.
-function validateNetwork (addr) {
-  try {
-    const network = process.env.NETWORK
-
-    // Return false if NETWORK is not defined.
-    if (!network || network === '') {
-      console.log('Warning: NETWORK environment variable is not defined!')
-      return false
-    }
-
-    // Convert the user-provided address to a cashaddress, for easy detection
-    // of the intended network.
-    const cashAddr = bchjs.Address.toCashAddress(addr)
-
-    // Return true if the network and address both match testnet
-    const addrIsTest = bchjs.Address.isTestnetAddress(cashAddr)
-    if (network === 'testnet' && addrIsTest) return true
-
-    // Return true if the network and address both match mainnet
-    const addrIsMain = bchjs.Address.isMainnetAddress(cashAddr)
-    if (network === 'mainnet' && addrIsMain) return true
-
-    return false
-  } catch (err) {
-    wlogger.error('Error in validateNetwork()')
-    return false
-  }
-}
-
-// Dynamically set these based on env vars. Allows unit testing.
-function setEnvVars () {
-  const BitboxHTTP = axios.create({
-    baseURL: process.env.RPC_BASEURL,
-    timeout: 15000
-  })
-  const username = process.env.RPC_USERNAME
-  const password = process.env.RPC_PASSWORD
-
-  const requestConfig = {
-    method: 'post',
-    auth: {
-      username: username,
-      password: password
-    },
-    data: {
-      jsonrpc: '1.0'
-    }
-  }
-
-  return { BitboxHTTP, username, password, requestConfig }
-}
-
-// Axios options used when calling axios.post() to talk with a full node.
-function getAxiosOptions () {
-  return {
-    method: 'post',
-    baseURL: process.env.RPC_BASEURL,
-    timeout: 15000,
-    auth: {
-      username: process.env.RPC_USERNAME,
-      password: process.env.RPC_PASSWORD
-    },
-    data: {
-      jsonrpc: '1.0'
-    }
-  }
-}
-
-// Error messages returned by a full node can be burried pretty deep inside the
-// error object returned by Axios. This function attempts to extract and interpret
-// error messages.
-// Returns an object. If successful, obj.msg is a string.
-// If there is a failure, obj.msg is false.
-function decodeError (err) {
-  try {
-    // Attempt to extract the full node error message.
-    if (
-      err.response &&
-      err.response.data &&
-      err.response.data.error &&
-      err.response.data.error.message
-    ) {
-      return { msg: err.response.data.error.message, status: 400 }
-    }
-
-    // Attempt to extract the Insight error message
-    if (err.response && err.response.data) {
-      return { msg: err.response.data, status: err.response.status }
-    }
-
-    // console.log(`err.message: ${err.message}`)
-    // console.log(`err: `, err)
-
-    // Attempt to detect a network connection error.
-    if (err.message && err.message.indexOf('ENOTFOUND') > -1) {
-      return {
-        msg:
-          'Network error: Could not communicate with full node or other external service.',
-        status: 503
-      }
-    }
-
-    // Different kind of network error
-    if (err.message && err.message.indexOf('ENETUNREACH') > -1) {
-      return {
-        msg:
-          'Network error: Could not communicate with full node or other external service.',
-        status: 503
-      }
-    }
-
-    // Different kind of network error
-    if (err.message && err.message.indexOf('EAI_AGAIN') > -1) {
-      return {
-        msg:
-          'Network error: Could not communicate with full node or other external service.',
-        status: 503
-      }
-    }
-
-    // Axios timeout (aborted) error, or service is down (connection refused).
-    if (
-      err.code &&
-      (err.code === 'ECONNABORTED' || err.code === 'ECONNREFUSED')
-    ) {
-      return {
-        msg:
-          'Network error: Could not communicate with full node or other external service.',
-        status: 503
-      }
-    }
-
-    // Handle general Error objects.
-    if (err.message) {
-      return {
-        message: err.message,
-        status: 422
-      }
-    }
-
-    return { msg: false, status: 500 }
-  } catch (err) {
-    console.error('unhandled error in route-utils.js/decodeError(): ', err)
-    wlogger.error('unhandled error in route-utils.js/decodeError(): ', err)
-    return { msg: false, status: 500 }
-  }
-}

src/routes/v4/slp.js

@@ -36,9 +36,13 @@ util.inspect.defaultOptions = { depth: 5 }
 // Determine the Access password for a private instance of SLPDB.
 // https://gist.github.com/christroutner/fc717ca704dec3dded8b52fae387eab2
 // Password for General Purpose (GP) SLPDB.
-const SLPDB_PASS_GP = process.env.SLPDB_PASS_GP ? process.env.SLPDB_PASS_GP : 'BITBOX'
+const SLPDB_PASS_GP = process.env.SLPDB_PASS_GP
+  ? process.env.SLPDB_PASS_GP
+  : 'BITBOX'
 // Password for Whitelist (WL) SLPDB.
-const SLPDB_PASS_WL = process.env.SLPDB_PASS_WL ? process.env.SLPDB_PASS_WL : 'BITBOX'
+const SLPDB_PASS_WL = process.env.SLPDB_PASS_WL
+  ? process.env.SLPDB_PASS_WL
+  : 'BITBOX'
 
 // const rawtransactions = require('./full-node/rawtransactions')
 const RawTransactions = require('./full-node/rawtransactions')
@@ -46,7 +50,7 @@ const rawTransactions = new RawTransactions()
 
 // Setup REST and TREST URLs used by slpjs
 // Dev note: this allows for unit tests to mock the URL.
-if (!process.env.REST_URL) process.env.REST_URL = 'https://bchn.fullstack.cash/v4/'
+if (!process.env.REST_URL) { process.env.REST_URL = 'https://bchn.fullstack.cash/v4/' }
 if (!process.env.TREST_URL) {
   process.env.TREST_URL = 'https://testnet.fullstack.cash/v4/'
 }
@@ -100,8 +104,13 @@ class Slp {
 
   // DRY error handler.
   errorHandler (err, res) {
+    // console.error('Entering slp.js/errorHandler(). err: ', err)
+
     // Attempt to decode the error message.
     const { msg, status } = _this.routeUtils.decodeError(err)
+    console.log('slp.js/errorHandler msg from decodeError: ', msg)
+    console.log('slp.js/errorHandler status from decodeError: ', status)
+
     if (msg) {
       res.status(status)
       return res.json({ error: msg })
@@ -215,7 +224,7 @@ class Slp {
 
       // Enforce array size rate limits
       if (!_this.routeUtils.validateArraySize(req, tokenIds)) {
-        res.status(429) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
+        res.status(400) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
         return res.json({
           error: 'Array too large.'
         })
@@ -545,7 +554,7 @@ class Slp {
 
       // Enforce array size rate limits
       if (!_this.routeUtils.validateArraySize(req, addresses)) {
-        res.status(429) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
+        res.status(400) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
         return res.json({
           error: 'Array too large.'
         })
@@ -914,7 +923,7 @@ class Slp {
 
     // Enforce array size rate limits
     if (!_this.routeUtils.validateArraySize(req, addresses)) {
-      res.status(429) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
+      res.status(400) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
       return res.json({
         error: 'Array too large.'
       })
@@ -975,7 +984,7 @@ class Slp {
 
       // Enforce array size rate limits
       if (!_this.routeUtils.validateArraySize(req, txids)) {
-        res.status(429) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
+        res.status(400) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
         return res.json({
           error: 'Array too large.'
         })
@@ -1403,7 +1412,7 @@ class Slp {
 
       // Enforce array size rate limits
       if (!_this.routeUtils.validateArraySize(req, txids)) {
-        res.status(429) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
+        res.status(400) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
         return res.json({
           error: 'Array too large.'
         })
@@ -1979,6 +1988,13 @@ class Slp {
     try {
       const utxos = req.body.utxos
 
+      // Extract a delay value if the user passed it in.
+      const usrObjIn = req.body.usrObj
+      let utxoDelay = 0
+      if (usrObjIn && usrObjIn.utxoDelay) {
+        utxoDelay = usrObjIn.utxoDelay
+      }
+
       // console.log('req: ', req)
       // console.log(`req._remoteAddress: ${req._remoteAddress}`)
 
@@ -1988,7 +2004,8 @@ class Slp {
         ip: req._remoteAddress,
         jwtToken: req.locals.jwtToken,
         proLimit: req.locals.proLimit,
-        apiLevel: req.locals.apiLevel
+        apiLevel: req.locals.apiLevel,
+        utxoDelay
       }
 
       // Validate inputs
@@ -2025,7 +2042,10 @@ class Slp {
         const theseUtxos = utxos[i].utxos
 
         // Get SLP token details.
-        const details = await _this.bchjs.SLP.Utils.tokenUtxoDetails(theseUtxos, usrObj)
+        const details = await _this.bchjs.SLP.Utils.tokenUtxoDetails(
+          theseUtxos,
+          usrObj
+        )
         // console.log('details: ', details)
 
         // Replace the original UTXO data with the hydrated data.
@@ -2036,12 +2056,13 @@ class Slp {
       return res.json({ slpUtxos: utxos })
     } catch (err) {
       wlogger.error('Error in slp.js/hydrateUtxos().', err)
-      console.error('Error in slp.js/hydrateUtxos().', err)
+      // console.error('Error in slp.js/hydrateUtxos().', err)
 
       // Decode the error message.
       const { msg, status } = routeUtils.decodeError(err)
-      console.log('msg: ', msg)
-      console.log('status: ', status)
+      // console.log('msg: ', msg)
+      // console.log('status: ', status)
+
       if (msg) {
         res.status(status)
         return res.json({ error: msg, message: msg, success: false })
@@ -2049,7 +2070,8 @@ class Slp {
 
       res.status(500)
       return res.json({
-        error: 'Error in hydrateUtxos()'
+        error: 'Undetermined error in hydrateUtxos()',
+        message: err.message
       })
     }
   }
@@ -2075,6 +2097,23 @@ class Slp {
     try {
       const utxos = req.body.utxos
 
+      // Extract a delay value if the user passed it in.
+      const usrObjIn = req.body.usrObj
+      let utxoDelay = 0
+      if (usrObjIn && usrObjIn.utxoDelay) {
+        utxoDelay = usrObjIn.utxoDelay
+      }
+
+      // Generate a user object that can be passed along with internal calls
+      // from bch-js.
+      const usrObj = {
+        ip: req._remoteAddress,
+        jwtToken: req.locals.jwtToken,
+        proLimit: req.locals.proLimit,
+        apiLevel: req.locals.apiLevel,
+        utxoDelay
+      }
+
       // Validate inputs
       if (!Array.isArray(utxos)) {
         res.status(422)
@@ -2110,7 +2149,10 @@ class Slp {
         // console.log(`theseUtxos: ${JSON.stringify(theseUtxos, null, 2)}`)
 
         // Get SLP token details.
-        const details = await _this.bchjs.SLP.Utils.tokenUtxoDetailsWL(theseUtxos)
+        const details = await _this.bchjs.SLP.Utils.tokenUtxoDetailsWL(
+          theseUtxos,
+          usrObj
+        )
         // console.log('details : ', details)
 
         // Replace the original UTXO data with the hydrated data.

src/routes/v4/util.js

@@ -4,7 +4,9 @@ const express = require('express')
 const router = express.Router()
 const axios = require('axios')
 
-const routeUtils = require('./route-utils')
+const RouteUtils = require('../../util/route-utils')
+const routeUtils = new RouteUtils()
+
 const wlogger = require('../../util/winston-logging')
 
 const util = require('util')
@@ -141,7 +143,7 @@ class UtilRoute {
 
       // Enforce array size rate limits
       if (!routeUtils.validateArraySize(req, addresses)) {
-        res.status(429) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
+        res.status(400) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
         return res.json({
           error: 'Array too large.'
         })

src/routes/v4/xpub.js

@@ -5,8 +5,10 @@
 'use strict'
 
 const express = require('express')
-// const axios = require('axios')
-const routeUtils = require('./route-utils')
+
+const RouteUtils = require('../../util/route-utils')
+const routeUtils = new RouteUtils()
+
 const wlogger = require('../../util/winston-logging')
 
 // const router = express.Router()

src/util/route-utils.js

@@ -154,6 +154,23 @@ class RouteUtils {
         }
       }
 
+      // Handle 429 errors thrown by nginx
+      if (err.error) {
+        // console.log('decodeError: err: ', err)
+
+        if (err.error.includes('429 Too Many Requests')) {
+          const internalMsg =
+            '429 error thrown by nginx caught by route-utils.js/decodeError()'
+          console.error(internalMsg)
+          wlogger.error(internalMsg)
+
+          return {
+            msg: '429 Too Many Requests',
+            status: 429
+          }
+        }
+      }
+
       // Handle general Error objects.
       if (err.message) {
         return {
@@ -169,6 +186,29 @@ class RouteUtils {
       return { msg: false, status: 500 }
     }
   }
+
+  // Dynamically set these based on env vars. Allows unit testing.
+  setEnvVars () {
+    const BitboxHTTP = axios.create({
+      baseURL: process.env.RPC_BASEURL,
+      timeout: 15000
+    })
+    const username = process.env.RPC_USERNAME
+    const password = process.env.RPC_PASSWORD
+
+    const requestConfig = {
+      method: 'post',
+      auth: {
+        username: username,
+        password: password
+      },
+      data: {
+        jsonrpc: '1.0'
+      }
+    }
+
+    return { BitboxHTTP, username, password, requestConfig }
+  }
 }
 
 module.exports = RouteUtils

test/v4/a01-electrumx.js

@@ -594,7 +594,7 @@ describe('#Electrumx', () => {
       assert.isArray(result.transactions)
     })
 
-    it('should throw 429 error if txid array is too large', async () => {
+    it('should throw 400 error if txid array is too large', async () => {
       const testArray = []
       for (var i = 0; i < 25; i++) testArray.push('')
 
@@ -603,7 +603,7 @@ describe('#Electrumx', () => {
       const result = await electrumxRoute.transactionDetailsBulk(req, res)
       // console.log(`result: ${util.inspect(result)}`)
 
-      expectRouteError(res, result, 'Array too large', 429)
+      expectRouteError(res, result, 'Array too large', 400)
     })
 
     it('should get details for a single txid', async () => {
@@ -841,7 +841,7 @@ describe('#Electrumx', () => {
       assert.isArray(result.headers)
     })
 
-    it('should throw 429 error if heights array is too large', async () => {
+    it('should throw 400 error if heights array is too large', async () => {
       const testArray = []
       for (var i = 0; i < 25; i++) testArray.push('')
 
@@ -849,7 +849,7 @@ describe('#Electrumx', () => {
 
       const result = await electrumxRoute.blockHeadersBulk(req, res)
 
-      expectRouteError(res, result, 'Array too large', 429)
+      expectRouteError(res, result, 'Array too large', 400)
     })
 
     it('should get details for a single height', async () => {

test/v4/encryption.js

@@ -82,7 +82,7 @@ describe('#Encryption Router', () => {
           .resolves(mockData.mockFulcrumTxHistory)
         sandbox
           .stub(encryptionRoute.bchjs.RawTransactions, 'getRawTransaction')
-          .resolves(mockData.mockTxDetails2)
+          .resolves([mockData.mockTxDetails2])
       }
 
       const result = await encryptionRoute.getPublicKey(req, res)
@@ -110,7 +110,7 @@ describe('#Encryption Router', () => {
       }
 
       const result = await encryptionRoute.getPublicKey(req, res)
-      // console.log(`result: ${JSON.stringify(result, null, 2)}`)
+      console.log(`result: ${JSON.stringify(result, null, 2)}`)
 
       assert.property(result, 'success')
       assert.equal(result.success, false)
@@ -130,7 +130,7 @@ describe('#Encryption Router', () => {
           .resolves(mockData.mockFulcrumNoSendBalance)
         sandbox
           .stub(encryptionRoute.bchjs.RawTransactions, 'getRawTransaction')
-          .resolves(mockData.mockNoSendTx)
+          .resolves([mockData.mockNoSendTx])
       }
 
       const result = await encryptionRoute.getPublicKey(req, res)

test/v4/integration/price.js

@@ -39,6 +39,14 @@ describe('#price', () => {
       const result = await price.getBCHAUSD(req, res)
       console.log(`result: ${util.inspect(result)}`)
 
+      assert.isNumber(result.usd)
+    })
+  })
+  describe('#getBCHUSD', () => {
+    it('should get the USD price of BCH', async () => {
+      const result = await price.getBCHUSD(req, res)
+      console.log(`result: ${util.inspect(result)}`)
+
       assert.isNumber(result.usd)
     })
   })

test/v4/mocks/encryption-mocks.js

@@ -55,64 +55,68 @@ const mockFulcrumTxHistory = {
   success: true,
   transactions: [
     {
-      height: 511463,
-      tx_hash:
-        'eff00a9538487ff44243c75fb13de19b5783454c42c81b9aff9afbfd09cbaec3'
-    },
-    {
-      height: 511464,
-      tx_hash:
-        '7e9aa7a74de2b30200a2d6fc748ff35a0c753221444194f720bb7f61ef1d9153'
-    },
-    {
-      height: 513373,
-      tx_hash:
-        '6960255abe64893073921e96bf3c053c82686e0fc22a565494fbe2a31e766975'
-    },
-    {
-      height: 513373,
-      tx_hash:
-        '9ea667bcfc9cd337bd6c5583d8094c1b1942bd2015d95b54189deac5070eeff0'
-    },
-    {
-      height: 560481,
-      tx_hash:
-        'ecc1b51bac767880382bf3190ff17abf78d0936843a022a943d871116ed50368'
-    },
-    {
-      height: 560615,
-      tx_hash:
-        'b3792d28377b975560e1b6f09e48aeff8438d4c6969ca578bd406393bd50bd7d'
-    },
-    {
-      height: 561568,
-      tx_hash:
-        '8bc2134c7e48e56e1769b3d7c4c1e3a0acc68e1e58160eee6fa67f3208c07262'
-    },
-    {
-      height: 561569,
-      tx_hash:
-        'ceb0cab0e37b59caf3ca29e1a698d19ff47f2827dd09cb2f3b91b9100b1dad1c'
-    },
-    {
-      height: 561572,
-      tx_hash:
-        '0f9b49cafeb9ae1d741cdb12137c92816aa8470944c270a78ba2e610bd59190d'
-    },
-    {
-      height: 561582,
-      tx_hash:
-        'e4a0ac48ff3f42fc342717a2a3d34248e5e85bae79d59bd20e1b60e61b1c500f'
-    },
-    {
-      height: 562106,
-      tx_hash:
-        '1afcc63b244182647909539ebe3f4a44b8ea4120a95edb8d9eebe5347b9491bb'
-    },
-    {
-      height: 562106,
-      tx_hash:
-        'c42f8f16d3baa2ee343ea89ef110dfe094992379d08edd30887b8ca7ee671c9a'
+      transactions: [
+        {
+          height: 511463,
+          tx_hash:
+            'eff00a9538487ff44243c75fb13de19b5783454c42c81b9aff9afbfd09cbaec3'
+        },
+        {
+          height: 511464,
+          tx_hash:
+            '7e9aa7a74de2b30200a2d6fc748ff35a0c753221444194f720bb7f61ef1d9153'
+        },
+        {
+          height: 513373,
+          tx_hash:
+            '6960255abe64893073921e96bf3c053c82686e0fc22a565494fbe2a31e766975'
+        },
+        {
+          height: 513373,
+          tx_hash:
+            '9ea667bcfc9cd337bd6c5583d8094c1b1942bd2015d95b54189deac5070eeff0'
+        },
+        {
+          height: 560481,
+          tx_hash:
+            'ecc1b51bac767880382bf3190ff17abf78d0936843a022a943d871116ed50368'
+        },
+        {
+          height: 560615,
+          tx_hash:
+            'b3792d28377b975560e1b6f09e48aeff8438d4c6969ca578bd406393bd50bd7d'
+        },
+        {
+          height: 561568,
+          tx_hash:
+            '8bc2134c7e48e56e1769b3d7c4c1e3a0acc68e1e58160eee6fa67f3208c07262'
+        },
+        {
+          height: 561569,
+          tx_hash:
+            'ceb0cab0e37b59caf3ca29e1a698d19ff47f2827dd09cb2f3b91b9100b1dad1c'
+        },
+        {
+          height: 561572,
+          tx_hash:
+            '0f9b49cafeb9ae1d741cdb12137c92816aa8470944c270a78ba2e610bd59190d'
+        },
+        {
+          height: 561582,
+          tx_hash:
+            'e4a0ac48ff3f42fc342717a2a3d34248e5e85bae79d59bd20e1b60e61b1c500f'
+        },
+        {
+          height: 562106,
+          tx_hash:
+            '1afcc63b244182647909539ebe3f4a44b8ea4120a95edb8d9eebe5347b9491bb'
+        },
+        {
+          height: 562106,
+          tx_hash:
+            'c42f8f16d3baa2ee343ea89ef110dfe094992379d08edd30887b8ca7ee671c9a'
+        }
+      ]
     }
   ]
 }
@@ -159,16 +163,25 @@ const mockTxDetails2 = {
 
 const mockFulcrumNoTxHistory = {
   success: true,
-  transactions: []
+  transactions: [
+    {
+      transactions: [],
+      address: 'bitcoincash:qrgqqkky28jdkv3w0ctrah0mz3jcsnsklc34gtukrh'
+    }
+  ]
 }
 
 const mockFulcrumNoSendBalance = {
   success: true,
   transactions: [
     {
-      height: 633578,
-      tx_hash:
-        'a3b62cd4f4c56ba52139179db14bffd4ab22a2e077f3c62bd5cf0541bfcaf023'
+      transactions: [
+        {
+          height: 633578,
+          tx_hash:
+            'a3b62cd4f4c56ba52139179db14bffd4ab22a2e077f3c62bd5cf0541bfcaf023'
+        }
+      ]
     }
   ]
 }

test/v4/price.js

@@ -277,4 +277,70 @@ describe('#PriceRouter', () => {
       assert.isNumber(result.usd)
     })
   })
+
+  describe('#getBCHUSD', () => {
+    // const getNetworkInfo = controlRoute.testableComponents.getNetworkInfo
+
+    it('should throw 500 when network issues', async () => {
+      uut.bchCoinexPriceUrl = 'http://fakeurl/api/'
+
+      await uut.getBCHUSD(req, res)
+      // console.log(`result: ${util.inspect(result)}`)
+
+      assert.isAbove(
+        res.statusCode,
+        499,
+        'HTTP status code 500 or greater expected.'
+      )
+      // console.log(res)
+      assert.include(
+        res.output.error,
+        'Network error: Could not communicate with full node or other external service'
+      )
+    })
+
+    it('returns proper error when downstream service stalls', async () => {
+      // Mock the timeout error.
+      sandbox.stub(uut.axios, 'request').throws({ code: 'ECONNABORTED' })
+
+      const result = await uut.getBCHUSD(req, res)
+      // console.log(`result: ${JSON.stringify(result, null, 2)}`)
+
+      assert.isAbove(res.statusCode, 499, 'HTTP status code 503 expected.')
+      assert.include(
+        result.error,
+        'Could not communicate with full node',
+        'Error message expected'
+      )
+    })
+
+    it('returns proper error when downstream service is down', async () => {
+      // Mock the timeout error.
+      sandbox.stub(uut.axios, 'request').throws({ code: 'ECONNREFUSED' })
+
+      const result = await uut.getBCHUSD(req, res)
+      // console.log(`result: ${JSON.stringify(result, null, 2)}`)
+
+      assert.isAbove(res.statusCode, 499, 'HTTP status code 503 expected.')
+      assert.include(
+        result.error,
+        'Could not communicate with full node',
+        'Error message expected'
+      )
+    })
+
+    it('should get the USD price of BCH', async () => {
+      // Mock the RPC call for unit tests.
+      if (process.env.TEST === 'unit') {
+        sandbox
+          .stub(uut.axios, 'request')
+          .resolves({ data: mockData.mockCoinexFeed })
+      }
+
+      const result = await uut.getBCHUSD(req, res)
+      // console.log(`result: ${util.inspect(result)}`)
+
+      assert.isNumber(result.usd)
+    })
+  })
 })

test/v4/rate-limit-unit.js

@@ -0,0 +1,505 @@
+/*
+  Unit tests for the route-ratelimit2.js middleware.
+*/
+
+'use strict'
+
+// Public npm libraries.
+const assert = require('chai').assert
+const sinon = require('sinon')
+const cloneDeep = require('lodash.clonedeep')
+
+const config = require('../../config')
+
+// Mocking data.
+const { mockReq, mockRes, mockNext } = require('./mocks/express-mocks')
+
+// Libraries under test
+const RateLimits = require('../../src/middleware/route-ratelimit')
+let uut = new RateLimits()
+
+let req, res, next
+
+describe('#rate-routelimit', () => {
+  let sandbox
+
+  before(async () => {
+    if (!process.env.JWT_AUTH_SERVER) {
+      process.env.JWT_AUTH_SERVER = 'http://fakeurl.com/'
+    }
+
+    // Wipe the Redis DB, which prevents false negatives when running integration
+    // tests back-to-back.
+    await uut.wipeRedis()
+  })
+
+  // Setup the mocks before each test.
+  beforeEach(() => {
+    // Mock the req and res objects used by Express routes.
+    req = cloneDeep(mockReq)
+    res = cloneDeep(mockRes)
+    next = mockNext
+
+    // Explicitly reset the parmas and body.
+    req.params = {}
+    req.body = {}
+    req.query = {}
+    req.locals = {}
+
+    sandbox = sinon.createSandbox()
+
+    uut = new RateLimits()
+  })
+
+  afterEach(() => {
+    sandbox.restore()
+  })
+
+  after(() => {
+    uut.closeRedis()
+  })
+
+  describe('#checkInternalIp', () => {
+    it('should return true for a request from localhost', () => {
+      req.ip = '::ffff:127.0.0.1'
+
+      const result = uut.checkInternalIp(req)
+
+      assert.equal(result, true)
+    })
+
+    it('should return true for a request from a Docker container', () => {
+      req.ip = '172.17.0.3'
+
+      const result = uut.checkInternalIp(req)
+
+      assert.equal(result, true)
+    })
+
+    it('should return false for a random ip address', () => {
+      req.ip = '123.456.7.8'
+
+      const result = uut.checkInternalIp(req)
+
+      assert.equal(result, false)
+    })
+
+    it('should return false when an error is encountered', () => {
+      req.ip = 4
+
+      const result = uut.checkInternalIp(req)
+
+      assert.equal(result, false)
+    })
+  })
+
+  describe('#isInWhitelist', () => {
+    it('should return false when no argument is passed in', () => {
+      const result = uut.isInWhitelist()
+
+      assert.equal(result, false)
+    })
+
+    it('should return false when origin is not in the whitelist', () => {
+      req.origin = 'blah.com'
+      req.get = sandbox.stub().returns(req.origin)
+
+      const result = uut.isInWhitelist(req)
+
+      assert.equal(result, false)
+
+      // Used to appease linter. Remove these.
+      res.blah = 4
+      next()
+    })
+
+    it('should return true when origin is in the whitelist', () => {
+      req.origin = 'message.fullstack.cash'
+      req.get = sandbox.stub().returns(req.origin)
+
+      const result = uut.isInWhitelist(req)
+
+      assert.equal(result, true)
+    })
+  })
+
+  describe('#decodeJwtToken', () => {
+    it('should return the default JWT payload if decoding fails', () => {
+      const jwt =
+        'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6IjVlODhhY2JmMDIyMWMxMDAxMmFkOTNmZiIsImVtYWlsIjoiY2hyaXMudHJvdXRuZXJAZ21haWwuY29tIiwiYXBpTGV2ZWwiOjQwLCJyYXRlTGltaXQiOjMsImlhdCI6MTYxNTE1NzA4NywiZXhwIjoxNjE3NzQ5MDg3fQ.RLNGuYAa-CcLdhTGD27tDeaxT6-GIdeR8T4JWZZLDZA'
+
+      const result = uut.decodeJwtToken(jwt)
+      // console.log('result: ', result)
+
+      assert.property(result, 'id')
+      assert.equal(result.id, '123.456.789.10')
+      assert.property(result, 'email')
+      assert.equal(result.email, 'test@bchtest.net')
+      assert.property(result, 'pointsToConsume')
+      assert.equal(result.pointsToConsume, config.anonRateLimit)
+      assert.property(result, 'duration')
+      assert.equal(result.duration, 30)
+      assert.property(result, 'exp')
+    })
+
+    it('should return the default JWT payload if no input is given', () => {
+      const result = uut.decodeJwtToken()
+      // console.log('result: ', result)
+
+      assert.property(result, 'id')
+      assert.equal(result.id, '123.456.789.10')
+      assert.property(result, 'email')
+      assert.equal(result.email, 'test@bchtest.net')
+      assert.property(result, 'pointsToConsume')
+      assert.equal(result.pointsToConsume, config.anonRateLimit)
+      assert.property(result, 'duration')
+      assert.equal(result.duration, 30)
+      assert.property(result, 'exp')
+    })
+
+    it('should correctly decode a JWT token', () => {
+      // Generate a new JWT token for the test.
+      const jwtPayload = {
+        id: '5dade3f5739e6c0ff034b9a1',
+        pointsToConsume: 10,
+        email: 'gooduser@test.com',
+        apiLevel: 40,
+        rateLimit: 100,
+        duration: 30
+      }
+      const jwtToken = uut.generateJwtToken(jwtPayload)
+
+      const result = uut.decodeJwtToken(jwtToken)
+      // console.log('result: ', result)
+
+      assert.property(result, 'id')
+      assert.equal(result.id, jwtPayload.id)
+      assert.property(result, 'email')
+      assert.equal(result.email, jwtPayload.email)
+      assert.property(result, 'pointsToConsume')
+      assert.equal(result.pointsToConsume, jwtPayload.pointsToConsume)
+      assert.property(result, 'duration')
+      assert.equal(result.duration, jwtPayload.duration)
+      assert.property(result, 'exp')
+    })
+
+    it('should return the default payload if there is an unhandled error', () => {
+      // Force an error.
+      sandbox.stub(uut, 'generateJwtToken').throws(new Error('test error'))
+
+      const result = uut.decodeJwtToken()
+      // console.log('result: ', result)
+
+      assert.property(result, 'id')
+      assert.equal(result.id, '123.456.789.10')
+      assert.property(result, 'email')
+      assert.equal(result.email, 'test@bchtest.net')
+      assert.property(result, 'pointsToConsume')
+      assert.equal(result.pointsToConsume, config.anonRateLimit)
+      assert.property(result, 'duration')
+      assert.equal(result.duration, 30)
+      assert.property(result, 'exp')
+    })
+  })
+
+  describe('#trackRateLimits', () => {
+    it('should apply anonymous rate limits if no JWT token is provided', async () => {
+      req.ip = '127.0.0.1'
+
+      const result = await uut.trackRateLimits(req, res)
+      // console.log(`result: `, result)
+
+      // console.log('res.locals.pointsToConsume: ', res.locals.pointsToConsume)
+
+      assert.equal(result, false, 'Rate limits not exceeded')
+      assert.equal(
+        res.locals.pointsToConsume,
+        config.anonRateLimit,
+        'Anonymous rate limits applied'
+      )
+    })
+
+    it('should apply 100 RPM rate limits when JWT token is provided', async () => {
+      // Generate a new JWT token for the test.
+      const jwtPayload = {
+        id: '5dade3f5739e6c0ff034b9a1',
+        pointsToConsume: 10
+      }
+      const jwtToken = uut.generateJwtToken(jwtPayload)
+
+      const result = await uut.trackRateLimits(req, res, jwtToken)
+      // console.log(`result: `, result)
+
+      // console.log('res.locals.pointsToConsume: ', res.locals.pointsToConsume)
+
+      assert.equal(result, false, 'Rate limits not exceeded')
+      assert.equal(res.locals.pointsToConsume, 10, '100 RPM limits applied')
+    })
+  })
+
+  describe('#applyRateLimits', () => {
+    it('should skip rate limits if basic auth token is used', async () => {
+      req.locals.proLimit = true
+
+      // console.log('next.callCount: ', next.callCount)
+      const startCallCount = next.callCount
+
+      await uut.applyRateLimits(req, res, next)
+
+      // console.log('next.callCount: ', next.callCount)
+      const endCallCount = next.callCount
+
+      assert.isAbove(
+        endCallCount,
+        startCallCount,
+        'Expecting next() to be called'
+      )
+    })
+
+    it('should skip rate limits if internal call passes basic auth token', async () => {
+      req.ip = '127.0.0.1'
+      req.body.usrObj = {
+        proLimit: true
+      }
+
+      // console.log('next.callCount: ', next.callCount)
+      const startCallCount = next.callCount
+
+      await uut.applyRateLimits(req, res, next)
+
+      // console.log('next.callCount: ', next.callCount)
+      const endCallCount = next.callCount
+
+      assert.isAbove(
+        endCallCount,
+        startCallCount,
+        'Expecting next() to be called'
+      )
+    })
+
+    it('should apply rate limits to anonymous users', async () => {
+      req.ip = '123.456.7.8'
+
+      // console.log('next.callCount: ', next.callCount)
+      const startCallCount = next.callCount
+
+      await uut.applyRateLimits(req, res, next)
+
+      // console.log('next.callCount: ', next.callCount)
+      const endCallCount = next.callCount
+
+      assert.isAbove(
+        endCallCount,
+        startCallCount,
+        'Expecting next() to be called'
+      )
+
+      assert.equal(
+        res.locals.pointsToConsume,
+        config.anonRateLimit,
+        'Anonymous rate limits applied'
+      )
+    })
+
+    it('should return 429 error when anonymous users exceed rate limit', async () => {
+      req.ip = '123.456.7.8'
+
+      // force req.locals.jwtToken to be empty.
+      req.locals.jwtToken = undefined
+
+      let val
+      for (let i = 0; i < 25; i++) {
+        console.log('req.locals: ', req.locals)
+        val = await uut.applyRateLimits(req, res, next)
+      }
+      console.log('val: ', val)
+
+      assert.property(val, 'error')
+      assert.include(
+        val.error,
+        'Too many requests. Your limits are currently 20 requests per minute.'
+      )
+
+      assert.equal(res.locals.rateLimitTriggered, true, 'Rate limits triggered')
+
+      assert.equal(
+        res.locals.pointsToConsume,
+        config.anonRateLimit,
+        'Anonymous rate limits applied'
+      )
+    })
+
+    it('should apply rate limits when JWT token is provided', async () => {
+      // Generate a new JWT token for the test.
+      const jwtPayload = {
+        id: '5dade3f5739e6c0ff034b9a1',
+        pointsToConsume: 10
+      }
+      const jwtToken = uut.generateJwtToken(jwtPayload)
+
+      req.ip = '123.456.7.8'
+      req.locals.jwtToken = jwtToken
+
+      // console.log('next.callCount: ', next.callCount)
+      const startCallCount = next.callCount
+
+      await uut.applyRateLimits(req, res, next)
+
+      // console.log('next.callCount: ', next.callCount)
+      const endCallCount = next.callCount
+
+      assert.isAbove(
+        endCallCount,
+        startCallCount,
+        'Expecting next() to be called'
+      )
+
+      assert.equal(
+        res.locals.pointsToConsume,
+        10,
+        'Anonymous rate limits applied'
+      )
+    })
+
+    it('should apply internal rate limits to internal calls', async () => {
+      req.ip = '127.0.0.1'
+
+      // console.log('next.callCount: ', next.callCount)
+      const startCallCount = next.callCount
+
+      await uut.applyRateLimits(req, res, next)
+
+      // console.log('next.callCount: ', next.callCount)
+      const endCallCount = next.callCount
+
+      assert.isAbove(
+        endCallCount,
+        startCallCount,
+        'Expecting next() to be called'
+      )
+
+      assert.equal(
+        res.locals.pointsToConsume,
+        10,
+        'Internal rate limits applied'
+      )
+    })
+
+    it('should return 429 error when internal calls exceed interal rate limit', async () => {
+      req.ip = '127.0.0.1'
+
+      let val
+      for (let i = 0; i < 1025; i++) {
+        val = await uut.applyRateLimits(req, res, next)
+      }
+
+      assert.property(val, 'error')
+      assert.include(
+        val.error,
+        'Too many requests. Your limits are currently 1000 requests per minute.'
+      )
+
+      assert.equal(res.locals.rateLimitTriggered, true, 'Rate limits triggered')
+
+      assert.equal(
+        res.locals.pointsToConsume,
+        10,
+        'Internal rate limits applied'
+      )
+    })
+
+    it('should apply JWT rate limits to internal calls when JWT passes through', async () => {
+      // Generate a new JWT token for the test.
+      const jwtPayload = {
+        id: '5dade3f5739e6c0ff034b9a1',
+        pointsToConsume: 10
+      }
+      const jwtToken = uut.generateJwtToken(jwtPayload)
+
+      req.ip = '127.0.0.1'
+      req.body.usrObj = {
+        jwtToken
+      }
+
+      // console.log('next.callCount: ', next.callCount)
+      const startCallCount = next.callCount
+
+      await uut.applyRateLimits(req, res, next)
+
+      // console.log('next.callCount: ', next.callCount)
+      const endCallCount = next.callCount
+
+      assert.isAbove(
+        endCallCount,
+        startCallCount,
+        'Expecting next() to be called'
+      )
+
+      assert.equal(
+        res.locals.pointsToConsume,
+        10,
+        'User JWT rate limits applied'
+      )
+    })
+
+    it('should return 429 error when internal calls using JWT pass-through exceeds rate limit', async () => {
+      // Generate a new JWT token for the test.
+      const jwtPayload = {
+        id: '5dade3f5739e6c0ff034b9a1',
+        pointsToConsume: 10
+      }
+      const jwtToken = uut.generateJwtToken(jwtPayload)
+
+      req.ip = '127.0.0.1'
+      req.body.usrObj = {
+        jwtToken
+      }
+
+      try {
+        let val
+        for (let i = 0; i < 120; i++) {
+          val = await uut.applyRateLimits(req, res, next)
+        }
+        console.log('val: ', val)
+
+        assert.property(val, 'error')
+        assert.include(
+          val.error,
+          'Too many requests. Your limits are currently 100 requests per minute.'
+        )
+
+        assert.equal(
+          res.locals.pointsToConsume,
+          10,
+          'User JWT rate limits applied'
+        )
+      } catch (err) {
+        console.log('err: ', err)
+      }
+    })
+
+    it('should move to the next middleware when encountering an unexpected internal error', async () => {
+      // Force the creation of the res and req locals property. Covers an
+      // otherwise untested code path.
+      req.locals = undefined
+      res.locals = undefined
+
+      // Force an error
+      sandbox.stub(uut, 'checkInternalIp').throws(new Error('test error'))
+
+      // console.log('next.callCount: ', next.callCount)
+      const startCallCount = next.callCount
+
+      await uut.applyRateLimits(req, res, next)
+
+      // console.log('next.callCount: ', next.callCount)
+      const endCallCount = next.callCount
+
+      assert.isAbove(
+        endCallCount,
+        startCallCount,
+        'Expecting next() to be called'
+      )
+    })
+  })
+})

test/v4/rate-limits.js

@@ -1,487 +0,0 @@
-'use strict'
-
-const chai = require('chai')
-const assert = chai.assert
-const sinon = require('sinon')
-
-// Used for debugging.
-const util = require('util')
-util.inspect.defaultOptions = { depth: 1 }
-
-// Mocking data.
-const { mockReq, mockRes, mockNext } = require('./mocks/express-mocks')
-
-// Libraries under test
-const RateLimits = require('../../src/middleware/route-ratelimit')
-let rateLimits = new RateLimits()
-
-// const controlRoute = require('../../src/routes/v4/full-node/control')
-const jwtAuth = require('../../src/middleware/jwt-auth')
-
-let req, res, next
-// let originalEnvVars // Used during transition from integration to unit tests.
-
-// JWT token used in tests.
-const jwt = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6IjVkYWRlM2Y1NzM5ZTZjMGZmMDM0YjlhMSIsImlhdCI6MTU3MTY3NzQ1MCwiZXhwIjoxNTc0MjY5NDUwfQ.SSz7F7ETyBB3eoNG2VKCzPOhddtB-vrtmEoj7PxicrQ'
-
-describe('#route-ratelimits & jwt-auth', () => {
-  let sandbox
-
-  before(async () => {
-    // Save existing environment variables.
-    // originalEnvVars = {
-    //   BITCOINCOM_BASEURL: process.env.BITCOINCOM_BASEURL,
-    //   RPC_BASEURL: process.env.RPC_BASEURL,
-    //   RPC_USERNAME: process.env.RPC_USERNAME,
-    //   RPC_PASSWORD: process.env.RPC_PASSWORD
-    // }
-
-    if (!process.env.JWT_AUTH_SERVER) { process.env.JWT_AUTH_SERVER = 'http://fakeurl.com/' }
-
-    // Wipe the Redis DB, which prevents false negatives when running integration
-    // tests back-to-back.
-    await rateLimits.wipeRedis()
-  })
-
-  // Setup the mocks before each test.
-  beforeEach(() => {
-    // Mock the req and res objects used by Express routes.
-    req = Object.assign({}, mockReq)
-    res = Object.assign({}, mockRes)
-    next = mockNext
-
-    // Explicitly reset the parmas and body.
-    req.params = {}
-    req.body = {}
-    req.query = {}
-    req.locals = {}
-
-    sandbox = sinon.createSandbox()
-  })
-
-  afterEach(() => {
-    sandbox.restore()
-  })
-
-  after(() => {
-    rateLimits.closeRedis()
-  })
-
-  describe('#jwt-auth.js', () => {
-    describe('#getTokenFromHeaders', () => {
-      it('should populate the req.locals object correctly', () => {
-        // Initialize req.locals
-        req.locals = {
-          proLimit: false,
-          apiLevel: 0
-        }
-
-        const header = `Token ${jwt}`
-        req.headers.authorization = header
-
-        jwtAuth.getTokenFromHeaders(req, res, next)
-
-        // console.log(`req.locals: ${JSON.stringify(req.locals, null, 2)}`)
-
-        assert.property(req.locals, 'proLimit')
-        assert.property(req.locals, 'apiLevel')
-        assert.property(req.locals, 'jwtToken')
-        assert.equal(req.locals.jwtToken, jwt)
-      })
-    })
-  })
-
-  describe('#getResource', () => {
-    it('should decode a blockchain request', () => {
-      const url =
-        '/blockchain/getTxOut/62a3ea958a463a372bc0caf2c374a7f60be9c624be63a0db8db78f05809df6d8/0?include_mempool=true'
-
-      const result = rateLimits.getResource(url)
-      // console.log(`result: ${JSON.stringify(result, null, 2)}`)
-
-      assert.equal(result, 'blockchain')
-    })
-  })
-
-  describe('#calcPoints', () => {
-    it('should return 50 points for anonymous user', () => {
-      const result = rateLimits.calcPoints()
-      // console.log(`result: ${result}`)
-
-      assert.equal(result, 50)
-    })
-
-    it('should return 50 points for free tier requesting full node access', () => {
-      const jwtInfo = {
-        apiLevel: 10,
-        resource: 'blockchain',
-        id: '5e3a0415eb29a962da2708b4'
-      }
-
-      const result = rateLimits.calcPoints(jwtInfo)
-      assert.equal(result, 50)
-    })
-
-    it('should return 50 points for free tier requesting indexer access', () => {
-      const jwtInfo = {
-        apiLevel: 10,
-        resource: 'blockbook',
-        id: '5e3a0415eb29a962da2708b4'
-      }
-
-      const result = rateLimits.calcPoints(jwtInfo)
-      assert.equal(result, 50)
-    })
-
-    it('should return 50 points for free tier requesting SLPDB access', () => {
-      const jwtInfo = {
-        apiLevel: 10,
-        resource: 'slp',
-        id: '5e3a0415eb29a962da2708b4'
-      }
-
-      const result = rateLimits.calcPoints(jwtInfo)
-      assert.equal(result, 50)
-    })
-
-    it('should return 10 points for full node tier requesting full node access', () => {
-      const jwtInfo = {
-        apiLevel: 20,
-        resource: 'blockchain',
-        id: '5e3a0415eb29a962da2708b4'
-      }
-
-      const result = rateLimits.calcPoints(jwtInfo)
-      assert.equal(result, 10)
-    })
-
-    it('should return 50 points for full-node tier requesting indexer access', () => {
-      const jwtInfo = {
-        apiLevel: 20,
-        resource: 'blockbook',
-        id: '5e3a0415eb29a962da2708b4'
-      }
-
-      const result = rateLimits.calcPoints(jwtInfo)
-      assert.equal(result, 50)
-    })
-
-    it('should return 50 points for full node tier requesting SLPDB access', () => {
-      const jwtInfo = {
-        apiLevel: 20,
-        resource: 'slp',
-        id: '5e3a0415eb29a962da2708b4'
-      }
-
-      const result = rateLimits.calcPoints(jwtInfo)
-      assert.equal(result, 50)
-    })
-
-    it('should return 10 point for indexer tier requesting full node access', () => {
-      const jwtInfo = {
-        apiLevel: 30,
-        resource: 'blockchain',
-        id: '5e3a0415eb29a962da2708b4'
-      }
-
-      const result = rateLimits.calcPoints(jwtInfo)
-      assert.equal(result, 10)
-    })
-
-    it('should return 10 points for indexer tier requesting indexer access', () => {
-      const jwtInfo = {
-        apiLevel: 30,
-        resource: 'blockbook',
-        id: '5e3a0415eb29a962da2708b4'
-      }
-
-      const result = rateLimits.calcPoints(jwtInfo)
-      assert.equal(result, 10)
-    })
-
-    it('should return 50 points for indexer tier requesting SLPDB access', () => {
-      const jwtInfo = {
-        apiLevel: 30,
-        resource: 'slp',
-        id: '5e3a0415eb29a962da2708b4'
-      }
-
-      const result = rateLimits.calcPoints(jwtInfo)
-      assert.equal(result, 50)
-    })
-
-    it('should return 10 point for SLP tier requesting full node access', () => {
-      const jwtInfo = {
-        apiLevel: 40,
-        resource: 'blockchain',
-        id: '5e3a0415eb29a962da2708b4'
-      }
-
-      const result = rateLimits.calcPoints(jwtInfo)
-      assert.equal(result, 10)
-    })
-
-    it('should return 10 points for SLP tier requesting indexer access', () => {
-      const jwtInfo = {
-        apiLevel: 40,
-        resource: 'blockbook',
-        id: '5e3a0415eb29a962da2708b4'
-      }
-
-      const result = rateLimits.calcPoints(jwtInfo)
-      assert.equal(result, 10)
-    })
-
-    it('should return 10 points for SLP tier requesting SLPDB access', () => {
-      const jwtInfo = {
-        apiLevel: 40,
-        resource: 'slp',
-        id: '5e3a0415eb29a962da2708b4'
-      }
-
-      const result = rateLimits.calcPoints(jwtInfo)
-      assert.equal(result, 10)
-    })
-  })
-
-  describe('#rateLimitByResource', () => {
-    // NOTE: this test will fail if you run multiple integration tests in a
-    // short period. Because it talks to the Redis DB.
-    it('should pass through rate-limit middleware', async () => {
-      req.baseUrl = '/v4'
-      req.path = '/control/getNetworkInfo'
-      req.url = req.path
-      req.method = 'GET'
-
-      // Call the route twice to trigger the rate handling.
-      await rateLimits.rateLimitByResource(req, res, next)
-      await rateLimits.rateLimitByResource(req, res, next)
-
-      // next() will be called if rate-limit is not triggered
-      assert.equal(next.called, true)
-    })
-
-    it('should trigger rate-limit handler if rate limits exceeds 5 request per minute', async () => {
-      req.baseUrl = '/v4'
-      req.path = '/control/getNetworkInfo'
-      req.url = req.path
-      req.method = 'GET'
-
-      for (let i = 0; i < 5; i++) {
-        next.reset() // reset the stubbed next() function.
-
-        await rateLimits.rateLimitByResource(req, res, next)
-        // console.log(`next() called: ${next.called}`)
-      }
-
-      // Note: next() will be called unless the rate-limit kicks in.
-      assert.equal(
-        next.called,
-        false,
-        'next should not be called if rate limit was triggered.'
-      )
-    })
-
-    it('should NOT trigger rate-limit for free-tier at 5 RPM', async () => {
-      // Create a new instance of the rate limit so we start with zeroed tracking.
-      rateLimits = new RateLimits()
-
-      req.baseUrl = '/v4'
-      req.path = '/control/getNetworkInfo'
-      req.url = req.path
-      req.method = 'GET'
-
-      req.locals.jwtToken = 'some-token'
-
-      const jwtInfo = {
-        apiLevel: 10,
-        id: '5e3a0415eb29a962da2708b1'
-      }
-
-      // Mock the call to the jwt library.
-      sandbox.stub(rateLimits.jwt, 'verify').returns(jwtInfo)
-
-      for (let i = 0; i < 5; i++) {
-        next.reset() // reset the stubbed next() function.
-
-        await rateLimits.rateLimitByResource(req, res, next)
-        // console.log(`next() called: ${next.called}`)
-      }
-
-      // console.log(`req.locals after test: ${util.inspect(req.locals)}`)
-
-      // Note: next() will be called unless the rate-limit kicks in.
-      assert.equal(
-        next.called,
-        true,
-        'next should be called if rate limit was not triggered.'
-      )
-    })
-
-    it('should trigger rate-limit for free tier after 20 RPM', async () => {
-      // Create a new instance of the rate limit so we start with zeroed tracking.
-      rateLimits = new RateLimits()
-
-      req.baseUrl = '/v4'
-      req.path = '/control/getNetworkInfo'
-      req.url = req.path
-      req.method = 'GET'
-
-      req.locals.jwtToken = 'some-token'
-
-      const jwtInfo = {
-        apiLevel: 10,
-        id: '5e3a0415eb29a962da2708b2'
-      }
-
-      // Mock the call to the jwt library.
-      sandbox.stub(rateLimits.jwt, 'verify').returns(jwtInfo)
-
-      for (let i = 0; i < 22; i++) {
-        next.reset() // reset the stubbed next() function.
-
-        await rateLimits.rateLimitByResource(req, res, next)
-        // console.log(`next() called: ${next.called}`)
-      }
-
-      // Note: next() will be called unless the rate-limit kicks in.
-      assert.equal(
-        next.called,
-        false,
-        'next should not be called if rate limit was triggered.'
-      )
-    })
-
-    it('should NOT trigger rate-limit handler for indexer-tier at 25 RPM', async () => {
-      // Create a new instance of the rate limit so we start with zeroed tracking.
-      rateLimits = new RateLimits()
-
-      req.baseUrl = '/v4'
-      req.path = '/control/getNetworkInfo'
-      req.url = req.path
-      req.method = 'GET'
-
-      req.locals.jwtToken = 'some-token'
-
-      const jwtInfo = {
-        apiLevel: 20,
-        id: '5e3a0415eb29a962da2708b3'
-      }
-
-      // Mock the call to the jwt library.
-      sandbox.stub(rateLimits.jwt, 'verify').returns(jwtInfo)
-
-      for (let i = 0; i < 25; i++) {
-        next.reset() // reset the stubbed next() function.
-
-        await rateLimits.rateLimitByResource(req, res, next)
-        // console.log(`next() called: ${next.called}`)
-      }
-
-      // console.log(`req.locals after test: ${util.inspect(req.locals)}`)
-
-      // Note: next() will be called unless the rate-limit kicks in.
-      assert.equal(
-        next.called,
-        true,
-        'next should be called if rate limit was not triggered.'
-      )
-    })
-
-    it('should still rate-limit at a higher RPM for pro-tier', async () => {
-      // Create a new instance of the rate limit so we start with zeroed tracking.
-      rateLimits = new RateLimits()
-
-      req.baseUrl = '/v4'
-      req.path = '/control/getNetworkInfo'
-      req.url = req.path
-      req.method = 'GET'
-
-      req.locals.jwtToken = 'some-token'
-
-      const jwtInfo = {
-        apiLevel: 20,
-        id: '5e3a0415eb29a962da2708b5'
-      }
-
-      // Mock the call to the jwt library.
-      sandbox.stub(rateLimits.jwt, 'verify').returns(jwtInfo)
-
-      for (let i = 0; i < 150; i++) {
-        next.reset() // reset the stubbed next() function.
-
-        await rateLimits.rateLimitByResource(req, res, next)
-        // console.log(`next() called: ${next.called}`)
-      }
-
-      // console.log(`req.locals after test: ${util.inspect(req.locals)}`)
-
-      // Note: next() will be called unless the rate-limit kicks in.
-      assert.equal(
-        next.called,
-        false,
-        'next should NOT be called if rate limit was triggered.'
-      )
-    })
-
-    // CT 2/24/21 This test may have been invalidated by the interal IP address
-    // passing that I implemented to get hydrateUtxos() working properly.
-    // I'm commenting this out until I can study the side effects of this change,
-    // and why exactly this test is breaking.
-    // it('should handle misconfigured token secret', async () => {
-    //   // Create a new instance of the rate limit so we start with zeroed tracking.
-    //   rateLimits = new RateLimits()
-    //
-    //   req.baseUrl = '/v4'
-    //   req.path = '/control/getNetworkInfo'
-    //   req.url = req.path
-    //   req.method = 'GET'
-    //
-    //   req.locals.jwtToken = 'some-token'
-    //
-    //   next.reset() // reset the stubbed next() function.
-    //
-    //   await rateLimits.rateLimitByResource(req, res, next)
-    //
-    //   // Issues with token secret should treat incoming requests as anonymous
-    //   // calls with 50 points, or 20 RPM.
-    //   assert.equal(res.locals.pointsToConsume, 50)
-    // })
-  })
-
-  describe('#isInWhitelist', () => {
-    it('should return false when no argument is passed in', () => {
-      const result = rateLimits.isInWhitelist()
-
-      assert.equal(result, false)
-    })
-
-    it('should return false when origin is not in the whitelist', () => {
-      const origin = 'blah.com'
-
-      const result = rateLimits.isInWhitelist(origin)
-
-      assert.equal(result, false)
-    })
-
-    it('should return true when origin is in the whitelist', () => {
-      const origin = 'message.fullstack.cash'
-
-      const result = rateLimits.isInWhitelist(origin)
-
-      assert.equal(result, true)
-    })
-  })
-})
-
-// Generates a Basic authorization header.
-// function generateAuthHeader (pass) {
-//   // https://en.wikipedia.org/wiki/Basic_access_authentication
-//   const username = 'BITBOX'
-//   const combined = `${username}:${pass}`
-//
-//   var base64Credential = Buffer.from(combined).toString('base64')
-//   var readyCredential = `Basic ${base64Credential}`
-//
-//   return readyCredential
-// }