fix(rateLimitSimple): Created simpler rate-limit middleware for rest.bitcoin.com

src/middleware/route-ratelimit.js

@@ -29,6 +29,11 @@ const rateLimitOptions = {
   duration: 1 // Per second
 }
 
+// This hard-coded value is temporary. It will be swapped out with an environment
+// variable when moved to production.
+const publicKey =
+  "03e6c358092a459f7da9420de770eef3e16cf3c9c54a3d3d14ac2d7f0b82af4d7d"
+
 // Set max requests per minute
 const maxRequests = process.env.RATE_LIMIT_MAX_REQUESTS
   ? parseInt(process.env.RATE_LIMIT_MAX_REQUESTS)
@@ -56,6 +61,7 @@ class RateLimits {
     redisClient.disconnect()
   }
 
+  // CT 2/7/20: Older rate-limiting code that does not scale well.
   /*
     This function controls the tierd request-per-minute (RPM) rate limits.
     This is an older implementation that is currently not used.
@@ -73,7 +79,6 @@ class RateLimits {
     will be downgraded to 0 on-the-fly. Indexer endpoints will effectively be
     downgraded to the anonymous access tier.
   */
-  // CT 2/7/20: Older rate-limiting code that does not scale well.
   async routeRateLimit(req, res, next) {
     // Disable rate limiting if 0 passed from RATE_LIMIT_MAX_REQUESTS
     if (maxRequests === 0) return next()
@@ -201,12 +206,12 @@ class RateLimits {
     uniqueRateLimits[route](req, res, next)
   }
 
+  // CT 2/7/20: I believe this is older code that is only used by routeRateLimit.
+  // It will probably be removed in the future.
   // This function returns an object with proLimit and apiLevel properties.
   // It does fine-grane analysis on the data coming from the auth servers and
   // uses its output to adjust rate limits on-the-fly based on the users
   // permission level.
-  // CT 2/7/20: I believe this is older code that is only used by routeRateLimit.
-  // It will probably be removed in the future.
   evalUserPermissioins(req, authData) {
     // console.log(`authData: ${JSON.stringify(authData, null, 2)}`)
 
@@ -241,10 +246,9 @@ class RateLimits {
     return retObj
   }
 
-  /*
-    This is the new rate limit function that uses the rate-limiter-flexible npm
-    library.
-  */
+  // This is the new rate limit function that uses the rate-limiter-flexible npm
+  // library. It uses fine-grain rate limiting based on the resources being
+  // consumed.
   async rateLimitByResource(req, res, next) {
     try {
       let userId
@@ -262,10 +266,6 @@ class RateLimits {
 
       // Decode the JWT token if one exists.
       if (req.locals.jwtToken) {
-        // Hexadecimal
-        const publicKey =
-          "03e6c358092a459f7da9420de770eef3e16cf3c9c54a3d3d14ac2d7f0b82af4d7d"
-
         const jwtOptions = {
           algorithms: ["ES256"]
         }
@@ -392,6 +392,101 @@ class RateLimits {
       throw err
     }
   }
+
+  // This is a variation of rateLimitByResource() function. This version will
+  // potentially be used by Bitcoin.com.
+  // Rather than using apiLevel, the rateLimit is explicitly recorded in the
+  // JWT token.
+  async rateLimitSimple(req, res, next) {
+    try {
+      let userId
+      let decoded = {}
+
+      // Create a res.locals object if not passed in.
+      if (!req.locals) {
+        req.locals = {
+          // default values
+          jwtToken: "",
+          proLimit: false,
+          rateLimit: 3
+        }
+      }
+
+      // Decode the JWT token if one exists.
+      if (req.locals.jwtToken) {
+        const jwtOptions = {
+          algorithms: ["ES256"]
+        }
+
+        const pemPublicKey = keyEncoder.encodePublic(publicKey, "raw", "pem")
+
+        // Validate the JWT token.
+        decoded = _this.jwt.verify(
+          req.locals.jwtToken,
+          pemPublicKey,
+          jwtOptions
+        )
+        // console.log(`decoded: ${JSON.stringify(decoded, null, 2)}`)
+
+        userId = decoded.id
+      } else {
+        wlogger.debug(`No JWT token found!`)
+      }
+
+      // Code here for the rate limiter is adapted from this example:
+      // https://github.com/animir/node-rate-limiter-flexible/wiki/Overall-example#authorized-and-not-authorized-users
+      try {
+        // Key for Redis key/value pair.
+        const key = userId ? userId : req.ip
+
+        const pointsToConsume = _this.calcPoints2(decoded)
+
+        wlogger.debug(`User ${key} consuming ${pointsToConsume}.`)
+
+        await _this.rateLimiter.consume(key, pointsToConsume)
+      } catch (err) {
+        // console.log(`err: `, err)
+
+        // Rate limited was triggered
+        res.status(429) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
+        return res.json({
+          error: `Too many requests. Your limits are currently ${maxRequests} requests per minute. Increase rate limits at https://account.bchjs.cash`
+        })
+      }
+    } catch (err) {
+      wlogger.error(`Error in route-ratelimit.js/rateLimitSimple(): `, err)
+      // throw err
+    }
+
+    next()
+  }
+
+  // Calculates the points consumed, based on the explicit rateLimit defined
+  // in the JWT token.
+  calcPoints2(jwtInfo) {
+    let retVal = 30 // By default, use anonymous tier.
+
+    try {
+      // console.log(`jwtInfo: ${JSON.stringify(jwtInfo, null, 2)}`)
+
+      const MAX_RATE_LIMIT = 100
+
+      const rateLimit = jwtInfo.rateLimit
+
+      // Only evaluate if user is using a JWT token.
+      if (jwtInfo.id) {
+        const points = Math.floor(MAX_RATE_LIMIT / rateLimit)
+
+        retVal = points
+      }
+    } catch (err) {
+      wlogger.error(`Error in route-ratelimit.js/calcPoints2()`)
+      // throw err
+      retVal = 30
+    }
+
+    return retVal
+  }
 }
 
 module.exports = RateLimits

test/v3/rate-limits.js

@@ -53,6 +53,7 @@ describe("#route-ratelimits & jwt-auth", () => {
     req.params = {}
     req.body = {}
     req.query = {}
+    req.locals = {}
 
     sandbox = sinon.createSandbox()
   })
@@ -403,6 +404,35 @@ describe("#route-ratelimits & jwt-auth", () => {
     })
   })
 
+  describe("#calcPoints2", () => {
+    it("should return 30 points for anonymous user", () => {
+      const result = rateLimits.calcPoints2({})
+      // console.log(`result: ${result}`)
+
+      assert.equal(result, 30)
+    })
+
+    it("should return 10 point for free tier", () => {
+      const jwtInfo = {
+        rateLimit: 10,
+        id: "5e3a0415eb29a962da2708b4"
+      }
+
+      const result = rateLimits.calcPoints2(jwtInfo)
+      assert.equal(result, 10)
+    })
+
+    it("should return 1 point for pro tier", () => {
+      const jwtInfo = {
+        rateLimit: 100,
+        id: "5e3a0415eb29a962da2708b4"
+      }
+
+      const result = rateLimits.calcPoints2(jwtInfo)
+      assert.equal(result, 1)
+    })
+  })
+
   describe("#rateLimitByResource", () => {
     it("should pass through rate-limit middleware", async () => {
       req.baseUrl = "/v3"
@@ -581,6 +611,185 @@ describe("#route-ratelimits & jwt-auth", () => {
       )
     })
   })
+
+  describe("#rateLimitSimple", () => {
+    it("should pass through rate-limit middleware", async () => {
+      req.baseUrl = "/v3"
+      req.path = "/control/getNetworkInfo"
+      req.url = req.path
+      req.method = "GET"
+
+      // Call the route twice to trigger the rate handling.
+      await rateLimits.rateLimitSimple(req, res, next)
+      await rateLimits.rateLimitSimple(req, res, next)
+
+      // next() will be called if rate-limit is not triggered
+      assert.equal(next.called, true)
+    })
+
+    it("should trigger rate-limit handler if rate limits exceeds 5 request per minute", async () => {
+      req.baseUrl = "/v3"
+      req.path = "/control/getNetworkInfo"
+      req.url = req.path
+      req.method = "GET"
+
+      for (let i = 0; i < 5; i++) {
+        next.reset() // reset the stubbed next() function.
+
+        await rateLimits.rateLimitSimple(req, res, next)
+        //console.log(`next() called: ${next.called}`)
+      }
+
+      // Note: next() will be called unless the rate-limit kicks in.
+      assert.equal(
+        next.called,
+        false,
+        `next should not be called if rate limit was triggered.`
+      )
+    })
+
+    it("should NOT trigger rate-limit for free-tier at 5 RPM", async () => {
+      // Create a new instance of the rate limit so we start with zeroed tracking.
+      rateLimits = new RateLimits()
+
+      req.baseUrl = "/v3"
+      req.path = "/control/getNetworkInfo"
+      req.url = req.path
+      req.method = "GET"
+
+      req.locals.jwtToken = "some-token"
+
+      const jwtInfo = {
+        rateLimit: 10,
+        id: "5e3a0415eb29a962da2708c1"
+      }
+
+      // Mock the call to the jwt library.
+      sandbox.stub(rateLimits.jwt, "verify").returns(jwtInfo)
+
+      for (let i = 0; i < 5; i++) {
+        next.reset() // reset the stubbed next() function.
+
+        await rateLimits.rateLimitSimple(req, res, next)
+        //console.log(`next() called: ${next.called}`)
+      }
+
+      //console.log(`req.locals after test: ${util.inspect(req.locals)}`)
+
+      // Note: next() will be called unless the rate-limit kicks in.
+      assert.equal(
+        next.called,
+        true,
+        `next should be called if rate limit was not triggered.`
+      )
+    })
+
+    it("should trigger rate-limit for free tier after 10 RPM", async () => {
+      // Create a new instance of the rate limit so we start with zeroed tracking.
+      rateLimits = new RateLimits()
+
+      req.baseUrl = "/v3"
+      req.path = "/control/getNetworkInfo"
+      req.url = req.path
+      req.method = "GET"
+
+      req.locals.jwtToken = "some-token"
+
+      const jwtInfo = {
+        rateLimit: 10,
+        id: "5e3a0415eb29a962da2708c2"
+      }
+
+      // Mock the call to the jwt library.
+      sandbox.stub(rateLimits.jwt, "verify").returns(jwtInfo)
+
+      for (let i = 0; i < 12; i++) {
+        next.reset() // reset the stubbed next() function.
+
+        await rateLimits.rateLimitSimple(req, res, next)
+        //console.log(`next() called: ${next.called}`)
+      }
+
+      // Note: next() will be called unless the rate-limit kicks in.
+      assert.equal(
+        next.called,
+        false,
+        `next should not be called if rate limit was triggered.`
+      )
+    })
+
+    it("should NOT trigger rate-limit handler for pro-tier at 25 RPM", async () => {
+      // Create a new instance of the rate limit so we start with zeroed tracking.
+      rateLimits = new RateLimits()
+
+      req.baseUrl = "/v3"
+      req.path = "/control/getNetworkInfo"
+      req.url = req.path
+      req.method = "GET"
+
+      req.locals.jwtToken = "some-token"
+
+      const jwtInfo = {
+        rateLimit: 100,
+        id: "5e3a0415eb29a962da2708c3"
+      }
+
+      // Mock the call to the jwt library.
+      sandbox.stub(rateLimits.jwt, "verify").returns(jwtInfo)
+
+      for (let i = 0; i < 25; i++) {
+        next.reset() // reset the stubbed next() function.
+
+        await rateLimits.rateLimitSimple(req, res, next)
+        //console.log(`next() called: ${next.called}`)
+      }
+
+      //console.log(`req.locals after test: ${util.inspect(req.locals)}`)
+
+      // Note: next() will be called unless the rate-limit kicks in.
+      assert.equal(
+        next.called,
+        true,
+        `next should be called if rate limit was not triggered.`
+      )
+    })
+
+    it("should still rate-limit at a higher RPM for pro-tier", async () => {
+      // Create a new instance of the rate limit so we start with zeroed tracking.
+      rateLimits = new RateLimits()
+
+      req.baseUrl = "/v3"
+      req.path = "/control/getNetworkInfo"
+      req.url = req.path
+      req.method = "GET"
+
+      req.locals.jwtToken = "some-token"
+
+      const jwtInfo = {
+        rateLimit: 100,
+        id: "5e3a0415eb29a962da2708c4"
+      }
+
+      // Mock the call to the jwt library.
+      sandbox.stub(rateLimits.jwt, "verify").returns(jwtInfo)
+
+      for (let i = 0; i < 150; i++) {
+        next.reset() // reset the stubbed next() function.
+
+        await rateLimits.rateLimitSimple(req, res, next)
+        //console.log(`next() called: ${next.called}`)
+      }
+
+      //console.log(`req.locals after test: ${util.inspect(req.locals)}`)
+
+      // Note: next() will be called unless the rate-limit kicks in.
+      assert.equal(
+        next.called,
+        false,
+        `next should NOT be called if rate limit was triggered.`
+      )
+    })
+  })
 })
 
 // Generates a Basic authorization header.