From dc1b4443098caa8987cb01868de9c0cb01a084a8 Mon Sep 17 00:00:00 2001 From: Chris Troutner Date: Mon, 10 Feb 2020 17:43:15 -0800 Subject: [PATCH] fix(rateLimitSimple): Created simpler rate-limit middleware for rest.bitcoin.com --- src/middleware/route-ratelimit.js | 117 +++++++++++++++-- test/v3/rate-limits.js | 209 ++++++++++++++++++++++++++++++ 2 files changed, 315 insertions(+), 11 deletions(-) diff --git a/src/middleware/route-ratelimit.js b/src/middleware/route-ratelimit.js index 9695370..1d71a6c 100644 --- a/src/middleware/route-ratelimit.js +++ b/src/middleware/route-ratelimit.js @@ -29,6 +29,11 @@ const rateLimitOptions = { duration: 1 // Per second } +// This hard-coded value is temporary. It will be swapped out with an environment +// variable when moved to production. +const publicKey = + "03e6c358092a459f7da9420de770eef3e16cf3c9c54a3d3d14ac2d7f0b82af4d7d" + // Set max requests per minute const maxRequests = process.env.RATE_LIMIT_MAX_REQUESTS ? parseInt(process.env.RATE_LIMIT_MAX_REQUESTS) @@ -56,6 +61,7 @@ class RateLimits { redisClient.disconnect() } + // CT 2/7/20: Older rate-limiting code that does not scale well. /* This function controls the tierd request-per-minute (RPM) rate limits. This is an older implementation that is currently not used. @@ -73,7 +79,6 @@ class RateLimits { will be downgraded to 0 on-the-fly. Indexer endpoints will effectively be downgraded to the anonymous access tier. */ - // CT 2/7/20: Older rate-limiting code that does not scale well. async routeRateLimit(req, res, next) { // Disable rate limiting if 0 passed from RATE_LIMIT_MAX_REQUESTS if (maxRequests === 0) return next() @@ -201,12 +206,12 @@ class RateLimits { uniqueRateLimits[route](req, res, next) } + // CT 2/7/20: I believe this is older code that is only used by routeRateLimit. + // It will probably be removed in the future. // This function returns an object with proLimit and apiLevel properties. // It does fine-grane analysis on the data coming from the auth servers and // uses its output to adjust rate limits on-the-fly based on the users // permission level. - // CT 2/7/20: I believe this is older code that is only used by routeRateLimit. - // It will probably be removed in the future. evalUserPermissioins(req, authData) { // console.log(`authData: ${JSON.stringify(authData, null, 2)}`) @@ -241,10 +246,9 @@ class RateLimits { return retObj } - /* - This is the new rate limit function that uses the rate-limiter-flexible npm - library. - */ + // This is the new rate limit function that uses the rate-limiter-flexible npm + // library. It uses fine-grain rate limiting based on the resources being + // consumed. async rateLimitByResource(req, res, next) { try { let userId @@ -262,10 +266,6 @@ class RateLimits { // Decode the JWT token if one exists. if (req.locals.jwtToken) { - // Hexadecimal - const publicKey = - "03e6c358092a459f7da9420de770eef3e16cf3c9c54a3d3d14ac2d7f0b82af4d7d" - const jwtOptions = { algorithms: ["ES256"] } @@ -392,6 +392,101 @@ class RateLimits { throw err } } + + // This is a variation of rateLimitByResource() function. This version will + // potentially be used by Bitcoin.com. + // Rather than using apiLevel, the rateLimit is explicitly recorded in the + // JWT token. + async rateLimitSimple(req, res, next) { + try { + let userId + let decoded = {} + + // Create a res.locals object if not passed in. + if (!req.locals) { + req.locals = { + // default values + jwtToken: "", + proLimit: false, + rateLimit: 3 + } + } + + // Decode the JWT token if one exists. + if (req.locals.jwtToken) { + const jwtOptions = { + algorithms: ["ES256"] + } + + const pemPublicKey = keyEncoder.encodePublic(publicKey, "raw", "pem") + + // Validate the JWT token. + decoded = _this.jwt.verify( + req.locals.jwtToken, + pemPublicKey, + jwtOptions + ) + // console.log(`decoded: ${JSON.stringify(decoded, null, 2)}`) + + userId = decoded.id + } else { + wlogger.debug(`No JWT token found!`) + } + + // Code here for the rate limiter is adapted from this example: + // https://github.com/animir/node-rate-limiter-flexible/wiki/Overall-example#authorized-and-not-authorized-users + try { + // Key for Redis key/value pair. + const key = userId ? userId : req.ip + + const pointsToConsume = _this.calcPoints2(decoded) + + wlogger.debug(`User ${key} consuming ${pointsToConsume}.`) + + await _this.rateLimiter.consume(key, pointsToConsume) + } catch (err) { + // console.log(`err: `, err) + + // Rate limited was triggered + res.status(429) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330 + return res.json({ + error: `Too many requests. Your limits are currently ${maxRequests} requests per minute. Increase rate limits at https://account.bchjs.cash` + }) + } + } catch (err) { + wlogger.error(`Error in route-ratelimit.js/rateLimitSimple(): `, err) + // throw err + } + + next() + } + + // Calculates the points consumed, based on the explicit rateLimit defined + // in the JWT token. + calcPoints2(jwtInfo) { + let retVal = 30 // By default, use anonymous tier. + + try { + // console.log(`jwtInfo: ${JSON.stringify(jwtInfo, null, 2)}`) + + const MAX_RATE_LIMIT = 100 + + const rateLimit = jwtInfo.rateLimit + + // Only evaluate if user is using a JWT token. + if (jwtInfo.id) { + const points = Math.floor(MAX_RATE_LIMIT / rateLimit) + + retVal = points + } + } catch (err) { + wlogger.error(`Error in route-ratelimit.js/calcPoints2()`) + // throw err + retVal = 30 + } + + return retVal + } } module.exports = RateLimits diff --git a/test/v3/rate-limits.js b/test/v3/rate-limits.js index 2838e83..2b832a1 100644 --- a/test/v3/rate-limits.js +++ b/test/v3/rate-limits.js @@ -53,6 +53,7 @@ describe("#route-ratelimits & jwt-auth", () => { req.params = {} req.body = {} req.query = {} + req.locals = {} sandbox = sinon.createSandbox() }) @@ -403,6 +404,35 @@ describe("#route-ratelimits & jwt-auth", () => { }) }) + describe("#calcPoints2", () => { + it("should return 30 points for anonymous user", () => { + const result = rateLimits.calcPoints2({}) + // console.log(`result: ${result}`) + + assert.equal(result, 30) + }) + + it("should return 10 point for free tier", () => { + const jwtInfo = { + rateLimit: 10, + id: "5e3a0415eb29a962da2708b4" + } + + const result = rateLimits.calcPoints2(jwtInfo) + assert.equal(result, 10) + }) + + it("should return 1 point for pro tier", () => { + const jwtInfo = { + rateLimit: 100, + id: "5e3a0415eb29a962da2708b4" + } + + const result = rateLimits.calcPoints2(jwtInfo) + assert.equal(result, 1) + }) + }) + describe("#rateLimitByResource", () => { it("should pass through rate-limit middleware", async () => { req.baseUrl = "/v3" @@ -581,6 +611,185 @@ describe("#route-ratelimits & jwt-auth", () => { ) }) }) + + describe("#rateLimitSimple", () => { + it("should pass through rate-limit middleware", async () => { + req.baseUrl = "/v3" + req.path = "/control/getNetworkInfo" + req.url = req.path + req.method = "GET" + + // Call the route twice to trigger the rate handling. + await rateLimits.rateLimitSimple(req, res, next) + await rateLimits.rateLimitSimple(req, res, next) + + // next() will be called if rate-limit is not triggered + assert.equal(next.called, true) + }) + + it("should trigger rate-limit handler if rate limits exceeds 5 request per minute", async () => { + req.baseUrl = "/v3" + req.path = "/control/getNetworkInfo" + req.url = req.path + req.method = "GET" + + for (let i = 0; i < 5; i++) { + next.reset() // reset the stubbed next() function. + + await rateLimits.rateLimitSimple(req, res, next) + //console.log(`next() called: ${next.called}`) + } + + // Note: next() will be called unless the rate-limit kicks in. + assert.equal( + next.called, + false, + `next should not be called if rate limit was triggered.` + ) + }) + + it("should NOT trigger rate-limit for free-tier at 5 RPM", async () => { + // Create a new instance of the rate limit so we start with zeroed tracking. + rateLimits = new RateLimits() + + req.baseUrl = "/v3" + req.path = "/control/getNetworkInfo" + req.url = req.path + req.method = "GET" + + req.locals.jwtToken = "some-token" + + const jwtInfo = { + rateLimit: 10, + id: "5e3a0415eb29a962da2708c1" + } + + // Mock the call to the jwt library. + sandbox.stub(rateLimits.jwt, "verify").returns(jwtInfo) + + for (let i = 0; i < 5; i++) { + next.reset() // reset the stubbed next() function. + + await rateLimits.rateLimitSimple(req, res, next) + //console.log(`next() called: ${next.called}`) + } + + //console.log(`req.locals after test: ${util.inspect(req.locals)}`) + + // Note: next() will be called unless the rate-limit kicks in. + assert.equal( + next.called, + true, + `next should be called if rate limit was not triggered.` + ) + }) + + it("should trigger rate-limit for free tier after 10 RPM", async () => { + // Create a new instance of the rate limit so we start with zeroed tracking. + rateLimits = new RateLimits() + + req.baseUrl = "/v3" + req.path = "/control/getNetworkInfo" + req.url = req.path + req.method = "GET" + + req.locals.jwtToken = "some-token" + + const jwtInfo = { + rateLimit: 10, + id: "5e3a0415eb29a962da2708c2" + } + + // Mock the call to the jwt library. + sandbox.stub(rateLimits.jwt, "verify").returns(jwtInfo) + + for (let i = 0; i < 12; i++) { + next.reset() // reset the stubbed next() function. + + await rateLimits.rateLimitSimple(req, res, next) + //console.log(`next() called: ${next.called}`) + } + + // Note: next() will be called unless the rate-limit kicks in. + assert.equal( + next.called, + false, + `next should not be called if rate limit was triggered.` + ) + }) + + it("should NOT trigger rate-limit handler for pro-tier at 25 RPM", async () => { + // Create a new instance of the rate limit so we start with zeroed tracking. + rateLimits = new RateLimits() + + req.baseUrl = "/v3" + req.path = "/control/getNetworkInfo" + req.url = req.path + req.method = "GET" + + req.locals.jwtToken = "some-token" + + const jwtInfo = { + rateLimit: 100, + id: "5e3a0415eb29a962da2708c3" + } + + // Mock the call to the jwt library. + sandbox.stub(rateLimits.jwt, "verify").returns(jwtInfo) + + for (let i = 0; i < 25; i++) { + next.reset() // reset the stubbed next() function. + + await rateLimits.rateLimitSimple(req, res, next) + //console.log(`next() called: ${next.called}`) + } + + //console.log(`req.locals after test: ${util.inspect(req.locals)}`) + + // Note: next() will be called unless the rate-limit kicks in. + assert.equal( + next.called, + true, + `next should be called if rate limit was not triggered.` + ) + }) + + it("should still rate-limit at a higher RPM for pro-tier", async () => { + // Create a new instance of the rate limit so we start with zeroed tracking. + rateLimits = new RateLimits() + + req.baseUrl = "/v3" + req.path = "/control/getNetworkInfo" + req.url = req.path + req.method = "GET" + + req.locals.jwtToken = "some-token" + + const jwtInfo = { + rateLimit: 100, + id: "5e3a0415eb29a962da2708c4" + } + + // Mock the call to the jwt library. + sandbox.stub(rateLimits.jwt, "verify").returns(jwtInfo) + + for (let i = 0; i < 150; i++) { + next.reset() // reset the stubbed next() function. + + await rateLimits.rateLimitSimple(req, res, next) + //console.log(`next() called: ${next.called}`) + } + + //console.log(`req.locals after test: ${util.inspect(req.locals)}`) + + // Note: next() will be called unless the rate-limit kicks in. + assert.equal( + next.called, + false, + `next should NOT be called if rate limit was triggered.` + ) + }) + }) }) // Generates a Basic authorization header.