From 0aa4d0ecd4f059c6a5092154a171090f6b1b708f Mon Sep 17 00:00:00 2001 From: Chris Troutner Date: Tue, 9 Mar 2021 12:43:30 -0800 Subject: [PATCH] fix(rate limits): Removing old rate limits library --- src/app.js | 16 +- src/middleware/route-ratelimit.js | 550 ++++++++++-------- src/middleware/route-ratelimit2.js | 419 ------------- ...rate-limit2-unit.js => rate-limit-unit.js} | 4 +- test/v4/rate-limits.js | 491 ---------------- 5 files changed, 325 insertions(+), 1155 deletions(-) delete mode 100644 src/middleware/route-ratelimit2.js rename test/v4/{rate-limit2-unit.js => rate-limit-unit.js} (99%) delete mode 100644 test/v4/rate-limits.js diff --git a/src/app.js b/src/app.js index 03e0d1d..cc3300c 100644 --- a/src/app.js +++ b/src/app.js @@ -3,12 +3,8 @@ const express = require('express') // Middleware -// const { routeRateLimit } = require("./middleware/route-ratelimit") -// const RateLimits = require('./middleware/route-ratelimit') -// const rateLimits = new RateLimits() - -const RateLimits2 = require('./middleware/route-ratelimit2') -const rateLimits2 = new RateLimits2() +const RateLimits = require('./middleware/route-ratelimit') +const rateLimits = new RateLimits() const path = require('path') const logger = require('morgan') @@ -102,14 +98,14 @@ const v4prefix = 'v4' // START Rate Limits const auth = new AuthMW() +// Ensure req.locals and res.locals objects exist. +app.use(`/${v4prefix}/`, rateLimits.populateLocals) + // Allow users to turn off rate limits with an environment variable. const DO_NOT_USE_RATE_LIMITS = process.env.DO_NOT_USE_RATE_LIMITS || false console.log(`DO_NOT_USE_RATE_LIMITS: ${DO_NOT_USE_RATE_LIMITS}`) -// Ensure req.locals and res.locals objects exist. -app.use(`/${v4prefix}/`, rateLimits2.populateLocals) - if (!DO_NOT_USE_RATE_LIMITS) { console.log('Rate limits are being used') // Inspect the header for a JWT token. @@ -120,7 +116,7 @@ if (!DO_NOT_USE_RATE_LIMITS) { app.use(`/${v4prefix}/`, auth.mw()) // Experimental rate limits - app.use(`/${v4prefix}/`, rateLimits2.applyRateLimits) + app.use(`/${v4prefix}/`, rateLimits.applyRateLimits) // Rate limit on all v4 routes // Establish and enforce rate limits. diff --git a/src/middleware/route-ratelimit.js b/src/middleware/route-ratelimit.js index 722a970..0745523 100644 --- a/src/middleware/route-ratelimit.js +++ b/src/middleware/route-ratelimit.js @@ -1,63 +1,61 @@ /* - Sets the rate limits for the anonymous and paid tiers. Current rate limits: - - 1000 points in 60 seconds - - 10 points per call for paid tier (100 RPM) - - 50 points per call for anonymous tier (20 RPM) +This file will replace the original rate-limit.js file. - Background: - The rate limits below were originially coded with the idea of charging on a - per-resource basis. However, that was confusing to end users trying to purchase - a subscription. So everything was simplied to two tiers: paid and anonymous +Sets the rate limits for the anonymous and paid tiers. Current rate limits: +- 1000 points in 60 seconds +- 10 points per call for paid tier (100 RPM) +- 50 points per call for anonymous tier (20 RPM) - CT 3/7/21 - This rate limits have been refactored to consider the following use cases: - - Users who want to buy a JWT token for 24 hour access. - - Users who want to buy different RPM tiers: 100, 250, 600 - - Basic Authentication which should not have any rate limits applied. - - Users that run bch-api locally and do not want any rate limits applied. +The rate limit handling is designed for these four use cases: +- Users who want to buy a JWT token for 24 hour access. +- Users who want to buy different RPM tiers: 100, 250, 600 +- Basic Authentication which should not have any rate limits applied. +- Local installations that do not want any authentication or rate limits at all. - The second two use cases also apply to internal rate limits. The internal rate - limits should not be applied to calls from those users. +The Basic Auth use cases is considered when determining internal rate limits. +The internal rate limits should not be applied to calls from those users. - A lot of attention has been paid to passing rate-limit information for the user - when they trigger an endpoint that makes a lot of internal API calls. Examples - are hydrateUtxos() and getPublicKey(). +A lot of attention has been paid to passing rate-limit information for the user +when they trigger an endpoint that makes a lot of internal API calls. Examples +are hydrateUtxos() and getPublicKey(). These keeps things fair by charging the +same for 'light' API calls and 'heavy' API calls. + +TODO: +- Add code for applying rate limits to whitelist domains. */ -'use strict' - // Public npm libraries. const jwt = require('jsonwebtoken') +const Redis = require('ioredis') +const { RateLimiterRedis } = require('rate-limiter-flexible') // local libraries. const wlogger = require('../util/winston-logging') const config = require('../../config') -const ANON_LIMITS = config.anonRateLimit -const WHITELIST_RATE_LIMIT = config.whitelistRateLimit -const WHITELIST_DOMAINS = config.whitelistDomains -const INTERNAL_RATE_LIMIT = 1 +let _this // Global pointer to instance of class, when 'this' context is lost. -// Redis +// Setup Redis to track rate limits for each user. const redisOptions = { enableOfflineQueue: false, port: process.env.REDIS_PORT ? process.env.REDIS_PORT : 6379, host: process.env.REDIS_HOST ? process.env.REDIS_HOST : '127.0.0.1' } console.log(`redisOptions: ${JSON.stringify(redisOptions, null, 2)}`) - -const Redis = require('ioredis') const redisClient = new Redis(redisOptions) - -// Rate limiter middleware lib. -const { RateLimiterRedis } = require('rate-limiter-flexible') const rateLimitOptions = { storeClient: redisClient, points: 1000, // Number of points duration: 60 // Per minute (per 60 seconds) } -let _this +// Constants +const ANON_LIMITS = config.anonRateLimit +// const WHITELIST_RATE_LIMIT = config.whitelistRateLimit +const WHITELIST_DOMAINS = config.whitelistDomains +const WHITELIST_POINTS_TO_CONSUME = config.whitelistRateLimit +const POINTS_PER_MINUTE = config.pointsPerMinute +const INTERNAL_POINTS_TO_CONSUME = 1 class RateLimits { constructor () { @@ -68,248 +66,249 @@ class RateLimits { this.config = config } - // Used to disconnect from the Redis DB. - // Called by unit tests so that node.js thread doesn't live forever. - closeRedis () { - redisClient.disconnect() - } - - async wipeRedis () { - await redisClient.flushdb() - } - - // This is the new rate limit function that uses the rate-limiter-flexible npm - // library. It uses fine-grain rate limiting based on the resources being - // consumed. - async rateLimitByResource (req, res, next) { + // This is the main middleware funciton of this library. All other functions + // support this function. + async applyRateLimits (req, res, next) { try { - let userId - let decoded = {} - - // Create a req.locals object if not passed in. - if (!req.locals) { - req.locals = { - // default values - jwtToken: '', - proLimit: false, - apiLevel: 0 - } + // Exit if the user has already authenticated with Basic Authentication. + if (req.locals.proLimit) { + console.log('External call, basic auth, skipping rate limits.') + wlogger.debug( + 'req.locals.proLimit = true; Using Basic Authentication instead of rate limits' + ) + return next() } - // Create a res.locals object if it does not exist. This is used for - // debugging. - if (!res.locals) { - res.locals = { - rateLimitTriggered: false - } - } + // Determine if the call is an external or internal API call. + const isInternal = _this.checkInternalIp(req) + console.log(`isInternal: ${isInternal}`) - // Decode the JWT token if one exists. - if (req.locals.jwtToken) { - try { - decoded = _this.jwt.verify( - req.locals.jwtToken, - _this.config.apiTokenSecret - ) - // console.log(`decoded: ${JSON.stringify(decoded, null, 2)}`) + // Determine if the call originates from another computer on the intranet. + const isWhitelistOrigin = _this.isInWhitelist(req) + console.log('isWhitelistOrigin: ', isWhitelistOrigin) - userId = decoded.id - } catch (err) { - // This handler will be triggered if the JWT token does not match the - // token secret. - wlogger.error( - `Last three letters of token secret: ${_this.config.apiTokenSecret.slice( - -3 - )}` - ) - wlogger.error( - 'Error trying to decode JWT token in route-ratelimit.js/newRateLimit(): ', - err + // Handle the use case of internally-generated requests. + if (isInternal) { + // Internal API calls should pass the authentication data in through the + // the usrObj in the body. + if (req.body && req.body.usrObj) { + if (req.body.usrObj.proLimit) { + console.log('Internal call, basic auth, skipping rate limits.') + + // If this is an internal call that originated from a user using + // Basic Authentication, then skip rate-limits. + return next() + } else { + console.log( + 'Internal call, applying rate limits. Using JWT if available.' + ) + + // Determine if user has exceeded their rate limits. Pass in the + // JWT token if one exists. + const hasExceededRateLimit = await _this.trackRateLimits( + req, + res, + req.body.usrObj.jwtToken + ) + + if (!hasExceededRateLimit) { + // Rate limits have not been exceeded. Processing can continue. + return next() + } else { + // trackRateLimits() returns the 'res' object with an error message + // and status code. + return hasExceededRateLimit + } + } + } else { + // This should be a corner case. Calls should not be going into this + // code path, so the system should throw up big warning signs when they + // do. + // This code path happens when an internal call is made but does not + // pass the usrObj. Legacy code needs to be refactored to use the usrObj + // and avoid this code path. This code path is 'pooled': all users + // share the same rate limits. Even at 1000 RPM, this pool will get + // exhausted easily. + const warnMsg = + 'Internal call. req.body.usrObj does not exist. Applying high-speed internal rate limits.' + console.log(warnMsg) + wlogger.info(warnMsg) + + const defaultPayload = { + id: '98.76.54.32', + email: 'internal@bchtest.net', + apiLevel: 40, + rateLimit: 100, + pointsToConsume: INTERNAL_POINTS_TO_CONSUME, + duration: 30 + } + + // Default values, in case there is an error. + const defaultJwt = _this.generateJwtToken(defaultPayload) + + // Track the rate limit for this user. Pass in the JWT token, if one + // is available. + const hasExceededRateLimit = await _this.trackRateLimits( + req, + res, + defaultJwt ) + + if (!hasExceededRateLimit) { + // Rate limits have not been exceeded. Processing can continue. + return next() + } else { + // trackRateLimits() returns the 'res' object with an error message + // and status code. + return hasExceededRateLimit + } } // - } else if (req.body && req.body.usrObj) { - // Same as above, but this code path is activated from internal calls to - // bch-js, like hydrateUtxo(), which passes the user object from the - // original API call. - - try { - decoded = _this.jwt.verify( - req.body.usrObj.jwtToken, - _this.config.apiTokenSecret - ) - // console.log(`decoded: ${JSON.stringify(decoded, null, 2)}`) - - userId = decoded.id - } catch (err) { - // This handler will be triggered if the JWT token does not match the - // token secret. - wlogger.error( - 'Error in route-ratelimit.js trying to decode JWT token in usrObj' - ) - } + // } else { - wlogger.debug('No JWT token found!') - } + // Handle the normal use-case of external requests + console.log( + 'External call, applying rate limits. Using JWT if available.' + ) - // Default value is 50 points per request = 20 RPM - let rateLimit = ANON_LIMITS - - // Only evaluate the JWT token if the user is not using Basic Authentication. - if (!req.locals.proLimit) { - // Code here for the rate limiter is adapted from this example: - // https://github.com/animir/node-rate-limiter-flexible/wiki/Overall-example#authorized-and-not-authorized-users - try { - // The resource being consumed: full node, indexer, SLPDB, etc. - const resource = _this.getResource(req.url) - wlogger.debug(`resource: ${resource}`) - - // Key will be the JWT ID if it exists, otherwise the IP address of the caller. - let key = userId || req.ip - res.locals.key = key // Feedback for tests. - // console.log(`key: ${key}`) - - // const pointsToConsume = userId ? 1 : 30 - decoded.resource = resource - let pointsToConsume = _this.calcPoints(decoded) - res.locals.pointsToConsume = pointsToConsume // Feedback for tests. - - // Retrieve the origin. - let origin = req.get('origin') - - // Handle calls coming from the intranet. - if (origin === undefined && key.indexOf('10.0.0.5') > -1) { - origin = 'slp-api' + // For calls originating from a whitelist domain, apply a high-RPM + // JWT token to the call. + if (isWhitelistOrigin) { + const defaultPayload = { + id: '77.77.77.77', + email: 'whitelist@bchtest.net', + apiLevel: 40, + rateLimit: 100, + pointsToConsume: WHITELIST_POINTS_TO_CONSUME, + duration: 30 } - wlogger.info(`origin: ${origin}`) + // Inject the high-RPM JWT token into the call. + req.locals.jwtToken = _this.generateJwtToken(defaultPayload) + } - // If the request originates from one of the approved wallet apps, then - // apply paid-access rate limits. - // console.log(`origin: ${JSON.stringify(origin, null, 2)}`) - // console.log(`whitelist: ${JSON.stringify(WHITELIST_DOMAINS, null, 2)}`) - const isInWhitelist = _this.isInWhitelist(origin) - if (isInWhitelist) { - pointsToConsume = WHITELIST_RATE_LIMIT - res.locals.pointsToConsume = pointsToConsume // Feedback for tests. - } + // Track the rate limit for this user. Pass in the JWT token, if one + // is available. + const hasExceededRateLimit = await _this.trackRateLimits( + req, + res, + req.locals.jwtToken + ) - // For internal calls, increase rate limits to as fast as possible. - if ( - // Comment out the line below when running bch-js e2e rate limit tests. - key.toString().indexOf('::ffff:127.0.0.1') > -1 || - // Do not comment out this line. - key.toString().indexOf('172.17.') > -1 - ) { - pointsToConsume = INTERNAL_RATE_LIMIT - res.locals.pointsToConsume = pointsToConsume // Feedback for tests. - } - - wlogger.info( - `User ${key} consuming ${pointsToConsume} point for resource ${resource}.` - ) - - rateLimit = Math.floor(1000 / pointsToConsume) - - // Update the key so that rate limits track both the user and the resource. - key = `${key}-${resource}` - - await _this.rateLimiter.consume(key, pointsToConsume) - } catch (err) { - // console.log('err: ', err) - - // Used for returning data for tests. - res.locals.rateLimitTriggered = true - // console.log('res.locals: ', res.locals) - - // Rate limited was triggered - res.status(429) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330 - return res.json({ - error: `Too many requests. Your limits are currently ${rateLimit} requests per minute. Increase rate limits at https://fullstack.cash` - }) + if (!hasExceededRateLimit) { + // Rate limits have not been exceeded. Processing can continue. + return next() + } else { + // trackRateLimits() returns the 'res' object with an error message + // and status code. + return hasExceededRateLimit } } } catch (err) { - wlogger.error('Error in route-ratelimit.js/newRateLimit(): ', err) - // throw err + wlogger.error('Error in route-ratelimit2.js/applyRateLimits(): ', err) } + // By default, move to the next middleware. next() } - // Calculates the points consumed, based on the jwt information and the route - // requested. - calcPoints (jwtInfo) { - let retVal = ANON_LIMITS // By default, use anonymous tier. + // A wrapper for Redis-based rate limiter. + // Will return false if the user has not exceeded the rate limit. Otherwise + // it will return the 'res' object with an error status and message, which + // should be returned by the middleware. + async trackRateLimits (req, res, jwtToken) { + // Anonymous rate limits are used by default. + let pointsToConsume = ANON_LIMITS + let key = req.ip // Use the IP address as the key, by default. try { - // console.log(`jwtInfo: ${JSON.stringify(jwtInfo, null, 2)}`) + // Decode the JWT token if it exists + if (jwtToken) { + const decoded = _this.decodeJwtToken(jwtToken) + // console.log(`decoded: ${JSON.stringify(decoded, null, 2)}`) - const apiLevel = jwtInfo.apiLevel - const resource = jwtInfo.resource + // Preferentially use the decoded ID in the JWT payload, as the key. + key = decoded.id - const level30Routes = ['insight', 'bitcore', 'blockbook', 'electrumx'] - const level40Routes = ['slp'] - - wlogger.debug(`apiLevel: ${apiLevel}`) - - // Only evaluate if user is using a JWT token. - if (jwtInfo.id) { - // SLP indexer routes - if (level40Routes.includes(resource)) { - if (apiLevel >= 40) retVal = 10 - // else if (apiLevel >= 10) retVal = 10 - else retVal = ANON_LIMITS - - // Normal indexer routes - } else if (level30Routes.includes(resource)) { - if (apiLevel >= 30) retVal = 10 - else retVal = ANON_LIMITS - - // Full node tier - } else if (apiLevel >= 20) { - retVal = 10 - - // Free tier, full node only. - } else { - retVal = ANON_LIMITS - } + pointsToConsume = decoded.pointsToConsume } + console.log(`rate limit key: ${key}`) - return retVal + // This function will throw an error if the user exceeds the rate limit. + // The 429 error response is handled by the catch(). + await _this.rateLimiter.consume(key, pointsToConsume) + + res.locals.pointsToConsume = pointsToConsume // Feedback for tests. + + // Signal that the user has not exceeded their rate limits. + return false } catch (err) { - wlogger.error('Error in route-ratelimit.js/calcPoints()') - // throw err - retVal = ANON_LIMITS - } + console.log('err: ', err) - return retVal + const rateLimit = Math.floor(POINTS_PER_MINUTE / pointsToConsume) + + res.locals.rateLimitTriggered = true + // console.log('res.locals: ', res.locals) + + // Rate limited was triggered + res.status(429) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330 + return res.json({ + error: `Too many requests. Your limits are currently ${rateLimit} requests per minute. Increase rate limits at https://fullstack.cash` + }) + } } - // This function parses the req.url property to identify what resource - // the user is requesting. - // This was created as a function so that it can be unit tested. Not sure - // what kind of variations will be seen in production. - getResource (url) { + // Attempts to decode a JWT token. Returns default values if it fails. + decodeJwtToken (jwtToken) { + const defaultPayload = { + id: '123.456.789.10', + email: 'test@bchtest.net', + apiLevel: 10, + rateLimit: 3, + pointsToConsume: ANON_LIMITS, + duration: 30 + } + try { - wlogger.debug(`url: ${JSON.stringify(url, null, 2)}`) + // Default values, in case there is an error. + const defaultJwt = _this.generateJwtToken(defaultPayload) - const splitUrl = url.split('/') - const resource = splitUrl[1] + // Generate a default payload to use, if the decoding of the user-provided + // jwt fails. + let decoded = _this.jwt.verify(defaultJwt, _this.config.apiTokenSecret) - return resource + try { + decoded = _this.jwt.verify(jwtToken, _this.config.apiTokenSecret) + } catch (err) { + wlogger.error('Error in route-ratelimit2.js/decodeJwtTokens(): ', err) + } + + return decoded } catch (err) { - wlogger.error('Error in getResource().') - throw err + wlogger.error( + 'Unhandled error in route-ratelimit2.js/deocdeJwtToken: ', + err + ) + + // Making sure there is an exp property. Not sure if this will cause an + // issue, using a hard-coded value. + defaultPayload.exp = 1574269450 + + return defaultPayload } } // Returns a boolean if the origin of the request matches a domain in the // whitelist. - isInWhitelist (origin) { + isInWhitelist (req) { try { const retVal = false // Default value. + // Retrieve the origin. + const origin = req.get('origin') + console.log(`origin: ${origin}`) + + // If the origin is not determinable, return false. if (!origin) return false // console.log(`WHITELIST_DOMAINS: ${JSON.stringify(WHITELIST_DOMAINS, null, 2)}`) @@ -317,9 +316,7 @@ class RateLimits { for (let i = 0; i < WHITELIST_DOMAINS.length; i++) { const thisDomain = WHITELIST_DOMAINS[i] - if (origin.toString().indexOf(thisDomain) > -1) { - return true - } + if (origin.includes(thisDomain)) return true } return retVal @@ -330,6 +327,93 @@ class RateLimits { return false } } + + // Checks the request object to see if it's IP address matches an internal + // IP address. That means the call is an internal API call and should be + // treated differently than an external API call. + checkInternalIp (req) { + try { + // Default value + let isInternal = false + + const ip = req.ip + + if (ip.includes('127.0.0.1')) isInternal = true + + if (ip.includes('172.17.')) isInternal = true + + // TODO: Add 192.168. + + return isInternal + } catch (err) { + wlogger.error( + 'Error in checkInternalIp(). Returning false be default. Err: ', + err + ) + return false + } + } + + // Used to disconnect from the Redis DB. + // Called by unit tests so that node.js thread doesn't live forever. + closeRedis () { + redisClient.disconnect() + } + + // Clear the redis database. Used by unit tests. + async wipeRedis () { + await redisClient.flushdb() + } + + // Generates a JWT token for testing purposes. This is not used in production. + // This function mirrors the kind of JWT token that would be generated by + // jwt-bch-api. + generateJwtToken (payload) { + try { + const jwtOptions = { + expiresIn: '30 days' + } + + const token = _this.jwt.sign( + payload, + _this.config.apiTokenSecret, + jwtOptions + ) + + return token + } catch (err) { + console.error('Error in generateJwtToken()') + throw err + } + } + + // Called when rate limits are not used. + populateLocals (req, res, next) { + try { + // Create a re*Q*.locals object if not passed in. + // req.locals.proLimit will be true if the user is using Basic Authentication. + if (!req.locals) { + req.locals = { + // default values + jwtToken: '', + proLimit: false, + apiLevel: 0 + } + } + + // Create a re*S*.locals object if it does not exist. + if (!res.locals) { + res.locals = { + rateLimitTriggered: false + } + } + + next() + } catch (err) { + console.error('Error in populateLocals(): ', err) + throw err + } + } } module.exports = RateLimits diff --git a/src/middleware/route-ratelimit2.js b/src/middleware/route-ratelimit2.js deleted file mode 100644 index 0745523..0000000 --- a/src/middleware/route-ratelimit2.js +++ /dev/null @@ -1,419 +0,0 @@ -/* -This file will replace the original rate-limit.js file. - -Sets the rate limits for the anonymous and paid tiers. Current rate limits: -- 1000 points in 60 seconds -- 10 points per call for paid tier (100 RPM) -- 50 points per call for anonymous tier (20 RPM) - -The rate limit handling is designed for these four use cases: -- Users who want to buy a JWT token for 24 hour access. -- Users who want to buy different RPM tiers: 100, 250, 600 -- Basic Authentication which should not have any rate limits applied. -- Local installations that do not want any authentication or rate limits at all. - -The Basic Auth use cases is considered when determining internal rate limits. -The internal rate limits should not be applied to calls from those users. - -A lot of attention has been paid to passing rate-limit information for the user -when they trigger an endpoint that makes a lot of internal API calls. Examples -are hydrateUtxos() and getPublicKey(). These keeps things fair by charging the -same for 'light' API calls and 'heavy' API calls. - -TODO: -- Add code for applying rate limits to whitelist domains. -*/ - -// Public npm libraries. -const jwt = require('jsonwebtoken') -const Redis = require('ioredis') -const { RateLimiterRedis } = require('rate-limiter-flexible') - -// local libraries. -const wlogger = require('../util/winston-logging') -const config = require('../../config') - -let _this // Global pointer to instance of class, when 'this' context is lost. - -// Setup Redis to track rate limits for each user. -const redisOptions = { - enableOfflineQueue: false, - port: process.env.REDIS_PORT ? process.env.REDIS_PORT : 6379, - host: process.env.REDIS_HOST ? process.env.REDIS_HOST : '127.0.0.1' -} -console.log(`redisOptions: ${JSON.stringify(redisOptions, null, 2)}`) -const redisClient = new Redis(redisOptions) -const rateLimitOptions = { - storeClient: redisClient, - points: 1000, // Number of points - duration: 60 // Per minute (per 60 seconds) -} - -// Constants -const ANON_LIMITS = config.anonRateLimit -// const WHITELIST_RATE_LIMIT = config.whitelistRateLimit -const WHITELIST_DOMAINS = config.whitelistDomains -const WHITELIST_POINTS_TO_CONSUME = config.whitelistRateLimit -const POINTS_PER_MINUTE = config.pointsPerMinute -const INTERNAL_POINTS_TO_CONSUME = 1 - -class RateLimits { - constructor () { - _this = this - - this.jwt = jwt - this.rateLimiter = new RateLimiterRedis(rateLimitOptions) - this.config = config - } - - // This is the main middleware funciton of this library. All other functions - // support this function. - async applyRateLimits (req, res, next) { - try { - // Exit if the user has already authenticated with Basic Authentication. - if (req.locals.proLimit) { - console.log('External call, basic auth, skipping rate limits.') - wlogger.debug( - 'req.locals.proLimit = true; Using Basic Authentication instead of rate limits' - ) - return next() - } - - // Determine if the call is an external or internal API call. - const isInternal = _this.checkInternalIp(req) - console.log(`isInternal: ${isInternal}`) - - // Determine if the call originates from another computer on the intranet. - const isWhitelistOrigin = _this.isInWhitelist(req) - console.log('isWhitelistOrigin: ', isWhitelistOrigin) - - // Handle the use case of internally-generated requests. - if (isInternal) { - // Internal API calls should pass the authentication data in through the - // the usrObj in the body. - if (req.body && req.body.usrObj) { - if (req.body.usrObj.proLimit) { - console.log('Internal call, basic auth, skipping rate limits.') - - // If this is an internal call that originated from a user using - // Basic Authentication, then skip rate-limits. - return next() - } else { - console.log( - 'Internal call, applying rate limits. Using JWT if available.' - ) - - // Determine if user has exceeded their rate limits. Pass in the - // JWT token if one exists. - const hasExceededRateLimit = await _this.trackRateLimits( - req, - res, - req.body.usrObj.jwtToken - ) - - if (!hasExceededRateLimit) { - // Rate limits have not been exceeded. Processing can continue. - return next() - } else { - // trackRateLimits() returns the 'res' object with an error message - // and status code. - return hasExceededRateLimit - } - } - } else { - // This should be a corner case. Calls should not be going into this - // code path, so the system should throw up big warning signs when they - // do. - // This code path happens when an internal call is made but does not - // pass the usrObj. Legacy code needs to be refactored to use the usrObj - // and avoid this code path. This code path is 'pooled': all users - // share the same rate limits. Even at 1000 RPM, this pool will get - // exhausted easily. - const warnMsg = - 'Internal call. req.body.usrObj does not exist. Applying high-speed internal rate limits.' - console.log(warnMsg) - wlogger.info(warnMsg) - - const defaultPayload = { - id: '98.76.54.32', - email: 'internal@bchtest.net', - apiLevel: 40, - rateLimit: 100, - pointsToConsume: INTERNAL_POINTS_TO_CONSUME, - duration: 30 - } - - // Default values, in case there is an error. - const defaultJwt = _this.generateJwtToken(defaultPayload) - - // Track the rate limit for this user. Pass in the JWT token, if one - // is available. - const hasExceededRateLimit = await _this.trackRateLimits( - req, - res, - defaultJwt - ) - - if (!hasExceededRateLimit) { - // Rate limits have not been exceeded. Processing can continue. - return next() - } else { - // trackRateLimits() returns the 'res' object with an error message - // and status code. - return hasExceededRateLimit - } - } - // - // - } else { - // Handle the normal use-case of external requests - console.log( - 'External call, applying rate limits. Using JWT if available.' - ) - - // For calls originating from a whitelist domain, apply a high-RPM - // JWT token to the call. - if (isWhitelistOrigin) { - const defaultPayload = { - id: '77.77.77.77', - email: 'whitelist@bchtest.net', - apiLevel: 40, - rateLimit: 100, - pointsToConsume: WHITELIST_POINTS_TO_CONSUME, - duration: 30 - } - - // Inject the high-RPM JWT token into the call. - req.locals.jwtToken = _this.generateJwtToken(defaultPayload) - } - - // Track the rate limit for this user. Pass in the JWT token, if one - // is available. - const hasExceededRateLimit = await _this.trackRateLimits( - req, - res, - req.locals.jwtToken - ) - - if (!hasExceededRateLimit) { - // Rate limits have not been exceeded. Processing can continue. - return next() - } else { - // trackRateLimits() returns the 'res' object with an error message - // and status code. - return hasExceededRateLimit - } - } - } catch (err) { - wlogger.error('Error in route-ratelimit2.js/applyRateLimits(): ', err) - } - - // By default, move to the next middleware. - next() - } - - // A wrapper for Redis-based rate limiter. - // Will return false if the user has not exceeded the rate limit. Otherwise - // it will return the 'res' object with an error status and message, which - // should be returned by the middleware. - async trackRateLimits (req, res, jwtToken) { - // Anonymous rate limits are used by default. - let pointsToConsume = ANON_LIMITS - let key = req.ip // Use the IP address as the key, by default. - - try { - // Decode the JWT token if it exists - if (jwtToken) { - const decoded = _this.decodeJwtToken(jwtToken) - // console.log(`decoded: ${JSON.stringify(decoded, null, 2)}`) - - // Preferentially use the decoded ID in the JWT payload, as the key. - key = decoded.id - - pointsToConsume = decoded.pointsToConsume - } - console.log(`rate limit key: ${key}`) - - // This function will throw an error if the user exceeds the rate limit. - // The 429 error response is handled by the catch(). - await _this.rateLimiter.consume(key, pointsToConsume) - - res.locals.pointsToConsume = pointsToConsume // Feedback for tests. - - // Signal that the user has not exceeded their rate limits. - return false - } catch (err) { - console.log('err: ', err) - - const rateLimit = Math.floor(POINTS_PER_MINUTE / pointsToConsume) - - res.locals.rateLimitTriggered = true - // console.log('res.locals: ', res.locals) - - // Rate limited was triggered - res.status(429) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330 - return res.json({ - error: `Too many requests. Your limits are currently ${rateLimit} requests per minute. Increase rate limits at https://fullstack.cash` - }) - } - } - - // Attempts to decode a JWT token. Returns default values if it fails. - decodeJwtToken (jwtToken) { - const defaultPayload = { - id: '123.456.789.10', - email: 'test@bchtest.net', - apiLevel: 10, - rateLimit: 3, - pointsToConsume: ANON_LIMITS, - duration: 30 - } - - try { - // Default values, in case there is an error. - const defaultJwt = _this.generateJwtToken(defaultPayload) - - // Generate a default payload to use, if the decoding of the user-provided - // jwt fails. - let decoded = _this.jwt.verify(defaultJwt, _this.config.apiTokenSecret) - - try { - decoded = _this.jwt.verify(jwtToken, _this.config.apiTokenSecret) - } catch (err) { - wlogger.error('Error in route-ratelimit2.js/decodeJwtTokens(): ', err) - } - - return decoded - } catch (err) { - wlogger.error( - 'Unhandled error in route-ratelimit2.js/deocdeJwtToken: ', - err - ) - - // Making sure there is an exp property. Not sure if this will cause an - // issue, using a hard-coded value. - defaultPayload.exp = 1574269450 - - return defaultPayload - } - } - - // Returns a boolean if the origin of the request matches a domain in the - // whitelist. - isInWhitelist (req) { - try { - const retVal = false // Default value. - - // Retrieve the origin. - const origin = req.get('origin') - console.log(`origin: ${origin}`) - - // If the origin is not determinable, return false. - if (!origin) return false - - // console.log(`WHITELIST_DOMAINS: ${JSON.stringify(WHITELIST_DOMAINS, null, 2)}`) - - for (let i = 0; i < WHITELIST_DOMAINS.length; i++) { - const thisDomain = WHITELIST_DOMAINS[i] - - if (origin.includes(thisDomain)) return true - } - - return retVal - } catch (err) { - wlogger.error( - 'Error in route-ratelimit.js/isInWhitelist(). Returning false by default.' - ) - return false - } - } - - // Checks the request object to see if it's IP address matches an internal - // IP address. That means the call is an internal API call and should be - // treated differently than an external API call. - checkInternalIp (req) { - try { - // Default value - let isInternal = false - - const ip = req.ip - - if (ip.includes('127.0.0.1')) isInternal = true - - if (ip.includes('172.17.')) isInternal = true - - // TODO: Add 192.168. - - return isInternal - } catch (err) { - wlogger.error( - 'Error in checkInternalIp(). Returning false be default. Err: ', - err - ) - return false - } - } - - // Used to disconnect from the Redis DB. - // Called by unit tests so that node.js thread doesn't live forever. - closeRedis () { - redisClient.disconnect() - } - - // Clear the redis database. Used by unit tests. - async wipeRedis () { - await redisClient.flushdb() - } - - // Generates a JWT token for testing purposes. This is not used in production. - // This function mirrors the kind of JWT token that would be generated by - // jwt-bch-api. - generateJwtToken (payload) { - try { - const jwtOptions = { - expiresIn: '30 days' - } - - const token = _this.jwt.sign( - payload, - _this.config.apiTokenSecret, - jwtOptions - ) - - return token - } catch (err) { - console.error('Error in generateJwtToken()') - throw err - } - } - - // Called when rate limits are not used. - populateLocals (req, res, next) { - try { - // Create a re*Q*.locals object if not passed in. - // req.locals.proLimit will be true if the user is using Basic Authentication. - if (!req.locals) { - req.locals = { - // default values - jwtToken: '', - proLimit: false, - apiLevel: 0 - } - } - - // Create a re*S*.locals object if it does not exist. - if (!res.locals) { - res.locals = { - rateLimitTriggered: false - } - } - - next() - } catch (err) { - console.error('Error in populateLocals(): ', err) - throw err - } - } -} - -module.exports = RateLimits diff --git a/test/v4/rate-limit2-unit.js b/test/v4/rate-limit-unit.js similarity index 99% rename from test/v4/rate-limit2-unit.js rename to test/v4/rate-limit-unit.js index 77e0fc5..4c06e5b 100644 --- a/test/v4/rate-limit2-unit.js +++ b/test/v4/rate-limit-unit.js @@ -15,12 +15,12 @@ const config = require('../../config') const { mockReq, mockRes, mockNext } = require('./mocks/express-mocks') // Libraries under test -const RateLimits = require('../../src/middleware/route-ratelimit2') +const RateLimits = require('../../src/middleware/route-ratelimit') let uut = new RateLimits() let req, res, next -describe('#rate-routelimit2', () => { +describe('#rate-routelimit', () => { let sandbox before(async () => { diff --git a/test/v4/rate-limits.js b/test/v4/rate-limits.js deleted file mode 100644 index c79068e..0000000 --- a/test/v4/rate-limits.js +++ /dev/null @@ -1,491 +0,0 @@ -/* - Unit tests for the rate limit middleware. -*/ - -'use strict' - -const chai = require('chai') -const assert = chai.assert -const sinon = require('sinon') - -// Used for debugging. -const util = require('util') -util.inspect.defaultOptions = { depth: 1 } - -// Mocking data. -const { mockReq, mockRes, mockNext } = require('./mocks/express-mocks') - -// Libraries under test -const RateLimits = require('../../src/middleware/route-ratelimit') -let rateLimits = new RateLimits() - -// const controlRoute = require('../../src/routes/v4/full-node/control') -const jwtAuth = require('../../src/middleware/jwt-auth') - -let req, res, next -// let originalEnvVars // Used during transition from integration to unit tests. - -// JWT token used in tests. -const jwt = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6IjVkYWRlM2Y1NzM5ZTZjMGZmMDM0YjlhMSIsImlhdCI6MTU3MTY3NzQ1MCwiZXhwIjoxNTc0MjY5NDUwfQ.SSz7F7ETyBB3eoNG2VKCzPOhddtB-vrtmEoj7PxicrQ' - -describe('#route-ratelimits & jwt-auth', () => { - let sandbox - - before(async () => { - // Save existing environment variables. - // originalEnvVars = { - // BITCOINCOM_BASEURL: process.env.BITCOINCOM_BASEURL, - // RPC_BASEURL: process.env.RPC_BASEURL, - // RPC_USERNAME: process.env.RPC_USERNAME, - // RPC_PASSWORD: process.env.RPC_PASSWORD - // } - - if (!process.env.JWT_AUTH_SERVER) { process.env.JWT_AUTH_SERVER = 'http://fakeurl.com/' } - - // Wipe the Redis DB, which prevents false negatives when running integration - // tests back-to-back. - await rateLimits.wipeRedis() - }) - - // Setup the mocks before each test. - beforeEach(() => { - // Mock the req and res objects used by Express routes. - req = Object.assign({}, mockReq) - res = Object.assign({}, mockRes) - next = mockNext - - // Explicitly reset the parmas and body. - req.params = {} - req.body = {} - req.query = {} - req.locals = {} - - sandbox = sinon.createSandbox() - }) - - afterEach(() => { - sandbox.restore() - }) - - after(() => { - rateLimits.closeRedis() - }) - - describe('#jwt-auth.js', () => { - describe('#getTokenFromHeaders', () => { - it('should populate the req.locals object correctly', () => { - // Initialize req.locals - req.locals = { - proLimit: false, - apiLevel: 0 - } - - const header = `Token ${jwt}` - req.headers.authorization = header - - jwtAuth.getTokenFromHeaders(req, res, next) - - // console.log(`req.locals: ${JSON.stringify(req.locals, null, 2)}`) - - assert.property(req.locals, 'proLimit') - assert.property(req.locals, 'apiLevel') - assert.property(req.locals, 'jwtToken') - assert.equal(req.locals.jwtToken, jwt) - }) - }) - }) - - describe('#getResource', () => { - it('should decode a blockchain request', () => { - const url = - '/blockchain/getTxOut/62a3ea958a463a372bc0caf2c374a7f60be9c624be63a0db8db78f05809df6d8/0?include_mempool=true' - - const result = rateLimits.getResource(url) - // console.log(`result: ${JSON.stringify(result, null, 2)}`) - - assert.equal(result, 'blockchain') - }) - }) - - describe('#calcPoints', () => { - it('should return 50 points for anonymous user', () => { - const result = rateLimits.calcPoints() - // console.log(`result: ${result}`) - - assert.equal(result, 50) - }) - - it('should return 50 points for free tier requesting full node access', () => { - const jwtInfo = { - apiLevel: 10, - resource: 'blockchain', - id: '5e3a0415eb29a962da2708b4' - } - - const result = rateLimits.calcPoints(jwtInfo) - assert.equal(result, 50) - }) - - it('should return 50 points for free tier requesting indexer access', () => { - const jwtInfo = { - apiLevel: 10, - resource: 'blockbook', - id: '5e3a0415eb29a962da2708b4' - } - - const result = rateLimits.calcPoints(jwtInfo) - assert.equal(result, 50) - }) - - it('should return 50 points for free tier requesting SLPDB access', () => { - const jwtInfo = { - apiLevel: 10, - resource: 'slp', - id: '5e3a0415eb29a962da2708b4' - } - - const result = rateLimits.calcPoints(jwtInfo) - assert.equal(result, 50) - }) - - it('should return 10 points for full node tier requesting full node access', () => { - const jwtInfo = { - apiLevel: 20, - resource: 'blockchain', - id: '5e3a0415eb29a962da2708b4' - } - - const result = rateLimits.calcPoints(jwtInfo) - assert.equal(result, 10) - }) - - it('should return 50 points for full-node tier requesting indexer access', () => { - const jwtInfo = { - apiLevel: 20, - resource: 'blockbook', - id: '5e3a0415eb29a962da2708b4' - } - - const result = rateLimits.calcPoints(jwtInfo) - assert.equal(result, 50) - }) - - it('should return 50 points for full node tier requesting SLPDB access', () => { - const jwtInfo = { - apiLevel: 20, - resource: 'slp', - id: '5e3a0415eb29a962da2708b4' - } - - const result = rateLimits.calcPoints(jwtInfo) - assert.equal(result, 50) - }) - - it('should return 10 point for indexer tier requesting full node access', () => { - const jwtInfo = { - apiLevel: 30, - resource: 'blockchain', - id: '5e3a0415eb29a962da2708b4' - } - - const result = rateLimits.calcPoints(jwtInfo) - assert.equal(result, 10) - }) - - it('should return 10 points for indexer tier requesting indexer access', () => { - const jwtInfo = { - apiLevel: 30, - resource: 'blockbook', - id: '5e3a0415eb29a962da2708b4' - } - - const result = rateLimits.calcPoints(jwtInfo) - assert.equal(result, 10) - }) - - it('should return 50 points for indexer tier requesting SLPDB access', () => { - const jwtInfo = { - apiLevel: 30, - resource: 'slp', - id: '5e3a0415eb29a962da2708b4' - } - - const result = rateLimits.calcPoints(jwtInfo) - assert.equal(result, 50) - }) - - it('should return 10 point for SLP tier requesting full node access', () => { - const jwtInfo = { - apiLevel: 40, - resource: 'blockchain', - id: '5e3a0415eb29a962da2708b4' - } - - const result = rateLimits.calcPoints(jwtInfo) - assert.equal(result, 10) - }) - - it('should return 10 points for SLP tier requesting indexer access', () => { - const jwtInfo = { - apiLevel: 40, - resource: 'blockbook', - id: '5e3a0415eb29a962da2708b4' - } - - const result = rateLimits.calcPoints(jwtInfo) - assert.equal(result, 10) - }) - - it('should return 10 points for SLP tier requesting SLPDB access', () => { - const jwtInfo = { - apiLevel: 40, - resource: 'slp', - id: '5e3a0415eb29a962da2708b4' - } - - const result = rateLimits.calcPoints(jwtInfo) - assert.equal(result, 10) - }) - }) - - describe('#rateLimitByResource', () => { - // NOTE: this test will fail if you run multiple integration tests in a - // short period. Because it talks to the Redis DB. - it('should pass through rate-limit middleware', async () => { - req.baseUrl = '/v4' - req.path = '/control/getNetworkInfo' - req.url = req.path - req.method = 'GET' - - // Call the route twice to trigger the rate handling. - await rateLimits.rateLimitByResource(req, res, next) - await rateLimits.rateLimitByResource(req, res, next) - - // next() will be called if rate-limit is not triggered - assert.equal(next.called, true) - }) - - it('should trigger rate-limit handler if rate limits exceeds 5 request per minute', async () => { - req.baseUrl = '/v4' - req.path = '/control/getNetworkInfo' - req.url = req.path - req.method = 'GET' - - for (let i = 0; i < 5; i++) { - next.reset() // reset the stubbed next() function. - - await rateLimits.rateLimitByResource(req, res, next) - // console.log(`next() called: ${next.called}`) - } - - // Note: next() will be called unless the rate-limit kicks in. - assert.equal( - next.called, - false, - 'next should not be called if rate limit was triggered.' - ) - }) - - it('should NOT trigger rate-limit for free-tier at 5 RPM', async () => { - // Create a new instance of the rate limit so we start with zeroed tracking. - rateLimits = new RateLimits() - - req.baseUrl = '/v4' - req.path = '/control/getNetworkInfo' - req.url = req.path - req.method = 'GET' - - req.locals.jwtToken = 'some-token' - - const jwtInfo = { - apiLevel: 10, - id: '5e3a0415eb29a962da2708b1' - } - - // Mock the call to the jwt library. - sandbox.stub(rateLimits.jwt, 'verify').returns(jwtInfo) - - for (let i = 0; i < 5; i++) { - next.reset() // reset the stubbed next() function. - - await rateLimits.rateLimitByResource(req, res, next) - // console.log(`next() called: ${next.called}`) - } - - // console.log(`req.locals after test: ${util.inspect(req.locals)}`) - - // Note: next() will be called unless the rate-limit kicks in. - assert.equal( - next.called, - true, - 'next should be called if rate limit was not triggered.' - ) - }) - - it('should trigger rate-limit for free tier after 20 RPM', async () => { - // Create a new instance of the rate limit so we start with zeroed tracking. - rateLimits = new RateLimits() - - req.baseUrl = '/v4' - req.path = '/control/getNetworkInfo' - req.url = req.path - req.method = 'GET' - - req.locals.jwtToken = 'some-token' - - const jwtInfo = { - apiLevel: 10, - id: '5e3a0415eb29a962da2708b2' - } - - // Mock the call to the jwt library. - sandbox.stub(rateLimits.jwt, 'verify').returns(jwtInfo) - - for (let i = 0; i < 22; i++) { - next.reset() // reset the stubbed next() function. - - await rateLimits.rateLimitByResource(req, res, next) - // console.log(`next() called: ${next.called}`) - } - - // Note: next() will be called unless the rate-limit kicks in. - assert.equal( - next.called, - false, - 'next should not be called if rate limit was triggered.' - ) - }) - - it('should NOT trigger rate-limit handler for indexer-tier at 25 RPM', async () => { - // Create a new instance of the rate limit so we start with zeroed tracking. - rateLimits = new RateLimits() - - req.baseUrl = '/v4' - req.path = '/control/getNetworkInfo' - req.url = req.path - req.method = 'GET' - - req.locals.jwtToken = 'some-token' - - const jwtInfo = { - apiLevel: 20, - id: '5e3a0415eb29a962da2708b3' - } - - // Mock the call to the jwt library. - sandbox.stub(rateLimits.jwt, 'verify').returns(jwtInfo) - - for (let i = 0; i < 25; i++) { - next.reset() // reset the stubbed next() function. - - await rateLimits.rateLimitByResource(req, res, next) - // console.log(`next() called: ${next.called}`) - } - - // console.log(`req.locals after test: ${util.inspect(req.locals)}`) - - // Note: next() will be called unless the rate-limit kicks in. - assert.equal( - next.called, - true, - 'next should be called if rate limit was not triggered.' - ) - }) - - it('should still rate-limit at a higher RPM for pro-tier', async () => { - // Create a new instance of the rate limit so we start with zeroed tracking. - rateLimits = new RateLimits() - - req.baseUrl = '/v4' - req.path = '/control/getNetworkInfo' - req.url = req.path - req.method = 'GET' - - req.locals.jwtToken = 'some-token' - - const jwtInfo = { - apiLevel: 20, - id: '5e3a0415eb29a962da2708b5' - } - - // Mock the call to the jwt library. - sandbox.stub(rateLimits.jwt, 'verify').returns(jwtInfo) - - for (let i = 0; i < 150; i++) { - next.reset() // reset the stubbed next() function. - - await rateLimits.rateLimitByResource(req, res, next) - // console.log(`next() called: ${next.called}`) - } - - // console.log(`req.locals after test: ${util.inspect(req.locals)}`) - - // Note: next() will be called unless the rate-limit kicks in. - assert.equal( - next.called, - false, - 'next should NOT be called if rate limit was triggered.' - ) - }) - - // CT 2/24/21 This test may have been invalidated by the interal IP address - // passing that I implemented to get hydrateUtxos() working properly. - // I'm commenting this out until I can study the side effects of this change, - // and why exactly this test is breaking. - // it('should handle misconfigured token secret', async () => { - // // Create a new instance of the rate limit so we start with zeroed tracking. - // rateLimits = new RateLimits() - // - // req.baseUrl = '/v4' - // req.path = '/control/getNetworkInfo' - // req.url = req.path - // req.method = 'GET' - // - // req.locals.jwtToken = 'some-token' - // - // next.reset() // reset the stubbed next() function. - // - // await rateLimits.rateLimitByResource(req, res, next) - // - // // Issues with token secret should treat incoming requests as anonymous - // // calls with 50 points, or 20 RPM. - // assert.equal(res.locals.pointsToConsume, 50) - // }) - }) - - describe('#isInWhitelist', () => { - it('should return false when no argument is passed in', () => { - const result = rateLimits.isInWhitelist() - - assert.equal(result, false) - }) - - it('should return false when origin is not in the whitelist', () => { - const origin = 'blah.com' - - const result = rateLimits.isInWhitelist(origin) - - assert.equal(result, false) - }) - - it('should return true when origin is in the whitelist', () => { - const origin = 'message.fullstack.cash' - - const result = rateLimits.isInWhitelist(origin) - - assert.equal(result, true) - }) - }) -}) - -// Generates a Basic authorization header. -// function generateAuthHeader (pass) { -// // https://en.wikipedia.org/wiki/Basic_access_authentication -// const username = 'BITBOX' -// const combined = `${username}:${pass}` -// -// var base64Credential = Buffer.from(combined).toString('base64') -// var readyCredential = `Basic ${base64Credential}` -// -// return readyCredential -// }