fullstack/bch-api
1
0
Fork 0
mirror of https://github.com/fullstack-cash/bch-api.git synced 2026-09-21 16:52:04 -07:00
Code Issues Packages Projects Releases Wiki Activity

fix(rate limits): Removing old rate limits library

Browse Source
This commit is contained in:
Chris Troutner
2021-03-09 12:43:30 -08:00
parent a475aca8bb
commit 0aa4d0ecd4
5 changed files with 325 additions and 1155 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/app.js
+6 -10
View File
@@ -3,12 +3,8 @@
const express = require('express')
// Middleware
// const { routeRateLimit } = require("./middleware/route-ratelimit")
// const RateLimits = require('./middleware/route-ratelimit')
// const rateLimits = new RateLimits()
const RateLimits2 = require('./middleware/route-ratelimit2')
const rateLimits2 = new RateLimits2()
const RateLimits = require('./middleware/route-ratelimit')
const rateLimits = new RateLimits()
const path = require('path')
const logger = require('morgan')
@@ -102,14 +98,14 @@ const v4prefix = 'v4'
// START Rate Limits
const auth = new AuthMW()
// Ensure req.locals and res.locals objects exist.
app.use(`/${v4prefix}/`, rateLimits.populateLocals)
// Allow users to turn off rate limits with an environment variable.
const DO_NOT_USE_RATE_LIMITS = process.env.DO_NOT_USE_RATE_LIMITS || false
console.log(`DO_NOT_USE_RATE_LIMITS: ${DO_NOT_USE_RATE_LIMITS}`)
// Ensure req.locals and res.locals objects exist.
app.use(`/${v4prefix}/`, rateLimits2.populateLocals)
if (!DO_NOT_USE_RATE_LIMITS) {
console.log('Rate limits are being used')
// Inspect the header for a JWT token.
@@ -120,7 +116,7 @@ if (!DO_NOT_USE_RATE_LIMITS) {
app.use(`/${v4prefix}/`, auth.mw())
// Experimental rate limits
app.use(`/${v4prefix}/`, rateLimits2.applyRateLimits)
app.use(`/${v4prefix}/`, rateLimits.applyRateLimits)
// Rate limit on all v4 routes
// Establish and enforce rate limits.
src/middleware/route-ratelimit.js
+312 -228
View File
@@ -1,63 +1,61 @@
/*
This file will replace the original rate-limit.js file.
Sets the rate limits for the anonymous and paid tiers. Current rate limits:
- 1000 points in 60 seconds
- 10 points per call for paid tier (100 RPM)
- 50 points per call for anonymous tier (20 RPM)
Background:
The rate limits below were originially coded with the idea of charging on a
per-resource basis. However, that was confusing to end users trying to purchase
a subscription. So everything was simplied to two tiers: paid and anonymous
CT 3/7/21
This rate limits have been refactored to consider the following use cases:
The rate limit handling is designed for these four use cases:
- Users who want to buy a JWT token for 24 hour access.
- Users who want to buy different RPM tiers: 100, 250, 600
- Basic Authentication which should not have any rate limits applied.
- Users that run bch-api locally and do not want any rate limits applied.
- Local installations that do not want any authentication or rate limits at all.
The second two use cases also apply to internal rate limits. The internal rate
limits should not be applied to calls from those users.
The Basic Auth use cases is considered when determining internal rate limits.
The internal rate limits should not be applied to calls from those users.
A lot of attention has been paid to passing rate-limit information for the user
when they trigger an endpoint that makes a lot of internal API calls. Examples
are hydrateUtxos() and getPublicKey().
*/
are hydrateUtxos() and getPublicKey(). These keeps things fair by charging the
same for 'light' API calls and 'heavy' API calls.
'use strict'
TODO:
- Add code for applying rate limits to whitelist domains.
*/
// Public npm libraries.
const jwt = require('jsonwebtoken')
const Redis = require('ioredis')
const { RateLimiterRedis } = require('rate-limiter-flexible')
// local libraries.
const wlogger = require('../util/winston-logging')
const config = require('../../config')
const ANON_LIMITS = config.anonRateLimit
const WHITELIST_RATE_LIMIT = config.whitelistRateLimit
const WHITELIST_DOMAINS = config.whitelistDomains
const INTERNAL_RATE_LIMIT = 1
let _this // Global pointer to instance of class, when 'this' context is lost.
// Redis
// Setup Redis to track rate limits for each user.
const redisOptions = {
enableOfflineQueue: false,
port: process.env.REDIS_PORT ? process.env.REDIS_PORT : 6379,
host: process.env.REDIS_HOST ? process.env.REDIS_HOST : '127.0.0.1'
}
console.log(`redisOptions: ${JSON.stringify(redisOptions, null, 2)}`)
const Redis = require('ioredis')
const redisClient = new Redis(redisOptions)
// Rate limiter middleware lib.
const { RateLimiterRedis } = require('rate-limiter-flexible')
const rateLimitOptions = {
storeClient: redisClient,
points: 1000, // Number of points
duration: 60 // Per minute (per 60 seconds)
}
let _this
// Constants
const ANON_LIMITS = config.anonRateLimit
// const WHITELIST_RATE_LIMIT = config.whitelistRateLimit
const WHITELIST_DOMAINS = config.whitelistDomains
const WHITELIST_POINTS_TO_CONSUME = config.whitelistRateLimit
const POINTS_PER_MINUTE = config.pointsPerMinute
const INTERNAL_POINTS_TO_CONSUME = 1
class RateLimits {
constructor () {
@@ -68,157 +66,187 @@ class RateLimits {
this.config = config
}
// Used to disconnect from the Redis DB.
// Called by unit tests so that node.js thread doesn't live forever.
closeRedis () {
redisClient.disconnect()
}
async wipeRedis () {
await redisClient.flushdb()
}
// This is the new rate limit function that uses the rate-limiter-flexible npm
// library. It uses fine-grain rate limiting based on the resources being
// consumed.
async rateLimitByResource (req, res, next) {
// This is the main middleware funciton of this library. All other functions
// support this function.
async applyRateLimits (req, res, next) {
try {
let userId
let decoded = {}
// Create a req.locals object if not passed in.
if (!req.locals) {
req.locals = {
// default values
jwtToken: '',
proLimit: false,
apiLevel: 0
}
// Exit if the user has already authenticated with Basic Authentication.
if (req.locals.proLimit) {
console.log('External call, basic auth, skipping rate limits.')
wlogger.debug(
'req.locals.proLimit = true; Using Basic Authentication instead of rate limits'
)
return next()
}
// Create a res.locals object if it does not exist. This is used for
// debugging.
if (!res.locals) {
res.locals = {
rateLimitTriggered: false
// Determine if the call is an external or internal API call.
const isInternal = _this.checkInternalIp(req)
console.log(`isInternal: ${isInternal}`)
// Determine if the call originates from another computer on the intranet.
const isWhitelistOrigin = _this.isInWhitelist(req)
console.log('isWhitelistOrigin: ', isWhitelistOrigin)
// Handle the use case of internally-generated requests.
if (isInternal) {
// Internal API calls should pass the authentication data in through the
// the usrObj in the body.
if (req.body && req.body.usrObj) {
if (req.body.usrObj.proLimit) {
console.log('Internal call, basic auth, skipping rate limits.')
// If this is an internal call that originated from a user using
// Basic Authentication, then skip rate-limits.
return next()
} else {
console.log(
'Internal call, applying rate limits. Using JWT if available.'
)
// Determine if user has exceeded their rate limits. Pass in the
// JWT token if one exists.
const hasExceededRateLimit = await _this.trackRateLimits(
req,
res,
req.body.usrObj.jwtToken
)
if (!hasExceededRateLimit) {
// Rate limits have not been exceeded. Processing can continue.
return next()
} else {
// trackRateLimits() returns the 'res' object with an error message
// and status code.
return hasExceededRateLimit
}
}
// Decode the JWT token if one exists.
if (req.locals.jwtToken) {
try {
decoded = _this.jwt.verify(
req.locals.jwtToken,
_this.config.apiTokenSecret
)
// console.log(`decoded: ${JSON.stringify(decoded, null, 2)}`)
userId = decoded.id
} catch (err) {
// This handler will be triggered if the JWT token does not match the
// token secret.
wlogger.error(
`Last three letters of token secret: ${_this.config.apiTokenSecret.slice(
-3
)}`
)
wlogger.error(
'Error trying to decode JWT token in route-ratelimit.js/newRateLimit(): ',
err
)
}
//
} else if (req.body && req.body.usrObj) {
// Same as above, but this code path is activated from internal calls to
// bch-js, like hydrateUtxo(), which passes the user object from the
// original API call.
try {
decoded = _this.jwt.verify(
req.body.usrObj.jwtToken,
_this.config.apiTokenSecret
)
// console.log(`decoded: ${JSON.stringify(decoded, null, 2)}`)
userId = decoded.id
} catch (err) {
// This handler will be triggered if the JWT token does not match the
// token secret.
wlogger.error(
'Error in route-ratelimit.js trying to decode JWT token in usrObj'
)
}
} else {
wlogger.debug('No JWT token found!')
// This should be a corner case. Calls should not be going into this
// code path, so the system should throw up big warning signs when they
// do.
// This code path happens when an internal call is made but does not
// pass the usrObj. Legacy code needs to be refactored to use the usrObj
// and avoid this code path. This code path is 'pooled': all users
// share the same rate limits. Even at 1000 RPM, this pool will get
// exhausted easily.
const warnMsg =
'Internal call. req.body.usrObj does not exist. Applying high-speed internal rate limits.'
console.log(warnMsg)
wlogger.info(warnMsg)
const defaultPayload = {
id: '98.76.54.32',
email: 'internal@bchtest.net',
apiLevel: 40,
rateLimit: 100,
pointsToConsume: INTERNAL_POINTS_TO_CONSUME,
duration: 30
}
// Default value is 50 points per request = 20 RPM
let rateLimit = ANON_LIMITS
// Default values, in case there is an error.
const defaultJwt = _this.generateJwtToken(defaultPayload)
// Only evaluate the JWT token if the user is not using Basic Authentication.
if (!req.locals.proLimit) {
// Code here for the rate limiter is adapted from this example:
// https://github.com/animir/node-rate-limiter-flexible/wiki/Overall-example#authorized-and-not-authorized-users
try {
// The resource being consumed: full node, indexer, SLPDB, etc.
const resource = _this.getResource(req.url)
wlogger.debug(`resource: ${resource}`)
// Key will be the JWT ID if it exists, otherwise the IP address of the caller.
let key = userId || req.ip
res.locals.key = key // Feedback for tests.
// console.log(`key: ${key}`)
// const pointsToConsume = userId ? 1 : 30
decoded.resource = resource
let pointsToConsume = _this.calcPoints(decoded)
res.locals.pointsToConsume = pointsToConsume // Feedback for tests.
// Retrieve the origin.
let origin = req.get('origin')
// Handle calls coming from the intranet.
if (origin === undefined && key.indexOf('10.0.0.5') > -1) {
origin = 'slp-api'
}
wlogger.info(`origin: ${origin}`)
// If the request originates from one of the approved wallet apps, then
// apply paid-access rate limits.
// console.log(`origin: ${JSON.stringify(origin, null, 2)}`)
// console.log(`whitelist: ${JSON.stringify(WHITELIST_DOMAINS, null, 2)}`)
const isInWhitelist = _this.isInWhitelist(origin)
if (isInWhitelist) {
pointsToConsume = WHITELIST_RATE_LIMIT
res.locals.pointsToConsume = pointsToConsume // Feedback for tests.
}
// For internal calls, increase rate limits to as fast as possible.
if (
// Comment out the line below when running bch-js e2e rate limit tests.
key.toString().indexOf('::ffff:127.0.0.1') > -1 ||
// Do not comment out this line.
key.toString().indexOf('172.17.') > -1
) {
pointsToConsume = INTERNAL_RATE_LIMIT
res.locals.pointsToConsume = pointsToConsume // Feedback for tests.
}
wlogger.info(
`User ${key} consuming ${pointsToConsume} point for resource ${resource}.`
// Track the rate limit for this user. Pass in the JWT token, if one
// is available.
const hasExceededRateLimit = await _this.trackRateLimits(
req,
res,
defaultJwt
)
rateLimit = Math.floor(1000 / pointsToConsume)
if (!hasExceededRateLimit) {
// Rate limits have not been exceeded. Processing can continue.
return next()
} else {
// trackRateLimits() returns the 'res' object with an error message
// and status code.
return hasExceededRateLimit
}
}
//
//
} else {
// Handle the normal use-case of external requests
console.log(
'External call, applying rate limits. Using JWT if available.'
)
// Update the key so that rate limits track both the user and the resource.
key = `${key}-${resource}`
// For calls originating from a whitelist domain, apply a high-RPM
// JWT token to the call.
if (isWhitelistOrigin) {
const defaultPayload = {
id: '77.77.77.77',
email: 'whitelist@bchtest.net',
apiLevel: 40,
rateLimit: 100,
pointsToConsume: WHITELIST_POINTS_TO_CONSUME,
duration: 30
}
await _this.rateLimiter.consume(key, pointsToConsume)
// Inject the high-RPM JWT token into the call.
req.locals.jwtToken = _this.generateJwtToken(defaultPayload)
}
// Track the rate limit for this user. Pass in the JWT token, if one
// is available.
const hasExceededRateLimit = await _this.trackRateLimits(
req,
res,
req.locals.jwtToken
)
if (!hasExceededRateLimit) {
// Rate limits have not been exceeded. Processing can continue.
return next()
} else {
// trackRateLimits() returns the 'res' object with an error message
// and status code.
return hasExceededRateLimit
}
}
} catch (err) {
// console.log('err: ', err)
wlogger.error('Error in route-ratelimit2.js/applyRateLimits(): ', err)
}
// By default, move to the next middleware.
next()
}
// A wrapper for Redis-based rate limiter.
// Will return false if the user has not exceeded the rate limit. Otherwise
// it will return the 'res' object with an error status and message, which
// should be returned by the middleware.
async trackRateLimits (req, res, jwtToken) {
// Anonymous rate limits are used by default.
let pointsToConsume = ANON_LIMITS
let key = req.ip // Use the IP address as the key, by default.
try {
// Decode the JWT token if it exists
if (jwtToken) {
const decoded = _this.decodeJwtToken(jwtToken)
// console.log(`decoded: ${JSON.stringify(decoded, null, 2)}`)
// Preferentially use the decoded ID in the JWT payload, as the key.
key = decoded.id
pointsToConsume = decoded.pointsToConsume
}
console.log(`rate limit key: ${key}`)
// This function will throw an error if the user exceeds the rate limit.
// The 429 error response is handled by the catch().
await _this.rateLimiter.consume(key, pointsToConsume)
res.locals.pointsToConsume = pointsToConsume // Feedback for tests.
// Signal that the user has not exceeded their rate limits.
return false
} catch (err) {
console.log('err: ', err)
const rateLimit = Math.floor(POINTS_PER_MINUTE / pointsToConsume)
// Used for returning data for tests.
res.locals.rateLimitTriggered = true
// console.log('res.locals: ', res.locals)
@@ -229,87 +257,58 @@ class RateLimits {
})
}
}
} catch (err) {
wlogger.error('Error in route-ratelimit.js/newRateLimit(): ', err)
// throw err
}
next()
// Attempts to decode a JWT token. Returns default values if it fails.
decodeJwtToken (jwtToken) {
const defaultPayload = {
id: '123.456.789.10',
email: 'test@bchtest.net',
apiLevel: 10,
rateLimit: 3,
pointsToConsume: ANON_LIMITS,
duration: 30
}
// Calculates the points consumed, based on the jwt information and the route
// requested.
calcPoints (jwtInfo) {
let retVal = ANON_LIMITS // By default, use anonymous tier.
try {
// console.log(`jwtInfo: ${JSON.stringify(jwtInfo, null, 2)}`)
// Default values, in case there is an error.
const defaultJwt = _this.generateJwtToken(defaultPayload)
const apiLevel = jwtInfo.apiLevel
const resource = jwtInfo.resource
// Generate a default payload to use, if the decoding of the user-provided
// jwt fails.
let decoded = _this.jwt.verify(defaultJwt, _this.config.apiTokenSecret)
const level30Routes = ['insight', 'bitcore', 'blockbook', 'electrumx']
const level40Routes = ['slp']
wlogger.debug(`apiLevel: ${apiLevel}`)
// Only evaluate if user is using a JWT token.
if (jwtInfo.id) {
// SLP indexer routes
if (level40Routes.includes(resource)) {
if (apiLevel >= 40) retVal = 10
// else if (apiLevel >= 10) retVal = 10
else retVal = ANON_LIMITS
// Normal indexer routes
} else if (level30Routes.includes(resource)) {
if (apiLevel >= 30) retVal = 10
else retVal = ANON_LIMITS
// Full node tier
} else if (apiLevel >= 20) {
retVal = 10
// Free tier, full node only.
} else {
retVal = ANON_LIMITS
}
}
return retVal
} catch (err) {
wlogger.error('Error in route-ratelimit.js/calcPoints()')
// throw err
retVal = ANON_LIMITS
}
return retVal
}
// This function parses the req.url property to identify what resource
// the user is requesting.
// This was created as a function so that it can be unit tested. Not sure
// what kind of variations will be seen in production.
getResource (url) {
try {
wlogger.debug(`url: ${JSON.stringify(url, null, 2)}`)
const splitUrl = url.split('/')
const resource = splitUrl[1]
return resource
decoded = _this.jwt.verify(jwtToken, _this.config.apiTokenSecret)
} catch (err) {
wlogger.error('Error in getResource().')
throw err
wlogger.error('Error in route-ratelimit2.js/decodeJwtTokens(): ', err)
}
return decoded
} catch (err) {
wlogger.error(
'Unhandled error in route-ratelimit2.js/deocdeJwtToken: ',
err
)
// Making sure there is an exp property. Not sure if this will cause an
// issue, using a hard-coded value.
defaultPayload.exp = 1574269450
return defaultPayload
}
}
// Returns a boolean if the origin of the request matches a domain in the
// whitelist.
isInWhitelist (origin) {
isInWhitelist (req) {
try {
const retVal = false // Default value.
// Retrieve the origin.
const origin = req.get('origin')
console.log(`origin: ${origin}`)
// If the origin is not determinable, return false.
if (!origin) return false
// console.log(`WHITELIST_DOMAINS: ${JSON.stringify(WHITELIST_DOMAINS, null, 2)}`)
@@ -317,9 +316,7 @@ class RateLimits {
for (let i = 0; i < WHITELIST_DOMAINS.length; i++) {
const thisDomain = WHITELIST_DOMAINS[i]
if (origin.toString().indexOf(thisDomain) > -1) {
return true
}
if (origin.includes(thisDomain)) return true
}
return retVal
@@ -330,6 +327,93 @@ class RateLimits {
return false
}
}
// Checks the request object to see if it's IP address matches an internal
// IP address. That means the call is an internal API call and should be
// treated differently than an external API call.
checkInternalIp (req) {
try {
// Default value
let isInternal = false
const ip = req.ip
if (ip.includes('127.0.0.1')) isInternal = true
if (ip.includes('172.17.')) isInternal = true
// TODO: Add 192.168.
return isInternal
} catch (err) {
wlogger.error(
'Error in checkInternalIp(). Returning false be default. Err: ',
err
)
return false
}
}
// Used to disconnect from the Redis DB.
// Called by unit tests so that node.js thread doesn't live forever.
closeRedis () {
redisClient.disconnect()
}
// Clear the redis database. Used by unit tests.
async wipeRedis () {
await redisClient.flushdb()
}
// Generates a JWT token for testing purposes. This is not used in production.
// This function mirrors the kind of JWT token that would be generated by
// jwt-bch-api.
generateJwtToken (payload) {
try {
const jwtOptions = {
expiresIn: '30 days'
}
const token = _this.jwt.sign(
payload,
_this.config.apiTokenSecret,
jwtOptions
)
return token
} catch (err) {
console.error('Error in generateJwtToken()')
throw err
}
}
// Called when rate limits are not used.
populateLocals (req, res, next) {
try {
// Create a re*Q*.locals object if not passed in.
// req.locals.proLimit will be true if the user is using Basic Authentication.
if (!req.locals) {
req.locals = {
// default values
jwtToken: '',
proLimit: false,
apiLevel: 0
}
}
// Create a re*S*.locals object if it does not exist.
if (!res.locals) {
res.locals = {
rateLimitTriggered: false
}
}
next()
} catch (err) {
console.error('Error in populateLocals(): ', err)
throw err
}
}
}
module.exports = RateLimits
src/middleware/route-ratelimit2.js
-419
View File
@@ -1,419 +0,0 @@
/*
This file will replace the original rate-limit.js file.
Sets the rate limits for the anonymous and paid tiers. Current rate limits:
- 1000 points in 60 seconds
- 10 points per call for paid tier (100 RPM)
- 50 points per call for anonymous tier (20 RPM)
The rate limit handling is designed for these four use cases:
- Users who want to buy a JWT token for 24 hour access.
- Users who want to buy different RPM tiers: 100, 250, 600
- Basic Authentication which should not have any rate limits applied.
- Local installations that do not want any authentication or rate limits at all.
The Basic Auth use cases is considered when determining internal rate limits.
The internal rate limits should not be applied to calls from those users.
A lot of attention has been paid to passing rate-limit information for the user
when they trigger an endpoint that makes a lot of internal API calls. Examples
are hydrateUtxos() and getPublicKey(). These keeps things fair by charging the
same for 'light' API calls and 'heavy' API calls.
TODO:
- Add code for applying rate limits to whitelist domains.
*/
// Public npm libraries.
const jwt = require('jsonwebtoken')
const Redis = require('ioredis')
const { RateLimiterRedis } = require('rate-limiter-flexible')
// local libraries.
const wlogger = require('../util/winston-logging')
const config = require('../../config')
let _this // Global pointer to instance of class, when 'this' context is lost.
// Setup Redis to track rate limits for each user.
const redisOptions = {
enableOfflineQueue: false,
port: process.env.REDIS_PORT ? process.env.REDIS_PORT : 6379,
host: process.env.REDIS_HOST ? process.env.REDIS_HOST : '127.0.0.1'
}
console.log(`redisOptions: ${JSON.stringify(redisOptions, null, 2)}`)
const redisClient = new Redis(redisOptions)
const rateLimitOptions = {
storeClient: redisClient,
points: 1000, // Number of points
duration: 60 // Per minute (per 60 seconds)
}
// Constants
const ANON_LIMITS = config.anonRateLimit
// const WHITELIST_RATE_LIMIT = config.whitelistRateLimit
const WHITELIST_DOMAINS = config.whitelistDomains
const WHITELIST_POINTS_TO_CONSUME = config.whitelistRateLimit
const POINTS_PER_MINUTE = config.pointsPerMinute
const INTERNAL_POINTS_TO_CONSUME = 1
class RateLimits {
constructor () {
_this = this
this.jwt = jwt
this.rateLimiter = new RateLimiterRedis(rateLimitOptions)
this.config = config
}
// This is the main middleware funciton of this library. All other functions
// support this function.
async applyRateLimits (req, res, next) {
try {
// Exit if the user has already authenticated with Basic Authentication.
if (req.locals.proLimit) {
console.log('External call, basic auth, skipping rate limits.')
wlogger.debug(
'req.locals.proLimit = true; Using Basic Authentication instead of rate limits'
)
return next()
}
// Determine if the call is an external or internal API call.
const isInternal = _this.checkInternalIp(req)
console.log(`isInternal: ${isInternal}`)
// Determine if the call originates from another computer on the intranet.
const isWhitelistOrigin = _this.isInWhitelist(req)
console.log('isWhitelistOrigin: ', isWhitelistOrigin)
// Handle the use case of internally-generated requests.
if (isInternal) {
// Internal API calls should pass the authentication data in through the
// the usrObj in the body.
if (req.body && req.body.usrObj) {
if (req.body.usrObj.proLimit) {
console.log('Internal call, basic auth, skipping rate limits.')
// If this is an internal call that originated from a user using
// Basic Authentication, then skip rate-limits.
return next()
} else {
console.log(
'Internal call, applying rate limits. Using JWT if available.'
)
// Determine if user has exceeded their rate limits. Pass in the
// JWT token if one exists.
const hasExceededRateLimit = await _this.trackRateLimits(
req,
res,
req.body.usrObj.jwtToken
)
if (!hasExceededRateLimit) {
// Rate limits have not been exceeded. Processing can continue.
return next()
} else {
// trackRateLimits() returns the 'res' object with an error message
// and status code.
return hasExceededRateLimit
}
}
} else {
// This should be a corner case. Calls should not be going into this
// code path, so the system should throw up big warning signs when they
// do.
// This code path happens when an internal call is made but does not
// pass the usrObj. Legacy code needs to be refactored to use the usrObj
// and avoid this code path. This code path is 'pooled': all users
// share the same rate limits. Even at 1000 RPM, this pool will get
// exhausted easily.
const warnMsg =
'Internal call. req.body.usrObj does not exist. Applying high-speed internal rate limits.'
console.log(warnMsg)
wlogger.info(warnMsg)
const defaultPayload = {
id: '98.76.54.32',
email: 'internal@bchtest.net',
apiLevel: 40,
rateLimit: 100,
pointsToConsume: INTERNAL_POINTS_TO_CONSUME,
duration: 30
}
// Default values, in case there is an error.
const defaultJwt = _this.generateJwtToken(defaultPayload)
// Track the rate limit for this user. Pass in the JWT token, if one
// is available.
const hasExceededRateLimit = await _this.trackRateLimits(
req,
res,
defaultJwt
)
if (!hasExceededRateLimit) {
// Rate limits have not been exceeded. Processing can continue.
return next()
} else {
// trackRateLimits() returns the 'res' object with an error message
// and status code.
return hasExceededRateLimit
}
}
//
//
} else {
// Handle the normal use-case of external requests
console.log(
'External call, applying rate limits. Using JWT if available.'
)
// For calls originating from a whitelist domain, apply a high-RPM
// JWT token to the call.
if (isWhitelistOrigin) {
const defaultPayload = {
id: '77.77.77.77',
email: 'whitelist@bchtest.net',
apiLevel: 40,
rateLimit: 100,
pointsToConsume: WHITELIST_POINTS_TO_CONSUME,
duration: 30
}
// Inject the high-RPM JWT token into the call.
req.locals.jwtToken = _this.generateJwtToken(defaultPayload)
}
// Track the rate limit for this user. Pass in the JWT token, if one
// is available.
const hasExceededRateLimit = await _this.trackRateLimits(
req,
res,
req.locals.jwtToken
)
if (!hasExceededRateLimit) {
// Rate limits have not been exceeded. Processing can continue.
return next()
} else {
// trackRateLimits() returns the 'res' object with an error message
// and status code.
return hasExceededRateLimit
}
}
} catch (err) {
wlogger.error('Error in route-ratelimit2.js/applyRateLimits(): ', err)
}
// By default, move to the next middleware.
next()
}
// A wrapper for Redis-based rate limiter.
// Will return false if the user has not exceeded the rate limit. Otherwise
// it will return the 'res' object with an error status and message, which
// should be returned by the middleware.
async trackRateLimits (req, res, jwtToken) {
// Anonymous rate limits are used by default.
let pointsToConsume = ANON_LIMITS
let key = req.ip // Use the IP address as the key, by default.
try {
// Decode the JWT token if it exists
if (jwtToken) {
const decoded = _this.decodeJwtToken(jwtToken)
// console.log(`decoded: ${JSON.stringify(decoded, null, 2)}`)
// Preferentially use the decoded ID in the JWT payload, as the key.
key = decoded.id
pointsToConsume = decoded.pointsToConsume
}
console.log(`rate limit key: ${key}`)
// This function will throw an error if the user exceeds the rate limit.
// The 429 error response is handled by the catch().
await _this.rateLimiter.consume(key, pointsToConsume)
res.locals.pointsToConsume = pointsToConsume // Feedback for tests.
// Signal that the user has not exceeded their rate limits.
return false
} catch (err) {
console.log('err: ', err)
const rateLimit = Math.floor(POINTS_PER_MINUTE / pointsToConsume)
res.locals.rateLimitTriggered = true
// console.log('res.locals: ', res.locals)
// Rate limited was triggered
res.status(429) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
return res.json({
error: `Too many requests. Your limits are currently ${rateLimit} requests per minute. Increase rate limits at https://fullstack.cash`
})
}
}
// Attempts to decode a JWT token. Returns default values if it fails.
decodeJwtToken (jwtToken) {
const defaultPayload = {
id: '123.456.789.10',
email: 'test@bchtest.net',
apiLevel: 10,
rateLimit: 3,
pointsToConsume: ANON_LIMITS,
duration: 30
}
try {
// Default values, in case there is an error.
const defaultJwt = _this.generateJwtToken(defaultPayload)
// Generate a default payload to use, if the decoding of the user-provided
// jwt fails.
let decoded = _this.jwt.verify(defaultJwt, _this.config.apiTokenSecret)
try {
decoded = _this.jwt.verify(jwtToken, _this.config.apiTokenSecret)
} catch (err) {
wlogger.error('Error in route-ratelimit2.js/decodeJwtTokens(): ', err)
}
return decoded
} catch (err) {
wlogger.error(
'Unhandled error in route-ratelimit2.js/deocdeJwtToken: ',
err
)
// Making sure there is an exp property. Not sure if this will cause an
// issue, using a hard-coded value.
defaultPayload.exp = 1574269450
return defaultPayload
}
}
// Returns a boolean if the origin of the request matches a domain in the
// whitelist.
isInWhitelist (req) {
try {
const retVal = false // Default value.
// Retrieve the origin.
const origin = req.get('origin')
console.log(`origin: ${origin}`)
// If the origin is not determinable, return false.
if (!origin) return false
// console.log(`WHITELIST_DOMAINS: ${JSON.stringify(WHITELIST_DOMAINS, null, 2)}`)
for (let i = 0; i < WHITELIST_DOMAINS.length; i++) {
const thisDomain = WHITELIST_DOMAINS[i]
if (origin.includes(thisDomain)) return true
}
return retVal
} catch (err) {
wlogger.error(
'Error in route-ratelimit.js/isInWhitelist(). Returning false by default.'
)
return false
}
}
// Checks the request object to see if it's IP address matches an internal
// IP address. That means the call is an internal API call and should be
// treated differently than an external API call.
checkInternalIp (req) {
try {
// Default value
let isInternal = false
const ip = req.ip
if (ip.includes('127.0.0.1')) isInternal = true
if (ip.includes('172.17.')) isInternal = true
// TODO: Add 192.168.
return isInternal
} catch (err) {
wlogger.error(
'Error in checkInternalIp(). Returning false be default. Err: ',
err
)
return false
}
}
// Used to disconnect from the Redis DB.
// Called by unit tests so that node.js thread doesn't live forever.
closeRedis () {
redisClient.disconnect()
}
// Clear the redis database. Used by unit tests.
async wipeRedis () {
await redisClient.flushdb()
}
// Generates a JWT token for testing purposes. This is not used in production.
// This function mirrors the kind of JWT token that would be generated by
// jwt-bch-api.
generateJwtToken (payload) {
try {
const jwtOptions = {
expiresIn: '30 days'
}
const token = _this.jwt.sign(
payload,
_this.config.apiTokenSecret,
jwtOptions
)
return token
} catch (err) {
console.error('Error in generateJwtToken()')
throw err
}
}
// Called when rate limits are not used.
populateLocals (req, res, next) {
try {
// Create a re*Q*.locals object if not passed in.
// req.locals.proLimit will be true if the user is using Basic Authentication.
if (!req.locals) {
req.locals = {
// default values
jwtToken: '',
proLimit: false,
apiLevel: 0
}
}
// Create a re*S*.locals object if it does not exist.
if (!res.locals) {
res.locals = {
rateLimitTriggered: false
}
}
next()
} catch (err) {
console.error('Error in populateLocals(): ', err)
throw err
}
}
}
module.exports = RateLimits
test/v4/rate-limit2-unit.js → test/v4/rate-limit-unit.js
+2 -2
View File
@@ -15,12 +15,12 @@ const config = require('../../config')
const { mockReq, mockRes, mockNext } = require('./mocks/express-mocks')
// Libraries under test
const RateLimits = require('../../src/middleware/route-ratelimit2')
const RateLimits = require('../../src/middleware/route-ratelimit')
let uut = new RateLimits()
let req, res, next
describe('#rate-routelimit2', () => {
describe('#rate-routelimit', () => {
let sandbox
before(async () => {
test/v4/rate-limits.js
-491
View File
@@ -1,491 +0,0 @@
/*
Unit tests for the rate limit middleware.
*/
'use strict'
const chai = require('chai')
const assert = chai.assert
const sinon = require('sinon')
// Used for debugging.
const util = require('util')
util.inspect.defaultOptions = { depth: 1 }
// Mocking data.
const { mockReq, mockRes, mockNext } = require('./mocks/express-mocks')
// Libraries under test
const RateLimits = require('../../src/middleware/route-ratelimit')
let rateLimits = new RateLimits()
// const controlRoute = require('../../src/routes/v4/full-node/control')
const jwtAuth = require('../../src/middleware/jwt-auth')
let req, res, next
// let originalEnvVars // Used during transition from integration to unit tests.
// JWT token used in tests.
const jwt = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6IjVkYWRlM2Y1NzM5ZTZjMGZmMDM0YjlhMSIsImlhdCI6MTU3MTY3NzQ1MCwiZXhwIjoxNTc0MjY5NDUwfQ.SSz7F7ETyBB3eoNG2VKCzPOhddtB-vrtmEoj7PxicrQ'
describe('#route-ratelimits & jwt-auth', () => {
let sandbox
before(async () => {
// Save existing environment variables.
// originalEnvVars = {
// BITCOINCOM_BASEURL: process.env.BITCOINCOM_BASEURL,
// RPC_BASEURL: process.env.RPC_BASEURL,
// RPC_USERNAME: process.env.RPC_USERNAME,
// RPC_PASSWORD: process.env.RPC_PASSWORD
// }
if (!process.env.JWT_AUTH_SERVER) { process.env.JWT_AUTH_SERVER = 'http://fakeurl.com/' }
// Wipe the Redis DB, which prevents false negatives when running integration
// tests back-to-back.
await rateLimits.wipeRedis()
})
// Setup the mocks before each test.
beforeEach(() => {
// Mock the req and res objects used by Express routes.
req = Object.assign({}, mockReq)
res = Object.assign({}, mockRes)
next = mockNext
// Explicitly reset the parmas and body.
req.params = {}
req.body = {}
req.query = {}
req.locals = {}
sandbox = sinon.createSandbox()
})
afterEach(() => {
sandbox.restore()
})
after(() => {
rateLimits.closeRedis()
})
describe('#jwt-auth.js', () => {
describe('#getTokenFromHeaders', () => {
it('should populate the req.locals object correctly', () => {
// Initialize req.locals
req.locals = {
proLimit: false,
apiLevel: 0
}
const header = `Token ${jwt}`
req.headers.authorization = header
jwtAuth.getTokenFromHeaders(req, res, next)
// console.log(`req.locals: ${JSON.stringify(req.locals, null, 2)}`)
assert.property(req.locals, 'proLimit')
assert.property(req.locals, 'apiLevel')
assert.property(req.locals, 'jwtToken')
assert.equal(req.locals.jwtToken, jwt)
})
})
})
describe('#getResource', () => {
it('should decode a blockchain request', () => {
const url =
'/blockchain/getTxOut/62a3ea958a463a372bc0caf2c374a7f60be9c624be63a0db8db78f05809df6d8/0?include_mempool=true'
const result = rateLimits.getResource(url)
// console.log(`result: ${JSON.stringify(result, null, 2)}`)
assert.equal(result, 'blockchain')
})
})
describe('#calcPoints', () => {
it('should return 50 points for anonymous user', () => {
const result = rateLimits.calcPoints()
// console.log(`result: ${result}`)
assert.equal(result, 50)
})
it('should return 50 points for free tier requesting full node access', () => {
const jwtInfo = {
apiLevel: 10,
resource: 'blockchain',
id: '5e3a0415eb29a962da2708b4'
}
const result = rateLimits.calcPoints(jwtInfo)
assert.equal(result, 50)
})
it('should return 50 points for free tier requesting indexer access', () => {
const jwtInfo = {
apiLevel: 10,
resource: 'blockbook',
id: '5e3a0415eb29a962da2708b4'
}
const result = rateLimits.calcPoints(jwtInfo)
assert.equal(result, 50)
})
it('should return 50 points for free tier requesting SLPDB access', () => {
const jwtInfo = {
apiLevel: 10,
resource: 'slp',
id: '5e3a0415eb29a962da2708b4'
}
const result = rateLimits.calcPoints(jwtInfo)
assert.equal(result, 50)
})
it('should return 10 points for full node tier requesting full node access', () => {
const jwtInfo = {
apiLevel: 20,
resource: 'blockchain',
id: '5e3a0415eb29a962da2708b4'
}
const result = rateLimits.calcPoints(jwtInfo)
assert.equal(result, 10)
})
it('should return 50 points for full-node tier requesting indexer access', () => {
const jwtInfo = {
apiLevel: 20,
resource: 'blockbook',
id: '5e3a0415eb29a962da2708b4'
}
const result = rateLimits.calcPoints(jwtInfo)
assert.equal(result, 50)
})
it('should return 50 points for full node tier requesting SLPDB access', () => {
const jwtInfo = {
apiLevel: 20,
resource: 'slp',
id: '5e3a0415eb29a962da2708b4'
}
const result = rateLimits.calcPoints(jwtInfo)
assert.equal(result, 50)
})
it('should return 10 point for indexer tier requesting full node access', () => {
const jwtInfo = {
apiLevel: 30,
resource: 'blockchain',
id: '5e3a0415eb29a962da2708b4'
}
const result = rateLimits.calcPoints(jwtInfo)
assert.equal(result, 10)
})
it('should return 10 points for indexer tier requesting indexer access', () => {
const jwtInfo = {
apiLevel: 30,
resource: 'blockbook',
id: '5e3a0415eb29a962da2708b4'
}
const result = rateLimits.calcPoints(jwtInfo)
assert.equal(result, 10)
})
it('should return 50 points for indexer tier requesting SLPDB access', () => {
const jwtInfo = {
apiLevel: 30,
resource: 'slp',
id: '5e3a0415eb29a962da2708b4'
}
const result = rateLimits.calcPoints(jwtInfo)
assert.equal(result, 50)
})
it('should return 10 point for SLP tier requesting full node access', () => {
const jwtInfo = {
apiLevel: 40,
resource: 'blockchain',
id: '5e3a0415eb29a962da2708b4'
}
const result = rateLimits.calcPoints(jwtInfo)
assert.equal(result, 10)
})
it('should return 10 points for SLP tier requesting indexer access', () => {
const jwtInfo = {
apiLevel: 40,
resource: 'blockbook',
id: '5e3a0415eb29a962da2708b4'
}
const result = rateLimits.calcPoints(jwtInfo)
assert.equal(result, 10)
})
it('should return 10 points for SLP tier requesting SLPDB access', () => {
const jwtInfo = {
apiLevel: 40,
resource: 'slp',
id: '5e3a0415eb29a962da2708b4'
}
const result = rateLimits.calcPoints(jwtInfo)
assert.equal(result, 10)
})
})
describe('#rateLimitByResource', () => {
// NOTE: this test will fail if you run multiple integration tests in a
// short period. Because it talks to the Redis DB.
it('should pass through rate-limit middleware', async () => {
req.baseUrl = '/v4'
req.path = '/control/getNetworkInfo'
req.url = req.path
req.method = 'GET'
// Call the route twice to trigger the rate handling.
await rateLimits.rateLimitByResource(req, res, next)
await rateLimits.rateLimitByResource(req, res, next)
// next() will be called if rate-limit is not triggered
assert.equal(next.called, true)
})
it('should trigger rate-limit handler if rate limits exceeds 5 request per minute', async () => {
req.baseUrl = '/v4'
req.path = '/control/getNetworkInfo'
req.url = req.path
req.method = 'GET'
for (let i = 0; i < 5; i++) {
next.reset() // reset the stubbed next() function.
await rateLimits.rateLimitByResource(req, res, next)
// console.log(`next() called: ${next.called}`)
}
// Note: next() will be called unless the rate-limit kicks in.
assert.equal(
next.called,
false,
'next should not be called if rate limit was triggered.'
)
})
it('should NOT trigger rate-limit for free-tier at 5 RPM', async () => {
// Create a new instance of the rate limit so we start with zeroed tracking.
rateLimits = new RateLimits()
req.baseUrl = '/v4'
req.path = '/control/getNetworkInfo'
req.url = req.path
req.method = 'GET'
req.locals.jwtToken = 'some-token'
const jwtInfo = {
apiLevel: 10,
id: '5e3a0415eb29a962da2708b1'
}
// Mock the call to the jwt library.
sandbox.stub(rateLimits.jwt, 'verify').returns(jwtInfo)
for (let i = 0; i < 5; i++) {
next.reset() // reset the stubbed next() function.
await rateLimits.rateLimitByResource(req, res, next)
// console.log(`next() called: ${next.called}`)
}
// console.log(`req.locals after test: ${util.inspect(req.locals)}`)
// Note: next() will be called unless the rate-limit kicks in.
assert.equal(
next.called,
true,
'next should be called if rate limit was not triggered.'
)
})
it('should trigger rate-limit for free tier after 20 RPM', async () => {
// Create a new instance of the rate limit so we start with zeroed tracking.
rateLimits = new RateLimits()
req.baseUrl = '/v4'
req.path = '/control/getNetworkInfo'
req.url = req.path
req.method = 'GET'
req.locals.jwtToken = 'some-token'
const jwtInfo = {
apiLevel: 10,
id: '5e3a0415eb29a962da2708b2'
}
// Mock the call to the jwt library.
sandbox.stub(rateLimits.jwt, 'verify').returns(jwtInfo)
for (let i = 0; i < 22; i++) {
next.reset() // reset the stubbed next() function.
await rateLimits.rateLimitByResource(req, res, next)
// console.log(`next() called: ${next.called}`)
}
// Note: next() will be called unless the rate-limit kicks in.
assert.equal(
next.called,
false,
'next should not be called if rate limit was triggered.'
)
})
it('should NOT trigger rate-limit handler for indexer-tier at 25 RPM', async () => {
// Create a new instance of the rate limit so we start with zeroed tracking.
rateLimits = new RateLimits()
req.baseUrl = '/v4'
req.path = '/control/getNetworkInfo'
req.url = req.path
req.method = 'GET'
req.locals.jwtToken = 'some-token'
const jwtInfo = {
apiLevel: 20,
id: '5e3a0415eb29a962da2708b3'
}
// Mock the call to the jwt library.
sandbox.stub(rateLimits.jwt, 'verify').returns(jwtInfo)
for (let i = 0; i < 25; i++) {
next.reset() // reset the stubbed next() function.
await rateLimits.rateLimitByResource(req, res, next)
// console.log(`next() called: ${next.called}`)
}
// console.log(`req.locals after test: ${util.inspect(req.locals)}`)
// Note: next() will be called unless the rate-limit kicks in.
assert.equal(
next.called,
true,
'next should be called if rate limit was not triggered.'
)
})
it('should still rate-limit at a higher RPM for pro-tier', async () => {
// Create a new instance of the rate limit so we start with zeroed tracking.
rateLimits = new RateLimits()
req.baseUrl = '/v4'
req.path = '/control/getNetworkInfo'
req.url = req.path
req.method = 'GET'
req.locals.jwtToken = 'some-token'
const jwtInfo = {
apiLevel: 20,
id: '5e3a0415eb29a962da2708b5'
}
// Mock the call to the jwt library.
sandbox.stub(rateLimits.jwt, 'verify').returns(jwtInfo)
for (let i = 0; i < 150; i++) {
next.reset() // reset the stubbed next() function.
await rateLimits.rateLimitByResource(req, res, next)
// console.log(`next() called: ${next.called}`)
}
// console.log(`req.locals after test: ${util.inspect(req.locals)}`)
// Note: next() will be called unless the rate-limit kicks in.
assert.equal(
next.called,
false,
'next should NOT be called if rate limit was triggered.'
)
})
// CT 2/24/21 This test may have been invalidated by the interal IP address
// passing that I implemented to get hydrateUtxos() working properly.
// I'm commenting this out until I can study the side effects of this change,
// and why exactly this test is breaking.
// it('should handle misconfigured token secret', async () => {
// // Create a new instance of the rate limit so we start with zeroed tracking.
// rateLimits = new RateLimits()
//
// req.baseUrl = '/v4'
// req.path = '/control/getNetworkInfo'
// req.url = req.path
// req.method = 'GET'
//
// req.locals.jwtToken = 'some-token'
//
// next.reset() // reset the stubbed next() function.
//
// await rateLimits.rateLimitByResource(req, res, next)
//
// // Issues with token secret should treat incoming requests as anonymous
// // calls with 50 points, or 20 RPM.
// assert.equal(res.locals.pointsToConsume, 50)
// })
})
describe('#isInWhitelist', () => {
it('should return false when no argument is passed in', () => {
const result = rateLimits.isInWhitelist()
assert.equal(result, false)
})
it('should return false when origin is not in the whitelist', () => {
const origin = 'blah.com'
const result = rateLimits.isInWhitelist(origin)
assert.equal(result, false)
})
it('should return true when origin is in the whitelist', () => {
const origin = 'message.fullstack.cash'
const result = rateLimits.isInWhitelist(origin)
assert.equal(result, true)
})
})
})
// Generates a Basic authorization header.
// function generateAuthHeader (pass) {
// // https://en.wikipedia.org/wiki/Basic_access_authentication
// const username = 'BITBOX'
// const combined = `${username}:${pass}`
//
// var base64Credential = Buffer.from(combined).toString('base64')
// var readyCredential = `Basic ${base64Credential}`
//
// return readyCredential
// }