/* Sets the rate limits for the anonymous and paid tiers. Current rate limits: - 1000 points in 60 seconds - 10 points per call for paid tier (100 RPM) - 50 points per call for anonymous tier (20 RPM) Background: The rate limits below were originially coded with the idea of charging on a per-resource basis. However, that was confusing to end users trying to purchase a subscription. So everything was simplied to two tiers: paid and anonymous CT 3/4/21: I increased the total points from 1,000 to 100,000 to prevent systems with Basic Authentication from hitting internal rate limits when calling hydrateUtxos(). */ 'use strict' // Public npm libraries. const jwt = require('jsonwebtoken') // local libraries. const wlogger = require('../util/winston-logging') const config = require('../../config') // Hard coding limits since basic-authentiation is assumed to be the primary access. const ANON_LIMITS = 333 const WHITELIST_RATE_LIMIT = config.whitelistRateLimit const WHITELIST_DOMAINS = config.whitelistDomains const INTERNAL_RATE_LIMIT = 1 // Redis const redisOptions = { enableOfflineQueue: false, port: process.env.REDIS_PORT ? process.env.REDIS_PORT : 6379, host: process.env.REDIS_HOST ? process.env.REDIS_HOST : '127.0.0.1' } console.log(`redisOptions: ${JSON.stringify(redisOptions, null, 2)}`) const Redis = require('ioredis') const redisClient = new Redis(redisOptions) // Rate limiter middleware lib. const { RateLimiterRedis } = require('rate-limiter-flexible') const rateLimitOptions = { storeClient: redisClient, points: 1000, // Number of points duration: 60 // Per minute (per 60 seconds) } let _this class RateLimits { constructor () { _this = this this.jwt = jwt this.rateLimiter = new RateLimiterRedis(rateLimitOptions) this.config = config } // Used to disconnect from the Redis DB. // Called by unit tests so that node.js thread doesn't live forever. closeRedis () { redisClient.disconnect() } async wipeRedis () { await redisClient.flushdb() } // This is the new rate limit function that uses the rate-limiter-flexible npm // library. It uses fine-grain rate limiting based on the resources being // consumed. async rateLimitByResource (req, res, next) { try { let userId let decoded = {} // Create a req.locals object if not passed in. if (!req.locals) { req.locals = { // default values jwtToken: '', proLimit: false, apiLevel: 0 } } // Create a res.locals object if it does not exist. This is used for // debugging. if (!res.locals) { res.locals = { rateLimitTriggered: false } } // Decode the JWT token if one exists. if (req.locals.jwtToken) { try { decoded = _this.jwt.verify( req.locals.jwtToken, _this.config.apiTokenSecret ) // console.log(`decoded: ${JSON.stringify(decoded, null, 2)}`) userId = decoded.id } catch (err) { // This handler will be triggered if the JWT token does not match the // token secret. wlogger.error( `Last three letters of token secret: ${_this.config.apiTokenSecret.slice( -3 )}` ) wlogger.error( 'Error trying to decode JWT token in route-ratelimit.js/newRateLimit(): ', err ) } // } else if (req.body && req.body.usrObj) { // Same as above, but this code path is activated from internal calls to // bch-js, like hydrateUtxo(), which passes the user object from the // original API call. try { decoded = _this.jwt.verify( req.body.usrObj.jwtToken, _this.config.apiTokenSecret ) // console.log(`decoded: ${JSON.stringify(decoded, null, 2)}`) userId = decoded.id } catch (err) { // This handler will be triggered if the JWT token does not match the // token secret. wlogger.error( 'Error in route-ratelimit.js trying to decode JWT token in usrObj' ) } } else { wlogger.debug('No JWT token found!') } // Default value is 50 points per request = 20 RPM let rateLimit = ANON_LIMITS // Only evaluate the JWT token if the user is not using Basic Authentication. if (!req.locals.proLimit && !req.body.usrObj.proLimit) { // Code here for the rate limiter is adapted from this example: // https://github.com/animir/node-rate-limiter-flexible/wiki/Overall-example#authorized-and-not-authorized-users try { // The resource being consumed: full node, indexer, SLPDB, etc. const resource = _this.getResource(req.url) wlogger.debug(`resource: ${resource}`) // Key will be the JWT ID if it exists, otherwise the IP address of the caller. let key = userId || req.ip res.locals.key = key // Feedback for tests. // console.log(`key: ${key}`) // const pointsToConsume = userId ? 1 : 30 decoded.resource = resource let pointsToConsume = _this.calcPoints(decoded) res.locals.pointsToConsume = pointsToConsume // Feedback for tests. // Retrieve the origin. let origin = req.get('origin') // Handle calls coming from the intranet. if (origin === undefined && key.indexOf('10.0.0.5') > -1) { origin = 'slp-api' } wlogger.info(`origin: ${origin}`) // If the request originates from one of the approved wallet apps, then // apply paid-access rate limits. // console.log(`origin: ${JSON.stringify(origin, null, 2)}`) // console.log(`whitelist: ${JSON.stringify(WHITELIST_DOMAINS, null, 2)}`) const isInWhitelist = _this.isInWhitelist(origin) if (isInWhitelist) { pointsToConsume = WHITELIST_RATE_LIMIT res.locals.pointsToConsume = pointsToConsume // Feedback for tests. } // For internal calls, increase rate limits to as fast as possible. if ( // Comment out the line below when running bch-js e2e rate limit tests. key.toString().indexOf('::ffff:127.0.0.1') > -1 || // Do not comment out this line. key.toString().indexOf('172.17.') > -1 ) { pointsToConsume = INTERNAL_RATE_LIMIT res.locals.pointsToConsume = pointsToConsume // Feedback for tests. } wlogger.info( `User ${key} consuming ${pointsToConsume} point for resource ${resource}.` ) rateLimit = Math.floor(100000 / pointsToConsume) // Update the key so that rate limits track both the user and the resource. key = `${key}-${resource}` await _this.rateLimiter.consume(key, pointsToConsume) } catch (err) { // console.log('err: ', err) // Used for returning data for tests. res.locals.rateLimitTriggered = true // console.log('res.locals: ', res.locals) // Rate limited was triggered res.status(429) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330 return res.json({ error: `Too many requests. Your limits are currently ${rateLimit} requests per minute. Increase rate limits at https://fullstack.cash` }) } } } catch (err) { wlogger.error('Error in route-ratelimit.js/newRateLimit(): ', err) // throw err } next() } // Calculates the points consumed, based on the jwt information and the route // requested. calcPoints (jwtInfo) { let retVal = ANON_LIMITS // By default, use anonymous tier. try { // console.log(`jwtInfo: ${JSON.stringify(jwtInfo, null, 2)}`) const apiLevel = jwtInfo.apiLevel const resource = jwtInfo.resource const level30Routes = ['insight', 'bitcore', 'blockbook', 'electrumx'] const level40Routes = ['slp'] wlogger.debug(`apiLevel: ${apiLevel}`) // Only evaluate if user is using a JWT token. if (jwtInfo.id) { // SLP indexer routes if (level40Routes.includes(resource)) { if (apiLevel >= 40) retVal = 10 // else if (apiLevel >= 10) retVal = 10 else retVal = ANON_LIMITS // Normal indexer routes } else if (level30Routes.includes(resource)) { if (apiLevel >= 30) retVal = 10 else retVal = ANON_LIMITS // Full node tier } else if (apiLevel >= 20) { retVal = 10 // Free tier, full node only. } else { retVal = ANON_LIMITS } } return retVal } catch (err) { wlogger.error('Error in route-ratelimit.js/calcPoints()') // throw err retVal = ANON_LIMITS } return retVal } // This function parses the req.url property to identify what resource // the user is requesting. // This was created as a function so that it can be unit tested. Not sure // what kind of variations will be seen in production. getResource (url) { try { wlogger.debug(`url: ${JSON.stringify(url, null, 2)}`) const splitUrl = url.split('/') const resource = splitUrl[1] return resource } catch (err) { wlogger.error('Error in getResource().') throw err } } // Returns a boolean if the origin of the request matches a domain in the // whitelist. isInWhitelist (origin) { try { const retVal = false // Default value. if (!origin) return false // console.log(`WHITELIST_DOMAINS: ${JSON.stringify(WHITELIST_DOMAINS, null, 2)}`) for (let i = 0; i < WHITELIST_DOMAINS.length; i++) { const thisDomain = WHITELIST_DOMAINS[i] if (origin.toString().indexOf(thisDomain) > -1) { return true } } return retVal } catch (err) { wlogger.error( 'Error in route-ratelimit.js/isInWhitelist(). Returning false by default.' ) return false } } } module.exports = RateLimits