From 7339b8554f4ea972e464c3519ca917c49b4c6809 Mon Sep 17 00:00:00 2001 From: Chris Troutner Date: Thu, 1 Oct 2020 17:25:52 -0700 Subject: [PATCH 1/6] fix(basic auth): Disabling Basic Authentication --- src/middleware/auth.js | 92 +++++++++--------- src/middleware/route-ratelimit.js | 22 ++--- test/v3/integration/rate-limits.js | 151 +++++++++++++++-------------- 3 files changed, 133 insertions(+), 132 deletions(-) diff --git a/src/middleware/auth.js b/src/middleware/auth.js index 128e9ba..32fae28 100644 --- a/src/middleware/auth.js +++ b/src/middleware/auth.js @@ -22,7 +22,7 @@ 'use strict' const passport = require('passport') -const BasicStrategy = require('passport-http').BasicStrategy +// const BasicStrategy = require('passport-http').BasicStrategy const AnonymousStrategy = require('passport-anonymous') const wlogger = require('../util/winston-logging') @@ -48,55 +48,55 @@ class AuthMW { passport.use(new AnonymousStrategy()) // Initialize passport for 'basic' authentication. - passport.use( - new BasicStrategy({ passReqToCallback: true }, function ( - req, - username, - password, - done - ) { - // console.log(`req: ${util.inspect(req)}`) - // console.log(`username: ${username}`) - // console.log(`password: ${password}`) - - // Create the req.locals property if it does not yet exist. - if (!req.locals) { - req.locals = { - // default values - proLimit: false, - apiLevel: 0 - } - } - - // Set pro-tier rate limit to flag to false by default. - req.locals.proLimit = false - - // Evaluate the username and password and set the rate limit accordingly. - // if (username === "BITBOX" && password === PRO_PASS) { - if (username === 'BITBOX') { - for (let i = 0; i < PRO_PASS.length; i++) { - const thisPass = PRO_PASS[i] - - if (password === thisPass) { - wlogger.verbose(`${req.url} called by ${password.slice(0, 6)}`) - - // Success - req.locals.proLimit = true - break - } - } - } - - // console.log(`req.locals: ${util.inspect(req.locals)}`) - - return done(null, true) - }) - ) + // passport.use( + // new BasicStrategy({ passReqToCallback: true }, function ( + // req, + // username, + // password, + // done + // ) { + // // console.log(`req: ${util.inspect(req)}`) + // // console.log(`username: ${username}`) + // // console.log(`password: ${password}`) + // + // // Create the req.locals property if it does not yet exist. + // if (!req.locals) { + // req.locals = { + // // default values + // proLimit: false, + // apiLevel: 0 + // } + // } + // + // // Set pro-tier rate limit to flag to false by default. + // req.locals.proLimit = false + // + // // Evaluate the username and password and set the rate limit accordingly. + // // if (username === "BITBOX" && password === PRO_PASS) { + // if (username === 'BITBOX') { + // for (let i = 0; i < PRO_PASS.length; i++) { + // const thisPass = PRO_PASS[i] + // + // if (password === thisPass) { + // wlogger.verbose(`${req.url} called by ${password.slice(0, 6)}`) + // + // // Success + // req.locals.proLimit = true + // break + // } + // } + // } + // + // // console.log(`req.locals: ${util.inspect(req.locals)}`) + // + // return done(null, true) + // }) + // ) } // Middleware called by the route. mw () { - return passport.authenticate(['basic', 'anonymous'], { + return passport.authenticate(['anonymous'], { session: false }) } diff --git a/src/middleware/route-ratelimit.js b/src/middleware/route-ratelimit.js index a53b52d..2576500 100644 --- a/src/middleware/route-ratelimit.js +++ b/src/middleware/route-ratelimit.js @@ -20,7 +20,7 @@ const redisClient = new Redis(redisOptions) const { RateLimiterRedis } = require('rate-limiter-flexible') const rateLimitOptions = { storeClient: redisClient, - points: 100, // Number of points + points: 1000, // Number of points duration: 60 // Per minute (per 60 seconds) } @@ -98,8 +98,8 @@ class RateLimits { wlogger.debug('No JWT token found!') } - // Used for displaying error message. Default value is 3. - let rateLimit = 3 + // Used for displaying error message. Default value is 30. + let rateLimit = 30 // Code here for the rate limiter is adapted from this example: // https://github.com/animir/node-rate-limiter-flexible/wiki/Overall-example#authorized-and-not-authorized-users @@ -131,7 +131,7 @@ class RateLimits { origin.toString().indexOf('sandbox.fullstack.cash') > -1 || origin === 'slp-api') ) { - pointsToConsume = 1 + pointsToConsume = 10 res.locals.pointsToConsume = pointsToConsume // Feedback for tests. } @@ -145,7 +145,7 @@ class RateLimits { `User ${key} consuming ${pointsToConsume} point for resource ${resource}.` ) - rateLimit = Math.floor(100 / pointsToConsume) + rateLimit = Math.floor(1000 / pointsToConsume) // Update the key so that rate limits track both the user and the resource. key = `${key}-${resource}` @@ -175,7 +175,7 @@ class RateLimits { // Calculates the points consumed, based on the jwt information and the route // requested. calcPoints (jwtInfo) { - let retVal = 30 // By default, use anonymous tier. + let retVal = 300 // By default, use anonymous tier. try { // console.log(`jwtInfo: ${JSON.stringify(jwtInfo, null, 2)}`) @@ -194,20 +194,20 @@ class RateLimits { if (level40Routes.includes(resource)) { if (apiLevel >= 40) retVal = 1 // else if (apiLevel >= 10) retVal = 10 - else retVal = 10 + else retVal = 100 // Normal indexer routes } else if (level30Routes.includes(resource)) { if (apiLevel >= 30) retVal = 1 - else retVal = 10 + else retVal = 100 // Full node tier } else if (apiLevel >= 20) { - retVal = 1 + retVal = 10 // Free tier, full node only. } else { - retVal = 10 + retVal = 100 } } @@ -215,7 +215,7 @@ class RateLimits { } catch (err) { wlogger.error('Error in route-ratelimit.js/calcPoints()') // throw err - retVal = 30 + retVal = 300 } return retVal diff --git a/test/v3/integration/rate-limits.js b/test/v3/integration/rate-limits.js index c26790b..cb4c65f 100644 --- a/test/v3/integration/rate-limits.js +++ b/test/v3/integration/rate-limits.js @@ -16,9 +16,10 @@ util.inspect.defaultOptions = { depth: 1 } // const SERVER = `http://192.168.0.36:12400/v3/` const SERVER = 'http://localhost:3000/v3/' +// const SERVER = 'https://api.fullstack.cash/v3/' const TEST_JWT = - 'eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6IjVlM2EwNDE1ZWIyOWE5NjJkYTI3MDhiNCIsImFwaUxldmVsIjowLCJyYXRlTGltaXQiOjEwLCJpYXQiOjE1ODA4NjA0NjcsImV4cCI6MTU4MzQ1MjQ2N30.fuY5S-YrF0J11h5uyMjPe7wiVkYRnIyXi4dL9-V-C6pLJm33p0dSq_pSheVVWw78n5kAvL_9kFHngbnmQiOJYQ' + 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6IjVlODhhY2JmMDIyMWMxMDAxMmFkOTNmZiIsImVtYWlsIjoiY2hyaXMudHJvdXRuZXJAZ21haWwuY29tIiwiYXBpTGV2ZWwiOjQwLCJyYXRlTGltaXQiOjMsImlhdCI6MTYwMDYyODk1MSwiZXhwIjoxNjAzMjIwOTUxfQ.JPXDJQsxJFtCGZjHOd-hRfJuY41Ef_FQ4ET06CtYdNk' describe('#rate limits', () => { it('should get control/getNetworkInfo() with no auth', async () => { @@ -35,7 +36,7 @@ describe('#rate limits', () => { assert.hasAnyKeys(result.data, ['version']) }) - it('should trigger rate-limit handler if rate limits exceeds 30 request per minute', async () => { + it('should trigger rate-limit handler if rate limits exceeds 5 request per minute', async () => { try { // Actual rate limit is 60 per minute X 4 nodes = 240 rpm. const options = { @@ -44,7 +45,7 @@ describe('#rate limits', () => { } const promises = [] - for (let i = 0; i < 30; i++) { + for (let i = 0; i < 5; i++) { const promise = axios(options) promises.push(promise) } @@ -60,79 +61,79 @@ describe('#rate limits', () => { } }) - it('should not trigger rate-limit handler if correct pro-tier password is used', async () => { - try { - const username = 'BITBOX' + // it('should not trigger rate-limit handler if correct pro-tier password is used', async () => { + // try { + // const username = 'BITBOX' + // + // // Pro-tier is accessed by using the right password. + // const password = 'BITBOX' + // // const password = "something" + // + // const combined = `${username}:${password}` + // const base64Credential = Buffer.from(combined).toString('base64') + // const readyCredential = `Basic ${base64Credential}` + // + // const options = { + // method: 'GET', + // url: `${SERVER}control/getNetworkInfo`, + // headers: { Authorization: readyCredential } + // } + // + // const promises = [] + // for (let i = 0; i < 30; i++) { + // const promise = axios(options) + // promises.push(promise) + // } + // + // await Promise.all(promises) + // + // assert.equal(true, true, 'Not throwing an error is a pass!') + // } catch (err) { + // // console.log(`err.response: ${util.inspect(err.response)}`) + // + // assert.equal( + // true, + // false, + // 'This error handler should not have been triggered. Is the password correct?' + // ) + // } + // }) - // Pro-tier is accessed by using the right password. - const password = 'BITBOX' - // const password = "something" - - const combined = `${username}:${password}` - const base64Credential = Buffer.from(combined).toString('base64') - const readyCredential = `Basic ${base64Credential}` - - const options = { - method: 'GET', - url: `${SERVER}control/getNetworkInfo`, - headers: { Authorization: readyCredential } - } - - const promises = [] - for (let i = 0; i < 30; i++) { - const promise = axios(options) - promises.push(promise) - } - - await Promise.all(promises) - - assert.equal(true, true, 'Not throwing an error is a pass!') - } catch (err) { - // console.log(`err.response: ${util.inspect(err.response)}`) - - assert.equal( - true, - false, - 'This error handler should not have been triggered. Is the password correct?' - ) - } - }) - - it('should trigger rate-limit handler if rate limits exceeds pro-tier limit', async () => { - try { - const username = 'BITBOX' - - // Pro-tier is accessed by using the right password. - const password = 'BITBOX' - // const password = "something" - - const combined = `${username}:${password}` - const base64Credential = Buffer.from(combined).toString('base64') - const readyCredential = `Basic ${base64Credential}` - - // Actual rate limit is 60 per minute X 4 nodes = 240 rpm. - const options = { - method: 'GET', - url: `${SERVER}control/getNetworkInfo`, - headers: { Authorization: readyCredential } - } - - const promises = [] - for (let i = 0; i < 80; i++) { - const promise = axios(options) - promises.push(promise) - } - - await Promise.all(promises) - - assert.equal(true, false, 'Unexpected result!') - } catch (err) { - // console.log(`err.response: ${util.inspect(err.response)}`) - - assert.equal(err.response.status, 429) - assert.include(err.response.data.error, 'Too many requests') - } - }) + // it('should trigger rate-limit handler if rate limits exceeds pro-tier limit', async () => { + // try { + // const username = 'BITBOX' + // + // // Pro-tier is accessed by using the right password. + // const password = 'BITBOX' + // // const password = "something" + // + // const combined = `${username}:${password}` + // const base64Credential = Buffer.from(combined).toString('base64') + // const readyCredential = `Basic ${base64Credential}` + // + // // Actual rate limit is 60 per minute X 4 nodes = 240 rpm. + // const options = { + // method: 'GET', + // url: `${SERVER}control/getNetworkInfo`, + // headers: { Authorization: readyCredential } + // } + // + // const promises = [] + // for (let i = 0; i < 80; i++) { + // const promise = axios(options) + // promises.push(promise) + // } + // + // await Promise.all(promises) + // + // assert.equal(true, false, 'Unexpected result!') + // } catch (err) { + // // console.log(`err.response: ${util.inspect(err.response)}`) + // + // assert.equal(err.response.status, 429) + // assert.include(err.response.data.error, 'Too many requests') + // } + // }) it('should unlock pro-tier for a valid JWT token', async () => { try { From d56e6d6b3a8c1a78e5952f3fa1400a2bcc5e5cf3 Mon Sep 17 00:00:00 2001 From: Chris Troutner Date: Thu, 1 Oct 2020 17:26:31 -0700 Subject: [PATCH 2/6] linting --- src/middleware/auth.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/middleware/auth.js b/src/middleware/auth.js index 32fae28..34b6a4a 100644 --- a/src/middleware/auth.js +++ b/src/middleware/auth.js @@ -24,7 +24,7 @@ const passport = require('passport') // const BasicStrategy = require('passport-http').BasicStrategy const AnonymousStrategy = require('passport-anonymous') -const wlogger = require('../util/winston-logging') +// const wlogger = require('../util/winston-logging') // Used for debugging and iterrogating JS objects. const util = require('util') @@ -33,9 +33,9 @@ util.inspect.defaultOptions = { depth: 1 } // let _this // Set default rate limit value for testing -const PRO_PASSES = process.env.PRO_PASS ? process.env.PRO_PASS : 'BITBOX' +// const PRO_PASSES = process.env.PRO_PASS ? process.env.PRO_PASS : 'BITBOX' // Convert the pro-tier password string into an array split by ':'. -const PRO_PASS = PRO_PASSES.split(':') +// const PRO_PASS = PRO_PASSES.split(':') // wlogger.verbose(`PRO_PASS set to: ${PRO_PASS}`) From 86bf504b782ceef4ad8c7c5c09b268805ef44fdd Mon Sep 17 00:00:00 2001 From: Chris Troutner Date: Thu, 1 Oct 2020 17:34:36 -0700 Subject: [PATCH 3/6] Fixing unit tests --- src/middleware/route-ratelimit.js | 4 +- test/v3/rate-limits.js | 70 +++++++++++++++---------------- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/src/middleware/route-ratelimit.js b/src/middleware/route-ratelimit.js index 2576500..7b8c815 100644 --- a/src/middleware/route-ratelimit.js +++ b/src/middleware/route-ratelimit.js @@ -192,13 +192,13 @@ class RateLimits { if (jwtInfo.id) { // SLP indexer routes if (level40Routes.includes(resource)) { - if (apiLevel >= 40) retVal = 1 + if (apiLevel >= 40) retVal = 10 // else if (apiLevel >= 10) retVal = 10 else retVal = 100 // Normal indexer routes } else if (level30Routes.includes(resource)) { - if (apiLevel >= 30) retVal = 1 + if (apiLevel >= 30) retVal = 10 else retVal = 100 // Full node tier diff --git a/test/v3/rate-limits.js b/test/v3/rate-limits.js index 02a1530..e47423d 100644 --- a/test/v3/rate-limits.js +++ b/test/v3/rate-limits.js @@ -105,14 +105,14 @@ describe('#route-ratelimits & jwt-auth', () => { }) describe('#calcPoints', () => { - it('should return 30 points for anonymous user', () => { + it('should return 300 points for anonymous user', () => { const result = rateLimits.calcPoints() // console.log(`result: ${result}`) - assert.equal(result, 30) + assert.equal(result, 300) }) - it('should return 10 points for free tier requesting full node access', () => { + it('should return 100 points for free tier requesting full node access', () => { const jwtInfo = { apiLevel: 10, resource: 'blockchain', @@ -120,10 +120,10 @@ describe('#route-ratelimits & jwt-auth', () => { } const result = rateLimits.calcPoints(jwtInfo) - assert.equal(result, 10) + assert.equal(result, 100) }) - it('should return 10 points for free tier requesting indexer access', () => { + it('should return 100 points for free tier requesting indexer access', () => { const jwtInfo = { apiLevel: 10, resource: 'blockbook', @@ -131,10 +131,10 @@ describe('#route-ratelimits & jwt-auth', () => { } const result = rateLimits.calcPoints(jwtInfo) - assert.equal(result, 10) + assert.equal(result, 100) }) - it('should return 10 points for free tier requesting SLPDB access', () => { + it('should return 100 points for free tier requesting SLPDB access', () => { const jwtInfo = { apiLevel: 10, resource: 'slp', @@ -142,10 +142,10 @@ describe('#route-ratelimits & jwt-auth', () => { } const result = rateLimits.calcPoints(jwtInfo) - assert.equal(result, 10) + assert.equal(result, 100) }) - it('should return 1 point for full node tier requesting full node access', () => { + it('should return 10 points for full node tier requesting full node access', () => { const jwtInfo = { apiLevel: 20, resource: 'blockchain', @@ -153,10 +153,10 @@ describe('#route-ratelimits & jwt-auth', () => { } const result = rateLimits.calcPoints(jwtInfo) - assert.equal(result, 1) + assert.equal(result, 10) }) - it('should return 10 points for full-node tier requesting indexer access', () => { + it('should return 100 points for full-node tier requesting indexer access', () => { const jwtInfo = { apiLevel: 20, resource: 'blockbook', @@ -164,10 +164,10 @@ describe('#route-ratelimits & jwt-auth', () => { } const result = rateLimits.calcPoints(jwtInfo) - assert.equal(result, 10) + assert.equal(result, 100) }) - it('should return 10 points for full node tier requesting SLPDB access', () => { + it('should return 100 points for full node tier requesting SLPDB access', () => { const jwtInfo = { apiLevel: 20, resource: 'slp', @@ -175,10 +175,10 @@ describe('#route-ratelimits & jwt-auth', () => { } const result = rateLimits.calcPoints(jwtInfo) - assert.equal(result, 10) + assert.equal(result, 100) }) - it('should return 1 point for indexer tier requesting full node access', () => { + it('should return 10 point for indexer tier requesting full node access', () => { const jwtInfo = { apiLevel: 30, resource: 'blockchain', @@ -186,32 +186,32 @@ describe('#route-ratelimits & jwt-auth', () => { } const result = rateLimits.calcPoints(jwtInfo) - assert.equal(result, 1) + assert.equal(result, 10) }) - it('should return 1 points for indexer tier requesting indexer access', () => { + it('should return 10 points for indexer tier requesting indexer access', () => { const jwtInfo = { apiLevel: 30, resource: 'blockbook', id: '5e3a0415eb29a962da2708b4' } - const result = rateLimits.calcPoints(jwtInfo) - assert.equal(result, 1) - }) - - it('should return 10 points for indexer tier requesting SLPDB access', () => { - const jwtInfo = { - apiLevel: 30, - resource: 'slp', - id: '5e3a0415eb29a962da2708b4' - } - const result = rateLimits.calcPoints(jwtInfo) assert.equal(result, 10) }) - it('should return 1 point for SLP tier requesting full node access', () => { + it('should return 100 points for indexer tier requesting SLPDB access', () => { + const jwtInfo = { + apiLevel: 30, + resource: 'slp', + id: '5e3a0415eb29a962da2708b4' + } + + const result = rateLimits.calcPoints(jwtInfo) + assert.equal(result, 100) + }) + + it('should return 10 point for SLP tier requesting full node access', () => { const jwtInfo = { apiLevel: 40, resource: 'blockchain', @@ -219,10 +219,10 @@ describe('#route-ratelimits & jwt-auth', () => { } const result = rateLimits.calcPoints(jwtInfo) - assert.equal(result, 1) + assert.equal(result, 10) }) - it('should return 1 points for SLP tier requesting indexer access', () => { + it('should return 10 points for SLP tier requesting indexer access', () => { const jwtInfo = { apiLevel: 40, resource: 'blockbook', @@ -230,10 +230,10 @@ describe('#route-ratelimits & jwt-auth', () => { } const result = rateLimits.calcPoints(jwtInfo) - assert.equal(result, 1) + assert.equal(result, 10) }) - it('should return 1 points for SLP tier requesting SLPDB access', () => { + it('should return 10 points for SLP tier requesting SLPDB access', () => { const jwtInfo = { apiLevel: 40, resource: 'slp', @@ -241,7 +241,7 @@ describe('#route-ratelimits & jwt-auth', () => { } const result = rateLimits.calcPoints(jwtInfo) - assert.equal(result, 1) + assert.equal(result, 10) }) }) @@ -442,7 +442,7 @@ describe('#route-ratelimits & jwt-auth', () => { // Issues with token secret should treat incoming requests as anonymous // calls with 30 points or 3 RPM. - assert.equal(res.locals.pointsToConsume, 30) + assert.equal(res.locals.pointsToConsume, 300) }) }) }) From 41a3cd91fcdd622e15d82ca28a290d4a9ea4505c Mon Sep 17 00:00:00 2001 From: Chris Troutner Date: Thu, 1 Oct 2020 20:20:28 -0700 Subject: [PATCH 4/6] fix(hydrateUtxos): Using local calls to the REST API --- src/routes/v3/slp.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/routes/v3/slp.js b/src/routes/v3/slp.js index 3a7ef33..accd8c8 100644 --- a/src/routes/v3/slp.js +++ b/src/routes/v3/slp.js @@ -13,8 +13,11 @@ const Slpdb = require('./services/slpdb') // const strftime = require('strftime') const wlogger = require('../../util/winston-logging') +const LOCAL_RESTURL = process.env.LOCAL_RESTURL + ? process.env.LOCAL_RESTURL + : 'https://api.fullstack.cash/v3/' const BCHJS = require('@psf/bch-js') -const bchjs = new BCHJS() +const bchjs = new BCHJS({ restURL: LOCAL_RESTURL }) // Used to convert error messages to strings, to safely pass to users. const util = require('util') From b4823be86ae2d2c55f84b89cdb14220c5a3d5853 Mon Sep 17 00:00:00 2001 From: Chris Troutner Date: Thu, 1 Oct 2020 20:46:49 -0700 Subject: [PATCH 5/6] fix(hydrateUtxos): Adding localhost to max rate limit --- src/middleware/route-ratelimit.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/middleware/route-ratelimit.js b/src/middleware/route-ratelimit.js index 7b8c815..2f6dfad 100644 --- a/src/middleware/route-ratelimit.js +++ b/src/middleware/route-ratelimit.js @@ -136,7 +136,10 @@ class RateLimits { } // Apply paid-access rate limits based on key/IP - if (key.toString().indexOf('172.17.') > -1) { + if ( + key.toString().indexOf('172.17.') > -1 || + key.toString.indexOf('::ffff:127.0.0.1') > -1 + ) { pointsToConsume = 1 res.locals.pointsToConsume = pointsToConsume // Feedback for tests. } From c664758654bc94d136b93062f976bf2e575490b9 Mon Sep 17 00:00:00 2001 From: Chris Troutner Date: Thu, 1 Oct 2020 20:51:01 -0700 Subject: [PATCH 6/6] Fixing bug --- src/middleware/route-ratelimit.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/middleware/route-ratelimit.js b/src/middleware/route-ratelimit.js index 2f6dfad..4ca8510 100644 --- a/src/middleware/route-ratelimit.js +++ b/src/middleware/route-ratelimit.js @@ -138,7 +138,7 @@ class RateLimits { // Apply paid-access rate limits based on key/IP if ( key.toString().indexOf('172.17.') > -1 || - key.toString.indexOf('::ffff:127.0.0.1') > -1 + key.toString().indexOf('::ffff:127.0.0.1') > -1 ) { pointsToConsume = 1 res.locals.pointsToConsume = pointsToConsume // Feedback for tests.