Merge pull request #38 from Permissionless-Software-Foundation/ct-unstable

fix(hydrateUtxos): Adding localhost to max rate limit

package-lock.json

@@ -366,9 +366,9 @@
       }
     },
     "@psf/bch-js": {
-      "version": "3.6.2",
+      "version": "3.6.3",
-      "resolved": "https://registry.npmjs.org/@psf/bch-js/-/bch-js-3.6.2.tgz",
+      "resolved": "https://registry.npmjs.org/@psf/bch-js/-/bch-js-3.6.3.tgz",
-      "integrity": "sha512-6r8NBB48WgWHAFeuwrlkaE3B87TT3o/8hdjU1DOWXUx7lMFIF1Aq25CffUxF3sHP0pSx3/+4ulQZFROHGTwnDQ==",
+      "integrity": "sha512-0nqmLprZU+1lh1WTnAqVHQD+bIPJIcEQ36rBNoyh4ejzWZoQ7pTMj2r2I7JGhWmxWdSQ+g4stT1OrC4qmH6jdw==",
       "requires": {
         "@uppy/core": "^1.10.4",
         "@uppy/tus": "^1.5.12",

package.json

@@ -28,7 +28,7 @@
     "node": ">=10.15.1"
   },
   "dependencies": {
-    "@psf/bch-js": "^3.6.2",
+    "@psf/bch-js": "^3.6.3",
     "apidoc": "^0.23.0",
     "axios": "^0.19.0",
     "bitcore-lib-cash": "^8.20.3",

src/middleware/auth.js

@@ -22,9 +22,9 @@
 'use strict'
 
 const passport = require('passport')
-const BasicStrategy = require('passport-http').BasicStrategy
+// const BasicStrategy = require('passport-http').BasicStrategy
 const AnonymousStrategy = require('passport-anonymous')
-const wlogger = require('../util/winston-logging')
+// const wlogger = require('../util/winston-logging')
 
 // Used for debugging and iterrogating JS objects.
 const util = require('util')
@@ -33,9 +33,9 @@ util.inspect.defaultOptions = { depth: 1 }
 // let _this
 
 // Set default rate limit value for testing
-const PRO_PASSES = process.env.PRO_PASS ? process.env.PRO_PASS : 'BITBOX'
+// const PRO_PASSES = process.env.PRO_PASS ? process.env.PRO_PASS : 'BITBOX'
 // Convert the pro-tier password string into an array split by ':'.
-const PRO_PASS = PRO_PASSES.split(':')
+// const PRO_PASS = PRO_PASSES.split(':')
 
 // wlogger.verbose(`PRO_PASS set to: ${PRO_PASS}`)
 
@@ -48,55 +48,55 @@ class AuthMW {
     passport.use(new AnonymousStrategy())
 
     // Initialize passport for 'basic' authentication.
-    passport.use(
+    //   passport.use(
-      new BasicStrategy({ passReqToCallback: true }, function (
+    //     new BasicStrategy({ passReqToCallback: true }, function (
-        req,
+    //       req,
-        username,
+    //       username,
-        password,
+    //       password,
-        done
+    //       done
-      ) {
+    //     ) {
-        // console.log(`req: ${util.inspect(req)}`)
+    //       // console.log(`req: ${util.inspect(req)}`)
-        // console.log(`username: ${username}`)
+    //       // console.log(`username: ${username}`)
-        // console.log(`password: ${password}`)
+    //       // console.log(`password: ${password}`)
-
+    //
-        // Create the req.locals property if it does not yet exist.
+    //       // Create the req.locals property if it does not yet exist.
-        if (!req.locals) {
+    //       if (!req.locals) {
-          req.locals = {
+    //         req.locals = {
-            // default values
+    //           // default values
-            proLimit: false,
+    //           proLimit: false,
-            apiLevel: 0
+    //           apiLevel: 0
-          }
+    //         }
-        }
+    //       }
-
+    //
-        // Set pro-tier rate limit to flag to false by default.
+    //       // Set pro-tier rate limit to flag to false by default.
-        req.locals.proLimit = false
+    //       req.locals.proLimit = false
-
+    //
-        // Evaluate the username and password and set the rate limit accordingly.
+    //       // Evaluate the username and password and set the rate limit accordingly.
-        // if (username === "BITBOX" && password === PRO_PASS) {
+    //       // if (username === "BITBOX" && password === PRO_PASS) {
-        if (username === 'BITBOX') {
+    //       if (username === 'BITBOX') {
-          for (let i = 0; i < PRO_PASS.length; i++) {
+    //         for (let i = 0; i < PRO_PASS.length; i++) {
-            const thisPass = PRO_PASS[i]
+    //           const thisPass = PRO_PASS[i]
-
+    //
-            if (password === thisPass) {
+    //           if (password === thisPass) {
-              wlogger.verbose(`${req.url} called by ${password.slice(0, 6)}`)
+    //             wlogger.verbose(`${req.url} called by ${password.slice(0, 6)}`)
-
+    //
-              // Success
+    //             // Success
-              req.locals.proLimit = true
+    //             req.locals.proLimit = true
-              break
+    //             break
-            }
+    //           }
-          }
+    //         }
-        }
+    //       }
-
+    //
-        // console.log(`req.locals: ${util.inspect(req.locals)}`)
+    //       // console.log(`req.locals: ${util.inspect(req.locals)}`)
-
+    //
-        return done(null, true)
+    //       return done(null, true)
-      })
+    //     })
-    )
+    //   )
   }
 
   // Middleware called by the route.
   mw () {
-    return passport.authenticate(['basic', 'anonymous'], {
+    return passport.authenticate(['anonymous'], {
       session: false
     })
   }

src/middleware/route-ratelimit.js

@@ -20,7 +20,7 @@ const redisClient = new Redis(redisOptions)
 const { RateLimiterRedis } = require('rate-limiter-flexible')
 const rateLimitOptions = {
   storeClient: redisClient,
-  points: 100, // Number of points
+  points: 1000, // Number of points
   duration: 60 // Per minute (per 60 seconds)
 }
 
@@ -98,8 +98,8 @@ class RateLimits {
         wlogger.debug('No JWT token found!')
       }
 
-      // Used for displaying error message. Default value is 3.
+      // Used for displaying error message. Default value is 30.
-      let rateLimit = 3
+      let rateLimit = 30
 
       // Code here for the rate limiter is adapted from this example:
       // https://github.com/animir/node-rate-limiter-flexible/wiki/Overall-example#authorized-and-not-authorized-users
@@ -131,12 +131,15 @@ class RateLimits {
             origin.toString().indexOf('sandbox.fullstack.cash') > -1 ||
             origin === 'slp-api')
         ) {
-          pointsToConsume = 1
+          pointsToConsume = 10
           res.locals.pointsToConsume = pointsToConsume // Feedback for tests.
         }
 
         // Apply paid-access rate limits based on key/IP
-        if (key.toString().indexOf('172.17.') > -1) {
+        if (
+          key.toString().indexOf('172.17.') > -1 ||
+          key.toString().indexOf('::ffff:127.0.0.1') > -1
+        ) {
           pointsToConsume = 1
           res.locals.pointsToConsume = pointsToConsume // Feedback for tests.
         }
@@ -145,7 +148,7 @@ class RateLimits {
           `User ${key} consuming ${pointsToConsume} point for resource ${resource}.`
         )
 
-        rateLimit = Math.floor(100 / pointsToConsume)
+        rateLimit = Math.floor(1000 / pointsToConsume)
 
         // Update the key so that rate limits track both the user and the resource.
         key = `${key}-${resource}`
@@ -175,7 +178,7 @@ class RateLimits {
   // Calculates the points consumed, based on the jwt information and the route
   // requested.
   calcPoints (jwtInfo) {
-    let retVal = 30 // By default, use anonymous tier.
+    let retVal = 300 // By default, use anonymous tier.
 
     try {
       // console.log(`jwtInfo: ${JSON.stringify(jwtInfo, null, 2)}`)
@@ -192,22 +195,22 @@ class RateLimits {
       if (jwtInfo.id) {
         // SLP indexer routes
         if (level40Routes.includes(resource)) {
-          if (apiLevel >= 40) retVal = 1
+          if (apiLevel >= 40) retVal = 10
           // else if (apiLevel >= 10) retVal = 10
-          else retVal = 10
+          else retVal = 100
 
           // Normal indexer routes
         } else if (level30Routes.includes(resource)) {
-          if (apiLevel >= 30) retVal = 1
+          if (apiLevel >= 30) retVal = 10
-          else retVal = 10
+          else retVal = 100
 
           // Full node tier
         } else if (apiLevel >= 20) {
-          retVal = 1
+          retVal = 10
 
           // Free tier, full node only.
         } else {
-          retVal = 10
+          retVal = 100
         }
       }
 
@@ -215,7 +218,7 @@ class RateLimits {
     } catch (err) {
       wlogger.error('Error in route-ratelimit.js/calcPoints()')
       // throw err
-      retVal = 30
+      retVal = 300
     }
 
     return retVal

src/routes/v3/slp.js

@@ -13,8 +13,11 @@ const Slpdb = require('./services/slpdb')
 // const strftime = require('strftime')
 const wlogger = require('../../util/winston-logging')
 
+const LOCAL_RESTURL = process.env.LOCAL_RESTURL
+  ? process.env.LOCAL_RESTURL
+  : 'https://api.fullstack.cash/v3/'
 const BCHJS = require('@psf/bch-js')
-const bchjs = new BCHJS()
+const bchjs = new BCHJS({ restURL: LOCAL_RESTURL })
 
 // Used to convert error messages to strings, to safely pass to users.
 const util = require('util')

test/v3/integration/rate-limits.js

@@ -16,9 +16,10 @@ util.inspect.defaultOptions = { depth: 1 }
 
 // const SERVER = `http://192.168.0.36:12400/v3/`
 const SERVER = 'http://localhost:3000/v3/'
+// const SERVER = 'https://api.fullstack.cash/v3/'
 
 const TEST_JWT =
-  'eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6IjVlM2EwNDE1ZWIyOWE5NjJkYTI3MDhiNCIsImFwaUxldmVsIjowLCJyYXRlTGltaXQiOjEwLCJpYXQiOjE1ODA4NjA0NjcsImV4cCI6MTU4MzQ1MjQ2N30.fuY5S-YrF0J11h5uyMjPe7wiVkYRnIyXi4dL9-V-C6pLJm33p0dSq_pSheVVWw78n5kAvL_9kFHngbnmQiOJYQ'
+  'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6IjVlODhhY2JmMDIyMWMxMDAxMmFkOTNmZiIsImVtYWlsIjoiY2hyaXMudHJvdXRuZXJAZ21haWwuY29tIiwiYXBpTGV2ZWwiOjQwLCJyYXRlTGltaXQiOjMsImlhdCI6MTYwMDYyODk1MSwiZXhwIjoxNjAzMjIwOTUxfQ.JPXDJQsxJFtCGZjHOd-hRfJuY41Ef_FQ4ET06CtYdNk'
 
 describe('#rate limits', () => {
   it('should get control/getNetworkInfo() with no auth', async () => {
@@ -35,7 +36,7 @@ describe('#rate limits', () => {
     assert.hasAnyKeys(result.data, ['version'])
   })
 
-  it('should trigger rate-limit handler if rate limits exceeds 30 request per minute', async () => {
+  it('should trigger rate-limit handler if rate limits exceeds 5 request per minute', async () => {
     try {
       // Actual rate limit is 60 per minute X 4 nodes = 240 rpm.
       const options = {
@@ -44,7 +45,7 @@ describe('#rate limits', () => {
       }
 
       const promises = []
-      for (let i = 0; i < 30; i++) {
+      for (let i = 0; i < 5; i++) {
         const promise = axios(options)
         promises.push(promise)
       }
@@ -60,79 +61,79 @@ describe('#rate limits', () => {
     }
   })
 
-  it('should not trigger rate-limit handler if correct pro-tier password is used', async () => {
+  // it('should not trigger rate-limit handler if correct pro-tier password is used', async () => {
-    try {
+  //   try {
-      const username = 'BITBOX'
+  //     const username = 'BITBOX'
+  //
+  //     // Pro-tier is accessed by using the right password.
+  //     const password = 'BITBOX'
+  //     // const password = "something"
+  //
+  //     const combined = `${username}:${password}`
+  //     const base64Credential = Buffer.from(combined).toString('base64')
+  //     const readyCredential = `Basic ${base64Credential}`
+  //
+  //     const options = {
+  //       method: 'GET',
+  //       url: `${SERVER}control/getNetworkInfo`,
+  //       headers: { Authorization: readyCredential }
+  //     }
+  //
+  //     const promises = []
+  //     for (let i = 0; i < 30; i++) {
+  //       const promise = axios(options)
+  //       promises.push(promise)
+  //     }
+  //
+  //     await Promise.all(promises)
+  //
+  //     assert.equal(true, true, 'Not throwing an error is a pass!')
+  //   } catch (err) {
+  //     // console.log(`err.response: ${util.inspect(err.response)}`)
+  //
+  //     assert.equal(
+  //       true,
+  //       false,
+  //       'This error handler should not have been triggered. Is the password correct?'
+  //     )
+  //   }
+  // })
 
-      // Pro-tier is accessed by using the right password.
+  // it('should trigger rate-limit handler if rate limits exceeds pro-tier limit', async () => {
-      const password = 'BITBOX'
+  //   try {
-      // const password = "something"
+  //     const username = 'BITBOX'
-
+  //
-      const combined = `${username}:${password}`
+  //     // Pro-tier is accessed by using the right password.
-      const base64Credential = Buffer.from(combined).toString('base64')
+  //     const password = 'BITBOX'
-      const readyCredential = `Basic ${base64Credential}`
+  //     // const password = "something"
-
+  //
-      const options = {
+  //     const combined = `${username}:${password}`
-        method: 'GET',
+  //     const base64Credential = Buffer.from(combined).toString('base64')
-        url: `${SERVER}control/getNetworkInfo`,
+  //     const readyCredential = `Basic ${base64Credential}`
-        headers: { Authorization: readyCredential }
+  //
-      }
+  //     // Actual rate limit is 60 per minute X 4 nodes = 240 rpm.
-
+  //     const options = {
-      const promises = []
+  //       method: 'GET',
-      for (let i = 0; i < 30; i++) {
+  //       url: `${SERVER}control/getNetworkInfo`,
-        const promise = axios(options)
+  //       headers: { Authorization: readyCredential }
-        promises.push(promise)
+  //     }
-      }
+  //
-
+  //     const promises = []
-      await Promise.all(promises)
+  //     for (let i = 0; i < 80; i++) {
-
+  //       const promise = axios(options)
-      assert.equal(true, true, 'Not throwing an error is a pass!')
+  //       promises.push(promise)
-    } catch (err) {
+  //     }
-      // console.log(`err.response: ${util.inspect(err.response)}`)
+  //
-
+  //     await Promise.all(promises)
-      assert.equal(
+  //
-        true,
+  //     assert.equal(true, false, 'Unexpected result!')
-        false,
+  //   } catch (err) {
-        'This error handler should not have been triggered. Is the password correct?'
+  //     // console.log(`err.response: ${util.inspect(err.response)}`)
-      )
+  //
-    }
+  //     assert.equal(err.response.status, 429)
-  })
+  //     assert.include(err.response.data.error, 'Too many requests')
-
+  //   }
-  it('should trigger rate-limit handler if rate limits exceeds pro-tier limit', async () => {
+  // })
-    try {
-      const username = 'BITBOX'
-
-      // Pro-tier is accessed by using the right password.
-      const password = 'BITBOX'
-      // const password = "something"
-
-      const combined = `${username}:${password}`
-      const base64Credential = Buffer.from(combined).toString('base64')
-      const readyCredential = `Basic ${base64Credential}`
-
-      // Actual rate limit is 60 per minute X 4 nodes = 240 rpm.
-      const options = {
-        method: 'GET',
-        url: `${SERVER}control/getNetworkInfo`,
-        headers: { Authorization: readyCredential }
-      }
-
-      const promises = []
-      for (let i = 0; i < 80; i++) {
-        const promise = axios(options)
-        promises.push(promise)
-      }
-
-      await Promise.all(promises)
-
-      assert.equal(true, false, 'Unexpected result!')
-    } catch (err) {
-      // console.log(`err.response: ${util.inspect(err.response)}`)
-
-      assert.equal(err.response.status, 429)
-      assert.include(err.response.data.error, 'Too many requests')
-    }
-  })
 
   it('should unlock pro-tier for a valid JWT token', async () => {
     try {

test/v3/rate-limits.js

@@ -105,14 +105,14 @@ describe('#route-ratelimits & jwt-auth', () => {
   })
 
   describe('#calcPoints', () => {
-    it('should return 30 points for anonymous user', () => {
+    it('should return 300 points for anonymous user', () => {
       const result = rateLimits.calcPoints()
       // console.log(`result: ${result}`)
 
-      assert.equal(result, 30)
+      assert.equal(result, 300)
     })
 
-    it('should return 10 points for free tier requesting full node access', () => {
+    it('should return 100 points for free tier requesting full node access', () => {
       const jwtInfo = {
         apiLevel: 10,
         resource: 'blockchain',
@@ -120,10 +120,10 @@ describe('#route-ratelimits & jwt-auth', () => {
       }
 
       const result = rateLimits.calcPoints(jwtInfo)
-      assert.equal(result, 10)
+      assert.equal(result, 100)
     })
 
-    it('should return 10 points for free tier requesting indexer access', () => {
+    it('should return 100 points for free tier requesting indexer access', () => {
       const jwtInfo = {
         apiLevel: 10,
         resource: 'blockbook',
@@ -131,10 +131,10 @@ describe('#route-ratelimits & jwt-auth', () => {
       }
 
       const result = rateLimits.calcPoints(jwtInfo)
-      assert.equal(result, 10)
+      assert.equal(result, 100)
     })
 
-    it('should return 10 points for free tier requesting SLPDB access', () => {
+    it('should return 100 points for free tier requesting SLPDB access', () => {
       const jwtInfo = {
         apiLevel: 10,
         resource: 'slp',
@@ -142,10 +142,10 @@ describe('#route-ratelimits & jwt-auth', () => {
       }
 
       const result = rateLimits.calcPoints(jwtInfo)
-      assert.equal(result, 10)
+      assert.equal(result, 100)
     })
 
-    it('should return 1 point for full node tier requesting full node access', () => {
+    it('should return 10 points for full node tier requesting full node access', () => {
       const jwtInfo = {
         apiLevel: 20,
         resource: 'blockchain',
@@ -153,10 +153,10 @@ describe('#route-ratelimits & jwt-auth', () => {
       }
 
       const result = rateLimits.calcPoints(jwtInfo)
-      assert.equal(result, 1)
+      assert.equal(result, 10)
     })
 
-    it('should return 10 points for full-node tier requesting indexer access', () => {
+    it('should return 100 points for full-node tier requesting indexer access', () => {
       const jwtInfo = {
         apiLevel: 20,
         resource: 'blockbook',
@@ -164,10 +164,10 @@ describe('#route-ratelimits & jwt-auth', () => {
       }
 
       const result = rateLimits.calcPoints(jwtInfo)
-      assert.equal(result, 10)
+      assert.equal(result, 100)
     })
 
-    it('should return 10 points for full node tier requesting SLPDB access', () => {
+    it('should return 100 points for full node tier requesting SLPDB access', () => {
       const jwtInfo = {
         apiLevel: 20,
         resource: 'slp',
@@ -175,10 +175,10 @@ describe('#route-ratelimits & jwt-auth', () => {
       }
 
       const result = rateLimits.calcPoints(jwtInfo)
-      assert.equal(result, 10)
+      assert.equal(result, 100)
     })
 
-    it('should return 1 point for indexer tier requesting full node access', () => {
+    it('should return 10 point for indexer tier requesting full node access', () => {
       const jwtInfo = {
         apiLevel: 30,
         resource: 'blockchain',
@@ -186,32 +186,32 @@ describe('#route-ratelimits & jwt-auth', () => {
       }
 
       const result = rateLimits.calcPoints(jwtInfo)
-      assert.equal(result, 1)
+      assert.equal(result, 10)
     })
 
-    it('should return 1 points for indexer tier requesting indexer access', () => {
+    it('should return 10 points for indexer tier requesting indexer access', () => {
       const jwtInfo = {
         apiLevel: 30,
         resource: 'blockbook',
         id: '5e3a0415eb29a962da2708b4'
       }
 
-      const result = rateLimits.calcPoints(jwtInfo)
-      assert.equal(result, 1)
-    })
-
-    it('should return 10 points for indexer tier requesting SLPDB access', () => {
-      const jwtInfo = {
-        apiLevel: 30,
-        resource: 'slp',
-        id: '5e3a0415eb29a962da2708b4'
-      }
-
       const result = rateLimits.calcPoints(jwtInfo)
       assert.equal(result, 10)
     })
 
-    it('should return 1 point for SLP tier requesting full node access', () => {
+    it('should return 100 points for indexer tier requesting SLPDB access', () => {
+      const jwtInfo = {
+        apiLevel: 30,
+        resource: 'slp',
+        id: '5e3a0415eb29a962da2708b4'
+      }
+
+      const result = rateLimits.calcPoints(jwtInfo)
+      assert.equal(result, 100)
+    })
+
+    it('should return 10 point for SLP tier requesting full node access', () => {
       const jwtInfo = {
         apiLevel: 40,
         resource: 'blockchain',
@@ -219,10 +219,10 @@ describe('#route-ratelimits & jwt-auth', () => {
       }
 
       const result = rateLimits.calcPoints(jwtInfo)
-      assert.equal(result, 1)
+      assert.equal(result, 10)
     })
 
-    it('should return 1 points for SLP tier requesting indexer access', () => {
+    it('should return 10 points for SLP tier requesting indexer access', () => {
       const jwtInfo = {
         apiLevel: 40,
         resource: 'blockbook',
@@ -230,10 +230,10 @@ describe('#route-ratelimits & jwt-auth', () => {
       }
 
       const result = rateLimits.calcPoints(jwtInfo)
-      assert.equal(result, 1)
+      assert.equal(result, 10)
     })
 
-    it('should return 1 points for SLP tier requesting SLPDB access', () => {
+    it('should return 10 points for SLP tier requesting SLPDB access', () => {
       const jwtInfo = {
         apiLevel: 40,
         resource: 'slp',
@@ -241,7 +241,7 @@ describe('#route-ratelimits & jwt-auth', () => {
       }
 
       const result = rateLimits.calcPoints(jwtInfo)
-      assert.equal(result, 1)
+      assert.equal(result, 10)
     })
   })
 
@@ -442,7 +442,7 @@ describe('#route-ratelimits & jwt-auth', () => {
 
       // Issues with token secret should treat incoming requests as anonymous
       // calls with 30 points or 3 RPM.
-      assert.equal(res.locals.pointsToConsume, 30)
+      assert.equal(res.locals.pointsToConsume, 300)
     })
   })
 })