Merge pull request #133 from Permissionless-Software-Foundation/ct-unstable

Updating example start shell script

.gitignore

@@ -30,6 +30,7 @@ start-fullstack-abc.sh
 start-fullstack-bchn.sh
 start-ss-main.sh
 start-decatur-bchn.sh
+shell-scripts/
 
 coverage
 start-my-infra

README.md

@@ -82,6 +82,20 @@ installation.
 
 `docker-compose up`
 
+## Rate Limits
+The rate limits for [api.fullstack.cash](https://api.fullstack.cash) are controlled by a JWT token. You can increase your rate limits by [purchasing a JWT token](https://https://fullstack.cash). If you're using bch-js, [check the readme for instructions on increasing rate limits](https://github.com/Permissionless-Software-Foundation/bch-js#api-key). For interacting with bch-api directly, you can then include the JWT token in the HTTP header like this:
+
+- `Authorization: Token <JWT token>`
+
+### cURL example
+Here is an example using curl for using the JWT token when making a call to the REST API:
+
+```
+export BCHJSTOKEN=eyJhbGciOiJ....
+
+curl "https://bchn.fullstack.cash/v4/electrumx/balance/bitcoincash:qr69kyzha07dcecrsvjwsj4s6slnlq4r8c30lxnur3" -X GET -H "Content-Type: application/json" -H "Authorization: Token ${BCHJSTOKEN}"
+```
+
 ## Support
 Have questions? Need help? Join our community support
 [Telegram channel](https://t.me/bch_js_toolkit)

config/index.js

@@ -16,6 +16,9 @@ const config = {
   whitelistRateLimit: process.env.WHITELIST_RATE_LIMIT
     ? Number(process.env.WHITELIST_RATE_LIMIT)
     : 10,
+  internalRateLimit: process.env.INTERNAL_RATE_LIMIT
+    ? Number(process.env.INTERNAL_RATE_LIMIT)
+    : 10,
   pointsPerMinute: process.env.POINTS_PER_MINUTE
     ? Number(process.env.POINTS_PER_MINUTE)
     : 10000,

package.json

@@ -16,20 +16,20 @@
     "dev": "nodemon ./dist/app.js",
     "test": "npm run lint && npm run test-v4",
     "lint": "standard --env mocha --fix",
-    "test-v4": "export NETWORK=mainnet && nyc --reporter=text mocha --timeout 60000 test/v4/",
+    "test-v4": "export NETWORK=mainnet && nyc --reporter=text mocha --exit --timeout 60000 test/v4/",
     "test:integration": "mocha test/v4/integration",
     "test:integration:slpdb": "mocha --timeout 25000 -g '#validate2Single' test/v4/integration/slp*.js",
     "coverage": "nyc report --reporter=text-lcov | coveralls",
     "coverage:report": "export NETWORK=mainnet && nyc --reporter=html mocha --timeout 25000 test/v4/",
     "docs": "./node_modules/.bin/apidoc -i src/routes/v4 -o docs",
     "test:temp1": "export NETWORK=mainnet && export TEST=integration && mocha --exit --timeout 25000 -g '#hydrateUtxos-' test/v4/integration/",
-    "test:temp2": "mocha test/v4/rate-limit2-unit.js"
+    "test:temp2": "mocha -g '#rate-routelimit' --timeout 30000 test/v4/"
   },
   "engines": {
     "node": ">=10.15.1"
   },
   "dependencies": {
-    "@psf/bch-js": "^4.17.6",
+    "@psf/bch-js": "^4.18.0",
     "apidoc": "^0.26.0",
     "axios": "^0.21.1",
     "bitcore-lib-cash": "^8.23.1",
@@ -70,7 +70,7 @@
     "nock": "^13.0.5",
     "nyc": "^15.0.0",
     "prettier": "^2.0.0",
-    "semantic-release": "^17.3.9",
+    "semantic-release": "^17.4.2",
     "sinon": "^9.0.0",
     "standard": "^14.3.1"
   },

src/middleware/route-ratelimit.js

@@ -56,7 +56,7 @@ const ANON_LIMITS = config.anonRateLimit
 const WHITELIST_DOMAINS = config.whitelistDomains
 const WHITELIST_POINTS_TO_CONSUME = config.whitelistRateLimit
 const POINTS_PER_MINUTE = config.pointsPerMinute
-const INTERNAL_POINTS_TO_CONSUME = 10
+const INTERNAL_POINTS_TO_CONSUME = config.internalRateLimit
 
 class RateLimits {
   constructor () {
@@ -154,6 +154,7 @@ class RateLimits {
             res,
             defaultJwt
           )
+          // console.log(`hasExceededRateLimit: `, hasExceededRateLimit)
 
           if (!hasExceededRateLimit) {
             // Rate limits have not been exceeded. Processing can continue.
@@ -195,6 +196,7 @@ class RateLimits {
           res,
           req.locals.jwtToken
         )
+        // console.log('hasExceededRateLimit: ', hasExceededRateLimit)
 
         if (!hasExceededRateLimit) {
           // Rate limits have not been exceeded. Processing can continue.
@@ -220,8 +222,12 @@ class RateLimits {
   async trackRateLimits (req, res, jwtToken) {
     // Anonymous rate limits are used by default.
     let pointsToConsume = ANON_LIMITS
+    // console.log('pointsToConsume: ', pointsToConsume)
+
     let key = req.ip // Use the IP address as the key, by default.
 
+    // console.log('jwtToken: ', jwtToken)
+
     try {
       // Decode the JWT token if it exists
       if (jwtToken) {
@@ -239,6 +245,10 @@ class RateLimits {
       // The 429 error response is handled by the catch().
       await _this.rateLimiter.consume(key, pointsToConsume)
 
+      // Debugging
+      // const rateLimitData = await _this.rateLimiter.consume(key, pointsToConsume)
+      // console.log(`rateLimitData: `, rateLimitData)
+
       res.locals.pointsToConsume = pointsToConsume // Feedback for tests.
 
       // Signal that the user has not exceeded their rate limits.

src/routes/v4/slp.js

@@ -1780,8 +1780,6 @@ class Slp {
 
   // Format the response from SLPDB into an object.
   async formatToRestObject (slpDBFormat) {
-    _this.BigNumber.set({ DECIMAL_PLACES: 8 })
-
     // console.log(`slpDBFormat.data: ${JSON.stringify(slpDBFormat.data, null, 2)}`)
 
     const transaction = slpDBFormat.data.u.length

src/util/route-utils.js

@@ -154,16 +154,23 @@ class RouteUtils {
         }
       }
 
-      // Handle 429 errors thrown by nginx
+      // Handle 429 errors
       if (err.error) {
-        // console.log('decodeError: err: ', err)
+        console.log('decodeError: err: ', err)
 
+        // Error thrown by nginx (usually the SLPDB load balancer.)
         if (err.error.includes('429 Too Many Requests')) {
           const internalMsg =
             '429 error thrown by nginx caught by route-utils.js/decodeError()'
           console.error(internalMsg)
           wlogger.error(internalMsg)
 
+          return {
+            msg: '429 Too Many Requests',
+            status: 429
+          }
+        } else if (err.error.includes('Too many requests')) {
+          // Error is being thrown by bch-api rate limit middleware.
           return {
             msg: '429 Too Many Requests',
             status: 429

src/util/winston-logging.js

@@ -15,7 +15,11 @@ var NETWORK = process.env.NETWORK
 var LOG_MAX_SIZE = process.env.LOG_MAX_SIZE ? process.env.LOG_MAX_SIZE : '1m'
 
 // Default 5 days.
-var LOG_MAX_FILES = process.env.LOG_MAX_FILES ? process.env.LOG_MAX_FILES : '5d'
+// This was causing a problem with popularity. Creating over a gigabyte of files.
+// var LOG_MAX_FILES = process.env.LOG_MAX_FILES ? process.env.LOG_MAX_FILES : '5d'
+
+// 250 files @ 1Meg each = 250 megs
+var LOG_MAX_FILES = process.env.LOG_MAX_FILES ? process.env.LOG_MAX_FILES : '250'
 
 // Configure daily-rotation transport.
 var transport = new winston.transports.DailyRotateFile({

start-dev-example.sh

@@ -48,12 +48,12 @@ export PRO_PASS=somerandomepassword:someotherrandompassword:aThirdPassword
 # that originate froma domain on the whitelist.
 export WHITELIST_DOMAINS=fullstack.cash,psfoundation.cash,torlist.cash
 
-# Rate Limits. Numbers are divided into 1000. e.g. 1000 / 50 = 20 RPM for ANON.
+# Rate Limits. Numbers are divided into 1000. e.g. 10000 / 500 = 20 RPM for ANON.
 # Requests use the ANON rate limit if they fail to pass in a JWT token.
 # ANON = 20 requests per minute (RPM)
-export ANON_RATE_LIMIT=50
-# 10 = 100 RPM
-export WHITELIST_RATE_LIMIT=10
+export ANON_RATE_LIMIT=500
+# 10 = 1000 RPM
+export WHITELIST_RATE_LIMIT=100
 
 # Set logging parameters
 #1m means no more than 1 megabyte

test/v4/raw-transactions.js

@@ -250,6 +250,7 @@ describe('#Raw-Transactions', () => {
         'Proper error message'
       )
     })
+
     it('returns proper error when downstream service stalls', async () => {
       req.body.hexes = [
         '0200000001b9b598d7d6d72fc486b2b3a3c03c79b5bade6ec9a77ced850515ab5e64edcc21010000006b483045022100a7b1b08956abb8d6f322aa709d8583c8ea492ba0585f1a6f4f9983520af74a5a0220411aee4a9a54effab617b0508c504c31681b15f9b187179b4874257badd4139041210360cfc66fdacb650bc4c83b4e351805181ee696b7d5ab4667c57b2786f51c413dffffffff0210270000000000001976a914eb4b180def88e3f5625b2d8ae2c098ff7d85f66488ac786e9800000000001976a914eb4b180def88e3f5625b2d8ae2c098ff7d85f66488ac00000000'

test/v4/route-utils.js

@@ -0,0 +1,34 @@
+/*
+  Unit tests for the route-utils.js library.
+*/
+
+const assert = require('chai').assert
+
+const RouteUtils = require('../../src/util/route-utils.js')
+
+describe('#route-utils', () => {
+  let uut
+
+  beforeEach(() => {
+    uut = new RouteUtils()
+  })
+
+  describe('#decodeError', () => {
+    it('should decode a 429 error from nginx', () => {
+      const err = {
+        error: '<html>\r\n<head><title>429 Too Many Requests</title></head>\r\n<body>\r\n<center><h1>429 Too Many Requests</h1></center>\r\n<hr><center>nginx/1.18.0 (Ubuntu)</center>\r\n</body>\r\n</html>\r\n',
+        level: 'error',
+        message: 'Error in slp.js/hydrateUtxos().',
+        timestamp: '2021-03-31T04:13:36.662Z'
+      }
+
+      const result = uut.decodeError(err)
+      // console.log('result: ', result)
+
+      assert.property(result, 'msg')
+      assert.equal(result.msg, '429 Too Many Requests')
+      assert.property(result, 'status')
+      assert.equal(result.status, 429)
+    })
+  })
+})