Merge pull request #131 from Permissionless-Software-Foundation/ct-unstable

fix(winston): Switching max files from 5 days to 250 files

README.md

@@ -82,6 +82,20 @@ installation.
 
 `docker-compose up`
 
+## Rate Limits
+The rate limits for [api.fullstack.cash](https://api.fullstack.cash) are controlled by a JWT token. You can increase your rate limits by [purchasing a JWT token](https://https://fullstack.cash). If you're using bch-js, [check the readme for instructions on increasing rate limits](https://github.com/Permissionless-Software-Foundation/bch-js#api-key). For interacting with bch-api directly, you can then include the JWT token in the HTTP header like this:
+
+- `Authorization: Token <JWT token>`
+
+### cURL example
+Here is an example using curl for using the JWT token when making a call to the REST API:
+
+```
+export BCHJSTOKEN=eyJhbGciOiJ....
+
+curl "https://bchn.fullstack.cash/v4/electrumx/balance/bitcoincash:qr69kyzha07dcecrsvjwsj4s6slnlq4r8c30lxnur3" -X GET -H "Content-Type: application/json" -H "Authorization: Token ${BCHJSTOKEN}"
+```
+
 ## Support
 Have questions? Need help? Join our community support
 [Telegram channel](https://t.me/bch_js_toolkit)

apidoc.json

@@ -0,0 +1,4 @@
+{
+    "sampleUrl": null
+
+  }

config/index.js

@@ -16,6 +16,9 @@ const config = {
   whitelistRateLimit: process.env.WHITELIST_RATE_LIMIT
     ? Number(process.env.WHITELIST_RATE_LIMIT)
     : 10,
+  internalRateLimit: process.env.INTERNAL_RATE_LIMIT
+    ? Number(process.env.INTERNAL_RATE_LIMIT)
+    : 10,
   pointsPerMinute: process.env.POINTS_PER_MINUTE
     ? Number(process.env.POINTS_PER_MINUTE)
     : 10000,

package.json

@@ -16,7 +16,7 @@
     "dev": "nodemon ./dist/app.js",
     "test": "npm run lint && npm run test-v4",
     "lint": "standard --env mocha --fix",
-    "test-v4": "export NETWORK=mainnet && nyc --reporter=text mocha --timeout 60000 test/v4/",
+    "test-v4": "export NETWORK=mainnet && nyc --reporter=text mocha --exit --timeout 60000 test/v4/",
     "test:integration": "mocha test/v4/integration",
     "test:integration:slpdb": "mocha --timeout 25000 -g '#validate2Single' test/v4/integration/slp*.js",
     "coverage": "nyc report --reporter=text-lcov | coveralls",
@@ -29,7 +29,7 @@
     "node": ">=10.15.1"
   },
   "dependencies": {
-    "@psf/bch-js": "^4.17.6",
+    "@psf/bch-js": "^4.18.0",
     "apidoc": "^0.26.0",
     "axios": "^0.21.1",
     "bitcore-lib-cash": "^8.23.1",
@@ -70,7 +70,7 @@
     "nock": "^13.0.5",
     "nyc": "^15.0.0",
     "prettier": "^2.0.0",
-    "semantic-release": "^17.3.9",
+    "semantic-release": "^17.4.2",
     "sinon": "^9.0.0",
     "standard": "^14.3.1"
   },

src/middleware/route-ratelimit.js

@@ -56,7 +56,7 @@ const ANON_LIMITS = config.anonRateLimit
 const WHITELIST_DOMAINS = config.whitelistDomains
 const WHITELIST_POINTS_TO_CONSUME = config.whitelistRateLimit
 const POINTS_PER_MINUTE = config.pointsPerMinute
-const INTERNAL_POINTS_TO_CONSUME = 10
+const INTERNAL_POINTS_TO_CONSUME = config.internalRateLimit
 
 class RateLimits {
   constructor () {

src/routes/v4/blockbook.js

@@ -1,521 +0,0 @@
-/*
-  Blockbook API route
-*/
-
-'use strict'
-
-const express = require('express')
-const axios = require('axios')
-const wlogger = require('../../util/winston-logging')
-
-const RouteUtils = require('../../util/route-utils')
-const routeUtils = new RouteUtils()
-
-// Library for easily switching the API paths to use different instances of
-// Blockbook.
-const BlockbookPath = require('../../util/blockbook-path')
-const BLOCKBOOKPATH = new BlockbookPath()
-// BLOCKBOOKPATH.toOpenBazaar()
-
-const router = express.Router()
-
-// Used for processing error messages before sending them to the user.
-const util = require('util')
-util.inspect.defaultOptions = { depth: 1 }
-
-const BCHJS = require('@psf/bch-js')
-const bchjs = new BCHJS()
-
-let _this
-
-class Blockbook {
-  constructor () {
-    _this = this
-
-    _this.axios = axios
-    _this.routeUtils = routeUtils
-    _this.bchjs = bchjs
-    _this.BLOCKBOOKPATH = BLOCKBOOKPATH
-
-    _this.router = router
-    _this.router.get('/', _this.root)
-    _this.router.get('/balance/:address', _this.balanceSingle)
-    _this.router.post('/balance', _this.balanceBulk)
-    _this.router.get('/utxos/:address', _this.utxosSingle)
-    _this.router.post('/utxos', _this.utxosBulk)
-    _this.router.get('/tx/:txid', _this.txSingle)
-    _this.router.post('/tx', _this.txBulk)
-  }
-
-  // DRY error handler.
-  errorHandler (err, res) {
-    // Attempt to decode the error message.
-    const { msg, status } = _this.routeUtils.decodeError(err)
-    if (msg) {
-      res.status(status)
-      return res.json({ error: msg })
-    }
-
-    res.status(500)
-    return res.json({ error: util.inspect(err) })
-  }
-
-  // Root API endpoint. Simply acknowledges that it exists.
-  root (req, res, next) {
-    return res.json({ status: 'address' })
-  }
-
-  // Query the Blockbook Node API for a balance on a single BCH address.
-  // Returns a Promise.
-  async balanceFromBlockbook (thisAddress) {
-    try {
-      // console.log(`BLOCKBOOK_URL: ${BLOCKBOOK_URL}`)
-
-      // Convert the address to a cashaddr without a prefix.
-      const addr = _this.bchjs.Address.toCashAddress(thisAddress)
-
-      const path = `${_this.BLOCKBOOKPATH.addrPath}${addr}`
-      // console.log(`path: ${path}`)
-
-      // Query the Blockbook Node API.
-      const options = {
-        method: 'get',
-        baseURL: path
-      }
-
-      const axiosResponse = await _this.axios.request(options)
-      const retData = axiosResponse.data
-      // console.log(`retData: ${util.inspect(retData)}`)
-
-      return retData
-    } catch (err) {
-      // Dev Note: Do not log error messages here. Throw them instead and let the
-      // parent function handle it.
-      wlogger.debug('Error in blockbook.js/balanceFromBlockbook()')
-      throw err
-    }
-  }
-
-  async balanceSingle (req, res, next) {
-    try {
-      const address = req.params.address
-
-      if (!address || address === '') {
-        res.status(400)
-        return res.json({ error: 'address can not be empty' })
-      }
-
-      // Reject if address is an array.
-      if (Array.isArray(address)) {
-        res.status(400)
-        return res.json({
-          error: 'address can not be an array. Use POST for bulk upload.'
-        })
-      }
-
-      wlogger.debug(
-        'Executing blockbook/balanceSingle with this address: ',
-        address
-      )
-
-      // Ensure the input is a valid BCH address.
-      try {
-        // const legacyAddr = bchjs.Address.toLegacyAddress(address)
-        _this.bchjs.Address.toLegacyAddress(address)
-      } catch (err) {
-        res.status(400)
-        return res.json({
-          error: `Invalid BCH address. Double check your address is valid: ${address}`
-        })
-      }
-
-      // Prevent a common user error. Ensure they are using the correct network address.
-      const networkIsValid = _this.routeUtils.validateNetwork(address)
-      if (!networkIsValid) {
-        res.status(400)
-        return res.json({
-          error:
-            'Invalid network. Trying to use a testnet address on mainnet, or vice versa.'
-        })
-      }
-
-      // Query the Blockbook Node API.
-      const retData = await _this.balanceFromBlockbook(address)
-
-      // Return the retrieved address information.
-      res.status(200)
-      return res.json(retData)
-    } catch (err) {
-      // Write out error to error log.
-      wlogger.error('Error in blockbook.js/balanceSingle().', err)
-
-      return _this.errorHandler(err, res)
-    }
-  }
-
-  // POST handler for bulk queries on address details
-  async balanceBulk (req, res, next) {
-    try {
-      let addresses = req.body.addresses
-      // const currentPage = req.body.page ? parseInt(req.body.page, 10) : 0
-
-      // Reject if addresses is not an array.
-      if (!Array.isArray(addresses)) {
-        res.status(400)
-        return res.json({
-          error: 'addresses needs to be an array. Use GET for single address.'
-        })
-      }
-
-      // Enforce array size rate limits
-      if (!_this.routeUtils.validateArraySize(req, addresses)) {
-        res.status(400) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
-        return res.json({
-          error: 'Array too large.'
-        })
-      }
-
-      wlogger.debug(
-        'Executing blockbook.js/balanceBulk with these addresses: ',
-        addresses
-      )
-
-      // Validate each element in the address array.
-      for (let i = 0; i < addresses.length; i++) {
-        const thisAddress = addresses[i]
-
-        // Ensure the input is a valid BCH address.
-        try {
-          _this.bchjs.Address.toLegacyAddress(thisAddress)
-        } catch (err) {
-          res.status(400)
-          return res.json({
-            error: `Invalid BCH address. Double check your address is valid: ${thisAddress}`
-          })
-        }
-
-        // Prevent a common user error. Ensure they are using the correct network address.
-        const networkIsValid = _this.routeUtils.validateNetwork(thisAddress)
-        if (!networkIsValid) {
-          res.status(400)
-          return res.json({
-            error: `Invalid network for address ${thisAddress}. Trying to use a testnet address on mainnet, or vice versa.`
-          })
-        }
-      }
-
-      // Loops through each address and creates an array of Promises, querying
-      // Insight API in parallel.
-      addresses = addresses.map(async (address, index) =>
-        // console.log(`address: ${address}`)
-        _this.balanceFromBlockbook(address)
-      )
-
-      // Wait for all parallel Insight requests to return.
-      const result = await _this.axios.all(addresses)
-
-      // Return the array of retrieved address information.
-      res.status(200)
-      return res.json(result)
-    } catch (err) {
-      wlogger.error('Error in blockbook.js/balanceBulk().', err)
-
-      return _this.errorHandler(err, res)
-    }
-  }
-
-  // Query the Blockbook API for utxos associated with a BCH address.
-  // Returns a Promise.
-  async utxosFromBlockbook (thisAddress) {
-    try {
-      // console.log(`BLOCKBOOK_URL: ${BLOCKBOOK_URL}`)
-
-      // Convert the address to a cashaddr without a prefix.
-      const addr = _this.bchjs.Address.toCashAddress(thisAddress)
-
-      const path = `${_this.BLOCKBOOKPATH.utxoPath}${addr}`
-      // console.log(`path: ${path}`)
-
-      // Query the Blockbook API.
-      // Query the Blockbook Node API.
-      const options = {
-        method: 'get',
-        baseURL: path
-      }
-      const axiosResponse = await _this.axios.request(options)
-      const retData = axiosResponse.data
-      // console.log(`retData: ${util.inspect(retData)}`)
-
-      // Add the satoshis property to each UTXO.
-      for (let i = 0; i < retData.length; i++) {
-        retData[i].satoshis = Number(retData[i].value)
-      }
-
-      return retData
-    } catch (err) {
-      // Dev Note: Do not log error messages here. Throw them instead and let the
-      // parent function handle it.
-      wlogger.debug('Error in blockbook.js/utxosFromBlockbook()')
-      throw err
-    }
-  }
-
-  // GET handler for single balance
-  async utxosSingle (req, res, next) {
-    try {
-      const address = req.params.address
-
-      if (!address || address === '') {
-        res.status(400)
-        return res.json({ error: 'address can not be empty' })
-      }
-
-      // Reject if address is an array.
-      if (Array.isArray(address)) {
-        res.status(400)
-        return res.json({
-          error: 'address can not be an array. Use POST for bulk upload.'
-        })
-      }
-
-      wlogger.debug(
-        'Executing blockbook/utxosSingle with this address: ',
-        address
-      )
-
-      // Ensure the input is a valid BCH address.
-      try {
-        // const legacyAddr = bchjs.Address.toLegacyAddress(address)
-        _this.bchjs.Address.toLegacyAddress(address)
-      } catch (err) {
-        res.status(400)
-        return res.json({
-          error: `Invalid BCH address. Double check your address is valid: ${address}`
-        })
-      }
-
-      // Prevent a common user error. Ensure they are using the correct network address.
-      const networkIsValid = _this.routeUtils.validateNetwork(address)
-      if (!networkIsValid) {
-        res.status(400)
-        return res.json({
-          error:
-            'Invalid network. Trying to use a testnet address on mainnet, or vice versa.'
-        })
-      }
-
-      // Query the Blockbook API.
-      const retData = await _this.utxosFromBlockbook(address)
-
-      // Return the retrieved address information.
-      res.status(200)
-      return res.json(retData)
-    } catch (err) {
-      // Write out error to error log.
-      wlogger.error('Error in blockbook.js/utxosSingle().', err)
-      return _this.errorHandler(err, res)
-    }
-  }
-
-  // POST handler for bulk queries on address utxos
-  async utxosBulk (req, res, next) {
-    try {
-      let addresses = req.body.addresses
-      // const currentPage = req.body.page ? parseInt(req.body.page, 10) : 0
-
-      // Reject if addresses is not an array.
-      if (!Array.isArray(addresses)) {
-        res.status(400)
-        return res.json({
-          error: 'addresses needs to be an array. Use GET for single address.'
-        })
-      }
-
-      // Enforce array size rate limits
-      if (!_this.routeUtils.validateArraySize(req, addresses)) {
-        res.status(400) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
-        return res.json({
-          error: 'Array too large.'
-        })
-      }
-
-      wlogger.debug(
-        'Executing blockbook.js/utxosBulk with these addresses: ',
-        addresses
-      )
-
-      // Validate each element in the address array.
-      for (let i = 0; i < addresses.length; i++) {
-        const thisAddress = addresses[i]
-
-        // Ensure the input is a valid BCH address.
-        try {
-          _this.bchjs.Address.toLegacyAddress(thisAddress)
-        } catch (err) {
-          res.status(400)
-          return res.json({
-            error: `Invalid BCH address. Double check your address is valid: ${thisAddress}`
-          })
-        }
-
-        // Prevent a common user error. Ensure they are using the correct network address.
-        const networkIsValid = _this.routeUtils.validateNetwork(thisAddress)
-        if (!networkIsValid) {
-          res.status(400)
-          return res.json({
-            error: `Invalid network for address ${thisAddress}. Trying to use a testnet address on mainnet, or vice versa.`
-          })
-        }
-      }
-
-      // Loops through each address and creates an array of Promises, querying
-      // Insight API in parallel.
-      addresses = addresses.map(async (address, index) =>
-        // console.log(`address: ${address}`)
-        _this.utxosFromBlockbook(address)
-      )
-
-      // Wait for all parallel Insight requests to return.
-      const result = await _this.axios.all(addresses)
-
-      // Return the array of retrieved address information.
-      res.status(200)
-      return res.json(result)
-    } catch (err) {
-      wlogger.error('Error in blockbook.js/utxosBulk().', err)
-      return _this.errorHandler(err, res)
-    }
-  }
-
-  // Query the Blockbook Node API for transactions on a single TXID.
-  // Returns a Promise.
-  async transactionsFromBlockbook (txid) {
-    try {
-      // console.log(`BLOCKBOOK_URL: ${BLOCKBOOK_URL}`)
-
-      const path = `${_this.BLOCKBOOKPATH.txPath}${txid}`
-      // console.log(`path: ${path}`)
-
-      // Query the Blockbook Node API.
-      const options = {
-        method: 'get',
-        baseURL: path
-      }
-      const axiosResponse = await _this.axios.request(options)
-      const retPromise = axiosResponse.data
-      // console.log(`retData: ${util.inspect(retData)}`)
-
-      return retPromise
-    } catch (err) {
-      // Dev Note: Do not log error messages here. Throw them instead and let the
-      // parent function handle it.
-      wlogger.debug('Error in blockbook.js/transactionsFromBlockbook()')
-      throw err
-    }
-  }
-
-  // GET handler for single transaction details.
-  async txSingle (req, res, next) {
-    try {
-      const txid = req.params.txid
-
-      if (!txid || txid === '') {
-        res.status(400)
-        return res.json({ error: 'txid can not be empty' })
-      }
-
-      // Reject if address is an array.
-      if (Array.isArray(txid)) {
-        res.status(400)
-        return res.json({
-          error: 'txid can not be an array. Use POST for bulk upload.'
-        })
-      }
-
-      // TODO: Add regex comparison of txid to ensure it's valid.
-      if (txid.length !== 64) {
-        res.status(400)
-        return res.json({
-          error: `txid must be of length 64 (not ${txid.length})`
-        })
-      }
-
-      wlogger.debug('Executing blockbook/txSingle with this txid: ', txid)
-
-      // Query the Blockbook Node API.
-      const retData = await _this.transactionsFromBlockbook(txid)
-
-      // Return the retrieved address information.
-      res.status(200)
-      return res.json(retData)
-    } catch (err) {
-      // Write out error to error log.
-      wlogger.error('Error in blockbook.js/txSingle().', err)
-
-      return _this.errorHandler(err, res)
-    }
-  }
-
-  // POST handler for bulk queries on tx details
-  async txBulk (req, res, next) {
-    try {
-      let txids = req.body.txids
-      // const currentPage = req.body.page ? parseInt(req.body.page, 10) : 0
-
-      // Reject if txids is not an array.
-      if (!Array.isArray(txids)) {
-        res.status(400)
-        return res.json({
-          error: 'txids need to be an array. Use GET for single address.'
-        })
-      }
-
-      // Enforce array size rate limits
-      if (!_this.routeUtils.validateArraySize(req, txids)) {
-        res.status(400) // https://github.com/Bitcoin-com/rest.bitcoin.com/issues/330
-        return res.json({
-          error: 'Array too large.'
-        })
-      }
-
-      wlogger.debug('Executing blockbook.js/txBulk with these txids: ', txids)
-
-      // Validate each element in the txids array.
-      for (let i = 0; i < txids.length; i++) {
-        const thisTxid = txids[i]
-
-        if (!thisTxid || thisTxid === '') {
-          res.status(400)
-          return res.json({ error: 'txid can not be empty' })
-        }
-
-        // TODO: Add regex comparison of txid to ensure it's valid.
-        if (thisTxid.length !== 64) {
-          res.status(400)
-          return res.json({
-            error: `txid must be of length 64 (not ${thisTxid.length})`
-          })
-        }
-      }
-
-      // Loops through each address and creates an array of Promises, querying
-      // Insight API in parallel.
-      txids = txids.map(async (txid, index) =>
-        // console.log(`address: ${address}`)
-        _this.transactionsFromBlockbook(txid)
-      )
-
-      // Wait for all parallel Insight requests to return.
-      const result = await _this.axios.all(txids)
-
-      // Return the array of retrieved address information.
-      res.status(200)
-      return res.json(result)
-    } catch (err) {
-      wlogger.error('Error in blockbook.js/txBulk().', err)
-      return _this.errorHandler(err, res)
-    }
-  }
-}
-
-module.exports = Blockbook

src/routes/v4/slp.js

@@ -1780,8 +1780,6 @@ class Slp {
 
   // Format the response from SLPDB into an object.
   async formatToRestObject (slpDBFormat) {
-    _this.BigNumber.set({ DECIMAL_PLACES: 8 })
-
     // console.log(`slpDBFormat.data: ${JSON.stringify(slpDBFormat.data, null, 2)}`)
 
     const transaction = slpDBFormat.data.u.length

src/util/route-utils.js

@@ -154,16 +154,23 @@ class RouteUtils {
         }
       }
 
-      // Handle 429 errors thrown by nginx
+      // Handle 429 errors
       if (err.error) {
-        // console.log('decodeError: err: ', err)
+        console.log('decodeError: err: ', err)
 
+        // Error thrown by nginx (usually the SLPDB load balancer.)
         if (err.error.includes('429 Too Many Requests')) {
           const internalMsg =
             '429 error thrown by nginx caught by route-utils.js/decodeError()'
           console.error(internalMsg)
           wlogger.error(internalMsg)
 
+          return {
+            msg: '429 Too Many Requests',
+            status: 429
+          }
+        } else if (err.error.includes('Too many requests')) {
+          // Error is being thrown by bch-api rate limit middleware.
           return {
             msg: '429 Too Many Requests',
             status: 429

src/util/winston-logging.js

@@ -15,7 +15,11 @@ var NETWORK = process.env.NETWORK
 var LOG_MAX_SIZE = process.env.LOG_MAX_SIZE ? process.env.LOG_MAX_SIZE : '1m'
 
 // Default 5 days.
-var LOG_MAX_FILES = process.env.LOG_MAX_FILES ? process.env.LOG_MAX_FILES : '5d'
+// This was causing a problem with popularity. Creating over a gigabyte of files.
+// var LOG_MAX_FILES = process.env.LOG_MAX_FILES ? process.env.LOG_MAX_FILES : '5d'
+
+// 250 files @ 1Meg each = 250 megs
+var LOG_MAX_FILES = process.env.LOG_MAX_FILES ? process.env.LOG_MAX_FILES : '250'
 
 // Configure daily-rotation transport.
 var transport = new winston.transports.DailyRotateFile({

test/v4/route-utils.js

@@ -0,0 +1,34 @@
+/*
+  Unit tests for the route-utils.js library.
+*/
+
+const assert = require('chai').assert
+
+const RouteUtils = require('../../src/util/route-utils.js')
+
+describe('#route-utils', () => {
+  let uut
+
+  beforeEach(() => {
+    uut = new RouteUtils()
+  })
+
+  describe('#decodeError', () => {
+    it('should decode a 429 error from nginx', () => {
+      const err = {
+        error: '<html>\r\n<head><title>429 Too Many Requests</title></head>\r\n<body>\r\n<center><h1>429 Too Many Requests</h1></center>\r\n<hr><center>nginx/1.18.0 (Ubuntu)</center>\r\n</body>\r\n</html>\r\n',
+        level: 'error',
+        message: 'Error in slp.js/hydrateUtxos().',
+        timestamp: '2021-03-31T04:13:36.662Z'
+      }
+
+      const result = uut.decodeError(err)
+      // console.log('result: ', result)
+
+      assert.property(result, 'msg')
+      assert.equal(result.msg, '429 Too Many Requests')
+      assert.property(result, 'status')
+      assert.equal(result.status, 429)
+    })
+  })
+})