Merge pull request #54 from Permissionless-Software-Foundation/ct-unstable

Added timer controller to remove duplicate offers

package.json

@@ -28,6 +28,8 @@
   },
   "repository": "Permissionless-Software-Foundation/bch-dex",
   "dependencies": {
+    "@chris.troutner/retry-queue": "1.0.5",
+    "@psf/bch-js": "^6.4.5",
     "axios": "0.27.2",
     "bch-message-lib": "2.2.1",
     "bcryptjs": "2.4.3",
@@ -51,7 +53,7 @@
     "koa2-ratelimit": "0.9.1",
     "libp2p": "^0.36.2",
     "line-reader": "0.4.0",
-    "minimal-slp-wallet": "5.1.0",
+    "minimal-slp-wallet": "5.2.1",
     "mongoose": "5.13.14",
     "node-fetch": "npm:@achingbrain/node-fetch@2.6.7",
     "nodemailer": "6.7.5",

production/rpi-docker/docker-compose.yml

@@ -43,7 +43,7 @@ services:
     #build:
     #  context: ./p2wdb/
     #  dockerfile: Dockerfile
-    image: christroutner/p2wdb-rpi:v3.1.8
+    image: christroutner/p2wdb-rpi:v3.1.9
     container_name: p2wdb
     environment:
       CONSUMER_URL: 'https://wa-usa-bch-consumer.fullstackcash.nl'

src/adapters/p2wdb-adapter.js

@@ -27,6 +27,10 @@ class P2wdbAdapter {
 
     // Allow the localConfig to overwrite the config file values.
     this.p2wdbURL = localConfig.p2wdbURL || config.p2wdbUrl
+
+    // Bind the 'this' object to all functions
+    this.write = this.write.bind(this)
+    this.checkForSufficientFunds = this.checkForSufficientFunds.bind(this)
   }
 
   // Write some data to the P2WDB

src/adapters/wallet.js

@@ -31,6 +31,9 @@ class WalletAdapter {
     this.bitcoinJs = bitcoinJs
     this.BchWallet = BchWallet
     this.bchWallet = {} // Will be replaced when initialized.
+
+    // Bind the 'this' object
+    this.moveTokens = this.moveTokens.bind(this)
   }
 
   // Open the wallet file, or create one if the file doesn't exist.

src/controllers/timer-controllers.js

@@ -33,6 +33,7 @@ class TimerControllers {
   startTimers () {
     setInterval(this.gcOrders, 60000 * 5)
     setInterval(this.gcOffers, 60000 * 5)
+    setInterval(this.checkDupOffers, 60000 * 4.5)
   }
 
   // Garbage Collect the Orders.
@@ -54,6 +55,16 @@ class TimerControllers {
       console.log('Error in timer-controllers.js/gcOffers(): ', err)
     }
   }
+
+  // Remove duplicate Offers
+  checkDupOffers () {
+    try {
+      _this.useCases.offer.removeDuplicateOffers()
+    } catch (err) {
+      // Do not throw an error. This is a top-level function.
+      console.log('Error in timer-controllers.js/checkDupOffers(): ', err)
+    }
+  }
 }
 
 export default TimerControllers

src/entities/offer.js

@@ -29,7 +29,7 @@ class OfferEntity {
       utxoVout,
       offerStatus,
       makerAddr,
-      ticker,
+      // ticker,
       tokenType
     } = offerData.data
 
@@ -67,9 +67,9 @@ class OfferEntity {
     if (!makerAddr || typeof makerAddr !== 'string') {
       throw new Error("Property 'makerAddr' must be a string.")
     }
-    if (!ticker || typeof ticker !== 'string') {
-      throw new Error("Property 'ticker' must be a string.")
-    }
+    // if (!ticker || typeof ticker !== 'string') {
+    //   throw new Error("Property 'ticker' must be a string.")
+    // }
     if (!tokenType || typeof tokenType !== 'number') {
       throw new Error("Property 'tokenType' must be a number.")
     }
@@ -95,7 +95,7 @@ class OfferEntity {
       p2wdbHash: offerData.hash,
       offerStatus: offerStatus || this.offerStatus[0],
       makerAddr,
-      ticker,
+      // ticker,
       tokenType
     }
 

src/entities/order.js

@@ -7,7 +7,10 @@
   The Order tracks the hdIndex address used to hold tokens or BCH for sale.
 */
 class Order {
-  validate (data) {
+  // TODO: Create a fullValidate() function that validates a fully-hydrated
+  // order model.
+
+  inputValidate (data) {
     const {
       messageType,
       messageClass,
@@ -16,8 +19,7 @@ class Order {
       rateInBaseUnit,
       minUnitsToExchange,
       numTokens,
-      makerAddr,
-      ticker
+      makerAddr
     } = data
 
     // Input Validation
@@ -45,9 +47,9 @@ class Order {
     if (!makerAddr || typeof makerAddr !== 'string') {
       throw new Error("Property 'makerAddr' must be a string.")
     }
-    if (!ticker || typeof ticker !== 'string') {
-      throw new Error("Property 'ticker' must be a string.")
-    }
+    // if (!ticker || typeof ticker !== 'string') {
+    //   throw new Error("Property 'ticker' must be a string.")
+    // }
 
     const offerData = {
       messageType,
@@ -57,8 +59,8 @@ class Order {
       rateInBaseUnit,
       minUnitsToExchange,
       numTokens,
-      makerAddr,
-      ticker
+      makerAddr
+      // ticker
     }
 
     return offerData

src/use-cases/offer/index.js

@@ -15,6 +15,7 @@
 
 // Global npm libraries
 import axios from 'axios'
+import RetryQueue from '@chris.troutner/retry-queue'
 
 // Local libraries
 import OfferEntity from '../../entities/offer.js'
@@ -43,9 +44,12 @@ class OfferUseCases {
     // Encapsulate dependencies
     this.config = config
     this.axios = axios
-
     this.offerEntity = new OfferEntity()
     this.OfferModel = this.adapters.localdb.Offer
+    this.retryQueue = new RetryQueue({ retryPeriod: 1000, attempts: 3 })
+
+    // Bind 'this' object to functions
+    this.detectNsfw = this.detectNsfw.bind(this)
   }
 
   // This method is called by the POST /offer REST API controller, which is
@@ -73,14 +77,23 @@ class OfferUseCases {
 
       // Verify that UTXO in offer is unspent. If it is spent, then ignore the
       // offer.
-      const txid = offerObj.data.utxoTxid
-      const vout = offerObj.data.utxoVout
-      const utxoStatus = await this.adapters.bchjs.Blockchain.getTxOut(
-        txid,
-        vout
-      )
+      // const txid = offerObj.data.utxoTxid
+      // const vout = offerObj.data.utxoVout
+      // const utxoStatus = await this.adapters.bchjs.Blockchain.getTxOut(
+      //   txid,
+      //   vout
+      // )
+      //
+      const utxo = {
+        tx_hash: offerObj.data.utxoTxid,
+        tx_pos: offerObj.data.utxoVout
+      }
+      // const utxoStatus = await this.adapters.wallet.bchWallet.utxoIsValid(utxo)
+      const utxoStatus = await this.retryQueue.addToQueue(this.adapters.wallet.bchWallet.utxoIsValid, utxo)
+
       console.log('utxoStatus: ', utxoStatus)
-      if (utxoStatus === null) return false
+      // if (utxoStatus === null) return false
+      if (!utxoStatus) return false
 
       // A new offer gets a status of 'posted'
       offerObj.data.offerStatus = 'posted'
@@ -90,18 +103,20 @@ class OfferUseCases {
 
       // Get data about the token.
       const tokenId = offerEntity.tokenId
-      const tokenData = await this.adapters.wallet.bchWallet.getTokenData(tokenId)
+      // const tokenData = await this.adapters.wallet.bchWallet.getTokenData(tokenId)
+      const tokenData = await this.retryQueue.addToQueue(this.adapters.wallet.bchWallet.getTokenData, tokenId)
       console.log(`tokenData: ${JSON.stringify(tokenData, null, 2)}`)
 
       // Generate a 'display category' for the token. This will allow the
       // front end UI to figure out how to display the token.
-      const displayCategory = await this.categorizeToken(offerEntity, tokenData)
+      const displayCategory = this.categorizeToken(offerEntity, tokenData)
       console.log('displayCategory: ', displayCategory)
       offerEntity.displayCategory = displayCategory
 
       // Detect if user set the NSFW flag.
       let nsfw = false
-      nsfw = await this.detectNsfw(tokenData)
+      // nsfw = await this.detectNsfw(tokenData)
+      nsfw = await this.retryQueue.addToQueue(this.detectNsfw, tokenData)
       offerEntity.nsfw = nsfw
 
       // Add offer to the local database.
@@ -165,7 +180,7 @@ class OfferUseCases {
   // The first three are easy to categorize. The simple-nft is a fungible token
   // with a quantity of 1, decimals of 0, and no minting baton. Categorizing this
   // type of token is the main reason why this function exists.
-  async categorizeToken (offerData, tokenData) {
+  categorizeToken (offerData, tokenData) {
     try {
       // console.log(`categorizeToken(): ${JSON.stringify(offerData, null, 2)}`)
 
@@ -497,6 +512,45 @@ class OfferUseCases {
     }
   }
 
+  // TODO: Write unit tests for this function, then add it to the Timer Controllers
+  // Looks for duplicate offers and removes the duplicate.
+  // Duplicates are checked for when an offer is created, and so should not
+  // exist. But emperical testing shows that they do. This function is called
+  // periodically by a timer, to clean up any duplicates that slipped through
+  // the cracks.
+  async removeDuplicateOffers () {
+    try {
+      const now = new Date()
+      console.log(`Starting removeDuplicateOffers() at ${now.toLocaleString()}`)
+
+      let duplicateFound = false
+
+      // Get all Offers in the database.
+      const offers = await this.OfferModel.find({})
+      // console.log('offers: ', offers)
+
+      const offerZcids = []
+
+      for (let i = 0; i < offers.length; i++) {
+        const thisOffer = offers[i]
+
+        if (offerZcids.includes(thisOffer.p2wdbHash)) {
+          await thisOffer.remove()
+          duplicateFound = true
+          continue
+        }
+
+        // Add the zcid to the array.
+        offerZcids.push(thisOffer.p2wdbHash)
+      }
+
+      return duplicateFound
+    } catch (err) {
+      console.error('Error in removeDuplicateOffers()')
+      throw err
+    }
+  }
+
   // This function is called by the garbage collection timer controller. It
   // checks the UTXO associated with each Offer in the database. If the UTXO
   // has been spent, the Offer is deleted from the database.

src/use-cases/order.js

@@ -2,9 +2,11 @@
   Order use-case library.
 */
 
+// Global npm libraries
+import RetryQueue from '@chris.troutner/retry-queue'
+
 // Local libraries
 import wlogger from '../adapters/wlogger.js'
-
 import OrderEntity from '../entities/order.js'
 import config from '../../config/index.js'
 
@@ -23,6 +25,10 @@ class OrderLib {
     this.OrderModel = this.adapters.localdb.Order
     this.bch = this.adapters.bch
     this.config = config
+    this.retryQueue = new RetryQueue({ retryPeriod: 1000, attempts: 3 })
+
+    // Bind subfunctions to the 'this' object.
+    this.ensureFunds = this.ensureFunds.bind(this)
   }
 
   // Create a new order model and add it to the Mongo database.
@@ -37,33 +43,33 @@ class OrderLib {
       entryObj.makerAddr = this.adapters.wallet.bchWallet.walletInfo.cashAddress
       console.log('entryObj.makerAddr: ', entryObj.makerAddr)
 
-      // Get Ticker for token ID.
-      // TODO: Move this below the orderEntity.validate() call.
-      const tokenData = await this.adapters.wallet.bchWallet.getTxData([entryObj.tokenId])
-      // console.log(`tokenData: ${JSON.stringify(tokenData, null, 2)}`)
-      entryObj.ticker = tokenData[0].tokenTicker
-
       // Input Validation
-      // TODO: Remove ticker from validation.
-      // TODO: Rename validate() to inputValidate(). Create fullValidate() that
-      // validates a fully hydrated Order entity.
-      const orderEntity = this.orderEntity.validate(entryObj)
+      const orderEntity = this.orderEntity.inputValidate(entryObj)
       console.log('orderEntity: ', orderEntity)
 
       // Ensure sufficient tokens exist to create the order.
-      await this.ensureFunds(orderEntity)
+      // await this.ensureFunds(orderEntity)
+      await this.retryQueue.addToQueue(this.ensureFunds, orderEntity)
+
+      // Get Ticker for token ID.
+      // const tokenData = await this.adapters.wallet.bchWallet.getTxData([entryObj.tokenId])
+      const tokenData = await this.retryQueue.addToQueue(this.adapters.wallet.bchWallet.getTxData, [entryObj.tokenId])
+      // console.log(`tokenData: ${JSON.stringify(tokenData, null, 2)}`)
+      entryObj.ticker = tokenData[0].tokenTicker
 
       // Move the tokens to holding address.
       const moveObj = {
         tokenId: orderEntity.tokenId,
         qty: orderEntity.numTokens
       }
-      const utxoInfo = await this.adapters.wallet.moveTokens(moveObj)
+      // const utxoInfo = await this.adapters.wallet.moveTokens(moveObj)
+      const utxoInfo = await this.retryQueue.addToQueue(this.adapters.wallet.moveTokens, moveObj)
       console.log('utxoInfo: ', utxoInfo)
 
       // Update the UTXO store for the wallet.
       await this.adapters.wallet.bchWallet.bchjs.Util.sleep(3000)
-      await this.adapters.wallet.bchWallet.getUtxos()
+      // await this.adapters.wallet.bchWallet.getUtxos()
+      await this.retryQueue.addToQueue(this.adapters.wallet.bchWallet.initialize, {})
 
       // Update the order with the new UTXO information.
       orderEntity.utxoTxid = utxoInfo.txid
@@ -79,7 +85,8 @@ class OrderLib {
         data: orderEntity,
         appId: this.config.p2wdbAppId
       }
-      const hash = await this.adapters.p2wdb.write(p2wdbObj)
+      // const hash = await this.adapters.p2wdb.write(p2wdbObj)
+      const hash = await this.retryQueue.addToQueue(this.adapters.p2wdb.write, p2wdbObj)
       // console.log('hash: ', hash)
 
       // Create a MongoDB model to hold the Order

test/unit/entities/order.entity.unit.js

@@ -21,10 +21,10 @@ describe('#Order-Entity', () => {
 
   afterEach(() => sandbox.restore())
 
-  describe('#validate', () => {
+  describe('#inputValidate', () => {
     it('should throw an error if data is not provided', () => {
       try {
-        uut.validate()
+        uut.inputValidate()
       } catch (err) {
         // console.log(err)
         assert.include(err.message, 'Cannot destructure property')
@@ -34,7 +34,7 @@ describe('#Order-Entity', () => {
     it('should throw an error if messageType is not included', () => {
       try {
         const data = {}
-        uut.validate(data)
+        uut.inputValidate(data)
       } catch (err) {
         // console.log(err)
         assert.include(
@@ -47,7 +47,7 @@ describe('#Order-Entity', () => {
     it('should throw an error if messageClass is not included', () => {
       try {
         const data = { messageType: 1 }
-        uut.validate(data)
+        uut.inputValidate(data)
       } catch (err) {
         // console.log(err)
         assert.include(
@@ -60,7 +60,7 @@ describe('#Order-Entity', () => {
     it('should throw an error if tokenId is not included', () => {
       try {
         const data = { messageType: 1, messageClass: 1 }
-        uut.validate(data)
+        uut.inputValidate(data)
       } catch (err) {
         // console.log(err)
         assert.include(err.message, "Property 'tokenId' must be a string.")
@@ -70,7 +70,7 @@ describe('#Order-Entity', () => {
     it('should throw an error if buyOrSell is not included', () => {
       try {
         const data = { messageType: 1, messageClass: 1, tokenId: 'fakeId' }
-        uut.validate(data)
+        uut.inputValidate(data)
       } catch (err) {
         // console.log(err)
         assert.include(err.message, "Property 'buyOrSell' must be a string.")
@@ -85,7 +85,7 @@ describe('#Order-Entity', () => {
           tokenId: 'fakeId',
           buyOrSell: 'buy'
         }
-        uut.validate(data)
+        uut.inputValidate(data)
       } catch (err) {
         // console.log(err)
         assert.include(
@@ -104,7 +104,7 @@ describe('#Order-Entity', () => {
           buyOrSell: 'buy',
           rateInBaseUnit: 1000
         }
-        uut.validate(data)
+        uut.inputValidate(data)
       } catch (err) {
         // console.log(err)
         assert.include(
@@ -124,7 +124,7 @@ describe('#Order-Entity', () => {
           rateInBaseUnit: 1000,
           minUnitsToExchange: 350
         }
-        uut.validate(data)
+        uut.inputValidate(data)
       } catch (err) {
         // console.log(err)
         assert.include(err.message, "Property 'numTokens' must be a number.")

test/unit/mocks/adapters/wallet.js

@@ -35,6 +35,8 @@ class MockBchWallet {
       }]
     }
     this.getTokenData = async () => {}
+    this.initialize = async () => {}
+    this.utxoIsValid = async () => {}
 
     // Environment variable is used by wallet-balance.unit.js to force an error.
     if (process.env.NO_UTXO) {

test/unit/use-cases/offer.use-case.unit.js

@@ -74,7 +74,7 @@ describe('#offer-use-case', () => {
       }
 
       // Mock dependencies
-      sandbox.stub(uut.adapters.bchjs.Blockchain, 'getTxOut').resolves(null)
+      sandbox.stub(uut.adapters.wallet.bchWallet, 'utxoIsValid').resolves(false)
       sandbox.stub(uut.adapters.wallet.bchWallet, 'getTokenData').resolves({})
       sandbox.stub(uut, 'categorizeToken').resolves('nft')
       sandbox.stub(uut, 'detectNsfw').resolves(false)
@@ -111,18 +111,7 @@ describe('#offer-use-case', () => {
       }
 
       // Mock dependencies
-      sandbox.stub(uut.adapters.bchjs.Blockchain, 'getTxOut').resolves({
-        bestblock:
-          '000000000000000000d2060b83f90f8187b92fcccb4a42aaa19ce5a305fe0ae3',
-        confirmations: 0,
-        value: 0,
-        scriptPubKey: {
-          asm: 'OP_RETURN 5262419 1 1145980243 38e97c5d7d3585a2cbf3f9580c82ca33985f9cb0845d4dcce220cb709f9538b0 00000000001e8480 0000000009a7ec80',
-          hex: '6a04534c500001010453454e442038e97c5d7d3585a2cbf3f9580c82ca33985f9cb0845d4dcce220cb709f9538b00800000000001e8480080000000009a7ec80',
-          type: 'nulldata'
-        },
-        coinbase: false
-      })
+      sandbox.stub(uut.adapters.wallet.bchWallet, 'utxoIsValid').resolves(true)
       sandbox.stub(uut, 'categorizeToken').resolves('fungible')
       sandbox.stub(uut.adapters.wallet.bchWallet, 'getTokenData').resolves({})
       sandbox.stub(uut, 'detectNsfw').resolves(false)
@@ -171,4 +160,35 @@ describe('#offer-use-case', () => {
       assert.equal(result, 'fungible')
     })
   })
+
+  describe('#removeDuplicateOffers', () => {
+    it('should remove duplicate entries and return true', async () => {
+      // Mock dependencies and force desired code path.
+      sandbox.stub(uut.OfferModel, 'find').resolves([
+        { p2wdbHash: 'a', remove: async () => {} },
+        { p2wdbHash: 'a', remove: async () => {} },
+        { p2wdbHash: 'b', remove: async () => {} }
+      ])
+      // sandbox.stub(uut.OfferModel, 'remove').resolves()
+
+      const result = await uut.removeDuplicateOffers()
+      console.log('result: ', result)
+
+      assert.equal(result, true)
+    })
+
+    it('should return false if there are no duplicate entries', async () => {
+      // Mock dependencies and force desired code path.
+      sandbox.stub(uut.OfferModel, 'find').resolves([
+        { p2wdbHash: 'a', remove: async () => {} },
+        { p2wdbHash: 'b', remove: async () => {} }
+      ])
+      // sandbox.stub(uut.OfferModel, 'remove').resolves()
+
+      const result = await uut.removeDuplicateOffers()
+      console.log('result: ', result)
+
+      assert.equal(result, false)
+    })
+  })
 })

test/unit/use-cases/order.use-case.unit.js

@@ -106,14 +106,12 @@ describe('#order-use-case', () => {
         numTokens: 1
       }
 
-      // Mock dependencies
-      // sandbox.stub(uut.adapters.wallet, 'burnPsf').resolves('fakeTxid')
-      // sandbox.stub(uut.adapters.wallet.bchWallet, 'getTxData').resolves({ tokenTicker: 'TROUT' })
-      sandbox.stub(uut.orderEntity, 'validate').returns(entryObj)
+      // Mock dependencies and force expected code path
+      sandbox.stub(uut.orderEntity, 'inputValidate').returns(entryObj)
       sandbox.stub(uut, 'ensureFunds').resolves()
       sandbox.stub(uut.adapters.wallet.bchWallet.bchjs.Util, 'sleep').resolves()
       sandbox.stub(uut.adapters.wallet, 'moveTokens').resolves({ txid: 'fakeTxid', vout: 0, hdIndex: 1 })
-      sandbox.stub(uut.adapters.wallet.bchWallet, 'getUtxos').resolves()
+      sandbox.stub(uut.adapters.wallet.bchWallet, 'initialize').resolves()
       sandbox.stub(uut.adapters.p2wdb, 'write').resolves('fakeHash')
 
       const result = await uut.createOrder(entryObj)

util/offers/deleteOffers.js

@@ -2,15 +2,15 @@
   This script has not been customized for offers yet.
 */
 
-const mongoose = require('mongoose')
+import mongoose from 'mongoose'
 
 // Force test environment
 // make sure environment variable is set before this file gets called.
 // see test script in package.json.
 // process.env.KOA_ENV = 'test'
-const config = require('../../config')
+import config from '../../config'
 
-const User = require('../../src/models/users')
+import User from '../../src/models/users'
 
 async function deleteUsers () {
   // Connect to the Mongo Database.

util/offers/getOffers.js

@@ -2,11 +2,10 @@
   Get all Offers in the database.
 */
 
-const mongoose = require('mongoose')
+import mongoose from 'mongoose'
 
-const config = require('../../config')
-
-const Offer = require('../../src/adapters/localdb/models/offer')
+import config from '../../config/index.js'
+import Offer from '../../src/adapters/localdb/models/offer.js'
 
 async function getOffers () {
   // Connect to the Mongo Database.

util/orders/deleteOrders.js

@@ -1,12 +1,12 @@
-const mongoose = require('mongoose')
+import mongoose from 'mongoose'
 
 // Force test environment
 // make sure environment variable is set before this file gets called.
 // see test script in package.json.
 // process.env.KOA_ENV = 'test'
-const config = require('../../config')
+import config from '../../config'
 
-const User = require('../../src/models/users')
+import User from '../../src/models/users'
 
 async function deleteUsers () {
   // Connect to the Mongo Database.

util/orders/getOrders.js

@@ -1,8 +1,6 @@
-const mongoose = require('mongoose')
-
-const config = require('../../config')
-
-const Order = require('../../src/adapters/localdb/models/order')
+import mongoose from 'mongoose'
+import config from '../../config'
+import Order from '../../src/adapters/localdb/models/order'
 
 async function getOrder () {
   // Connect to the Mongo Database.

util/wallet/sweep-funds.js

@@ -7,11 +7,12 @@
 */
 
 // Public npm libraries
-const BCHJS = require('@psf/bch-js')
-const BchTokenSweep = require('bch-token-sweep/index')
+import BCHJS from '@psf/bch-js'
+
+import BchTokenSweep from 'bch-token-sweep/index'
 
 // Local libraries
-const WalletAdapter = require('../../src/adapters/wallet')
+import WalletAdapter from '../../src/adapters/wallet'
 
 // Constants
 const EMTPY_ADDR_CUTOFF = 15