Merge branch 'master' into transactionControllerRefractorPt3

8 files changed, 233 insertions, 54 deletions

diff --git a/app/manifest.json b/app/manifest.json
index 293981d70..7ec32ea78 100644
--- a/app/manifest.json
+++ b/app/manifest.json
@@ -1,7 +1,7 @@
 {
   "name": "MetaMask",
   "short_name": "Metamask",
-  "version": "3.9.9",
+  "version": "3.9.11",
   "manifest_version": 2,
   "author": "https://metamask.io",
   "description": "Ethereum Browser Extension",
diff --git a/app/scripts/controllers/transactions.js b/app/scripts/controllers/transactions.js
index 6c4701283..3f2a7ce16 100644
--- a/app/scripts/controllers/transactions.js
+++ b/app/scripts/controllers/transactions.js
@@ -55,6 +55,17 @@ module.exports = class TransactionController extends EventEmitter {
           status: 'submitted',
         })
       },
+      getConfirmedTransactions: (address) => {
+        return this.getFilteredTxList({
+          from: address,
+          status: 'confirmed',
+          err: undefined,
+        })
+      },
+      giveUpOnTransaction: (txId) => {
+        const msg = `Gave up submitting after 3500 blocks un-mined.`
+        this.setTxStatusFailed(txId, msg)
+      },
     })
 
     this.pendingTxTracker = new PendingTransactionTracker({
@@ -259,4 +270,4 @@ module.exports = class TransactionController extends EventEmitter {
     })
     this.memStore.updateState({ unapprovedTxs, selectedAddressTxList })
   }
-}
\ No newline at end of file
+}
diff --git a/app/scripts/lib/auto-reload.js b/app/scripts/lib/auto-reload.js
index 6abce73ea..cce31c3d2 100644
--- a/app/scripts/lib/auto-reload.js
+++ b/app/scripts/lib/auto-reload.js
@@ -2,33 +2,55 @@ module.exports = setupDappAutoReload
 
 function setupDappAutoReload (web3, observable) {
   // export web3 as a global, checking for usage
+  let hasBeenWarned = false
+  let reloadInProgress = false
+  let lastTimeUsed
+  let lastSeenNetwork
+
   global.web3 = new Proxy(web3, {
-    get: (_web3, name) => {
-      // get the time of use
-      if (name !== '_used') {
-        console.warn('MetaMask: web3 will be deprecated in the near future in favor of the ethereumProvider \nhttps://github.com/ethereum/mist/releases/tag/v0.9.0')
-        _web3._used = Date.now()
+    get: (_web3, key) => {
+      // show warning once on web3 access
+      if (!hasBeenWarned && key !== 'currentProvider') {
+        console.warn('MetaMask: web3 will be deprecated in the near future in favor of the ethereumProvider \nhttps://github.com/MetaMask/faq/blob/master/detecting_metamask.md#web3-deprecation')
+        hasBeenWarned = true
       }
-      return _web3[name]
+      // get the time of use
+      lastTimeUsed = Date.now()
+      // return value normally
+      return _web3[key]
     },
-    set: (_web3, name, value) => {
-      _web3[name] = value
+    set: (_web3, key, value) => {
+      // set value normally
+      _web3[key] = value
     },
   })
-  var networkVersion
 
   observable.subscribe(function (state) {
-    // get the initial network
-    const curentNetVersion = state.networkVersion
-    if (!networkVersion) networkVersion = curentNetVersion
-
-    if (curentNetVersion !== networkVersion && web3._used) {
-      const timeSinceUse = Date.now() - web3._used
-      // if web3 was recently used then delay the reloading of the page
-      timeSinceUse > 500 ? triggerReset() : setTimeout(triggerReset, 500)
-      // prevent reentry into if statement if state updates again before
-      // reload
-      networkVersion = curentNetVersion
+    // if reload in progress, no need to check reload logic
+    if (reloadInProgress) return
+
+    const currentNetwork = state.networkVersion
+
+    // set the initial network
+    if (!lastSeenNetwork) {
+      lastSeenNetwork = currentNetwork
+      return
+    }
+
+    // skip reload logic if web3 not used
+    if (!lastTimeUsed) return
+
+    // if network did not change, exit
+    if (currentNetwork === lastSeenNetwork) return
+
+    // initiate page reload
+    reloadInProgress = true
+    const timeSinceUse = Date.now() - lastTimeUsed
+    // if web3 was recently used then delay the reloading of the page
+    if (timeSinceUse > 500) {
+      triggerReset()
+    } else {
+      setTimeout(triggerReset, 500)
     }
   })
 }
diff --git a/app/scripts/lib/inpage-provider.js b/app/scripts/lib/inpage-provider.js
index fd032a673..c63af06dc 100644
--- a/app/scripts/lib/inpage-provider.js
+++ b/app/scripts/lib/inpage-provider.js
@@ -27,7 +27,7 @@ function MetamaskInpageProvider (connectionStream) {
   )
 
   // ignore phishing warning message (handled elsewhere)
-  multiStream.ignoreStream('phishing') 
+  multiStream.ignoreStream('phishing')
 
   // connect to async provider
   const asyncProvider = self.asyncProvider = new StreamProvider()
@@ -43,8 +43,9 @@ function MetamaskInpageProvider (connectionStream) {
   // handle sendAsync requests via asyncProvider
   self.sendAsync = function (payload, cb) {
     // rewrite request ids
-    var request = eachJsonMessage(payload, (message) => {
-      var newId = createRandomId()
+    var request = eachJsonMessage(payload, (_message) => {
+      const message = Object.assign({}, _message)
+      const newId = createRandomId()
       self.idMap[newId] = message.id
       message.id = newId
       return message
@@ -80,7 +81,7 @@ MetamaskInpageProvider.prototype.send = function (payload) {
     case 'eth_coinbase':
       // read from localStorage
       selectedAddress = self.publicConfigStore.getState().selectedAddress
-      result = selectedAddress
+      result = selectedAddress || null
       break
 
     case 'eth_uninstallFilter':
@@ -90,7 +91,7 @@ MetamaskInpageProvider.prototype.send = function (payload) {
 
     case 'net_version':
       const networkVersion = self.publicConfigStore.getState().networkVersion
-      result = networkVersion
+      result = networkVersion || null
       break
 
     // throw not-supported Error
diff --git a/app/scripts/lib/nonce-tracker.js b/app/scripts/lib/nonce-tracker.js
index 8328e81ec..0029ac953 100644
--- a/app/scripts/lib/nonce-tracker.js
+++ b/app/scripts/lib/nonce-tracker.js
@@ -1,13 +1,14 @@
-const EthQuery = require('eth-query')
+const EthQuery = require('ethjs-query')
 const assert = require('assert')
 const Mutex = require('await-semaphore').Mutex
 
 class NonceTracker {
 
-  constructor ({ provider, getPendingTransactions }) {
+  constructor ({ provider, getPendingTransactions, getConfirmedTransactions }) {
     this.provider = provider
     this.ethQuery = new EthQuery(provider)
     this.getPendingTransactions = getPendingTransactions
+    this.getConfirmedTransactions = getConfirmedTransactions
     this.lockMap = {}
   }
 
@@ -25,21 +26,28 @@ class NonceTracker {
     await this._globalMutexFree()
     // await lock free, then take lock
     const releaseLock = await this._takeMutex(address)
-    // calculate next nonce
-    // we need to make sure our base count
-    // and pending count are from the same block
-    const currentBlock = await this._getCurrentBlock()
-    const pendingTransactions = this.getPendingTransactions(address)
-    const pendingCount = pendingTransactions.length
-    assert(Number.isInteger(pendingCount), `nonce-tracker - pendingCount is not an integer - got: (${typeof pendingCount}) "${pendingCount}"`)
-    const baseCountHex = await this._getTxCount(address, currentBlock)
-    const baseCount = parseInt(baseCountHex, 16)
-    assert(Number.isInteger(baseCount), `nonce-tracker - baseCount is not an integer - got: (${typeof baseCount}) "${baseCount}"`)
-    const nextNonce = baseCount + pendingCount
+    // evaluate multiple nextNonce strategies
+    const nonceDetails = {}
+    const networkNonceResult = await this._getNetworkNextNonce(address)
+    const highestLocallyConfirmed = this._getHighestLocallyConfirmed(address)
+    const nextNetworkNonce = networkNonceResult.nonce
+    const highestLocalNonce = highestLocallyConfirmed
+    const highestSuggested = Math.max(nextNetworkNonce, highestLocalNonce)
+
+    const pendingTxs = this.getPendingTransactions(address)
+    const localNonceResult = this._getHighestContinuousFrom(pendingTxs, highestSuggested) || 0
+
+    nonceDetails.params = {
+      highestLocalNonce,
+      highestSuggested,
+      nextNetworkNonce,
+    }
+    nonceDetails.local = localNonceResult
+    nonceDetails.network = networkNonceResult
+
+    const nextNonce = Math.max(networkNonceResult.nonce, localNonceResult.nonce)
     assert(Number.isInteger(nextNonce), `nonce-tracker - nextNonce is not an integer - got: (${typeof nextNonce}) "${nextNonce}"`)
-    // collect the numbers used to calculate the nonce for debugging
-    const blockNumber = currentBlock.number
-    const nonceDetails = { blockNumber, baseCount, baseCountHex, pendingCount }
+
     // return nonce and release cb
     return { nextNonce, nonceDetails, releaseLock }
   }
@@ -53,15 +61,6 @@ class NonceTracker {
     })
   }
 
-  async _getTxCount (address, currentBlock) {
-    const blockNumber = currentBlock.number
-    return new Promise((resolve, reject) => {
-      this.ethQuery.getTransactionCount(address, blockNumber, (err, result) => {
-        err ? reject(err) : resolve(result)
-      })
-    })
-  }
-
   async _globalMutexFree () {
     const globalMutex = this._lookupMutex('global')
     const release = await globalMutex.acquire()
@@ -83,12 +82,68 @@ class NonceTracker {
     return mutex
   }
 
+  async _getNetworkNextNonce (address) {
+    // calculate next nonce
+    // we need to make sure our base count
+    // and pending count are from the same block
+    const currentBlock = await this._getCurrentBlock()
+    const blockNumber = currentBlock.blockNumber
+    const baseCountBN = await this.ethQuery.getTransactionCount(address, blockNumber || 'latest')
+    const baseCount = baseCountBN.toNumber()
+    assert(Number.isInteger(baseCount), `nonce-tracker - baseCount is not an integer - got: (${typeof baseCount}) "${baseCount}"`)
+    const nonceDetails = { blockNumber, baseCount }
+    return { name: 'network', nonce: baseCount, details: nonceDetails }
+  }
+
+  _getHighestLocallyConfirmed (address) {
+    const confirmedTransactions = this.getConfirmedTransactions(address)
+    const highest = this._getHighestNonce(confirmedTransactions)
+    return Number.isInteger(highest) ? highest + 1 : 0
+  }
+
+  _reduceTxListToUniqueNonces (txList) {
+    const reducedTxList = txList.reduce((reducedList, txMeta, index) => {
+      if (!index) return [txMeta]
+      const nonceMatches = txList.filter((txData) => {
+        return txMeta.txParams.nonce === txData.txParams.nonce
+      })
+      if (nonceMatches.length > 1) return reducedList
+      reducedList.push(txMeta)
+      return reducedList
+    }, [])
+    return reducedTxList
+  }
+
+  _getHighestNonce (txList) {
+    const nonces = txList.map((txMeta) => {
+      const nonce = txMeta.txParams.nonce
+      assert(typeof nonce, 'string', 'nonces should be hex strings')
+      return parseInt(nonce, 16)
+    })
+    const highestNonce = Math.max.apply(null, nonces)
+    return highestNonce
+  }
+
+  _getHighestContinuousFrom (txList, startPoint) {
+    const nonces = txList.map((txMeta) => {
+      const nonce = txMeta.txParams.nonce
+      assert(typeof nonce, 'string', 'nonces should be hex strings')
+      return parseInt(nonce, 16)
+    })
+
+    let highest = startPoint
+    while (nonces.includes(highest)) {
+      highest++
+    }
+
+    return { name: 'local', nonce: highest, details: { startPoint, highest } }
+  }
+
   // this is a hotfix for the fact that the blockTracker will
   // change when the network changes
   _getBlockTracker () {
     return this.provider._blockTracker
   }
-
 }
 
 module.exports = NonceTracker
diff --git a/app/scripts/lib/pending-tx-tracker.js b/app/scripts/lib/pending-tx-tracker.js
index 19720db3f..b90851b58 100644
--- a/app/scripts/lib/pending-tx-tracker.js
+++ b/app/scripts/lib/pending-tx-tracker.js
@@ -1,6 +1,7 @@
 const EventEmitter = require('events')
 const EthQuery = require('ethjs-query')
 const sufficientBalance = require('./util').sufficientBalance
+const RETRY_LIMIT = 3500 // Retry 3500 blocks, or about 1 day.
 /*
 
   Utility class for tracking the transactions as they
@@ -28,6 +29,7 @@ module.exports = class PendingTransactionTracker extends EventEmitter {
     this.getBalance = config.getBalance
     this.getPendingTransactions = config.getPendingTransactions
     this.publishTransaction = config.publishTransaction
+    this.giveUpOnTransaction = config.giveUpOnTransaction
   }
 
   //  checks if a signed tx is in a block and
@@ -100,6 +102,10 @@ module.exports = class PendingTransactionTracker extends EventEmitter {
     if (balance === undefined) return
     if (!('retryCount' in txMeta)) txMeta.retryCount = 0
 
+    if (txMeta.retryCount > RETRY_LIMIT) {
+      return this.giveUpOnTransaction(txMeta.id)
+    }
+
     // if the value of the transaction is greater then the balance, fail.
     if (!sufficientBalance(txMeta.txParams, balance)) {
       const insufficientFundsError = new Error('Insufficient balance during rebroadcast.')
@@ -160,4 +166,4 @@ module.exports = class PendingTransactionTracker extends EventEmitter {
     }
     nonceGlobalLock.releaseLock()
   }
-}
\ No newline at end of file
+}
diff --git a/app/scripts/migrations/019.js b/app/scripts/migrations/019.js
new file mode 100644
index 000000000..072c96370
--- /dev/null
+++ b/app/scripts/migrations/019.js
@@ -0,0 +1,83 @@
+
+const version = 19
+
+/*
+
+This migration sets transactions as failed
+whos nonce is too high
+
+*/
+
+const clone = require('clone')
+
+module.exports = {
+  version,
+
+  migrate: function (originalVersionedData) {
+    const versionedData = clone(originalVersionedData)
+    versionedData.meta.version = version
+    try {
+      const state = versionedData.data
+      const newState = transformState(state)
+      versionedData.data = newState
+    } catch (err) {
+      console.warn(`MetaMask Migration #${version}` + err.stack)
+    }
+    return Promise.resolve(versionedData)
+  },
+}
+
+function transformState (state) {
+  const newState = state
+  const transactions = newState.TransactionController.transactions
+
+  newState.TransactionController.transactions = transactions.map((txMeta, _, txList) => {
+    if (txMeta.status !== 'submitted') return txMeta
+
+    const confirmedTxs = txList.filter((tx) => tx.status === 'confirmed')
+    .filter((tx) => tx.txParams.from === txMeta.txParams.from)
+    .filter((tx) => tx.metamaskNetworkId.from === txMeta.metamaskNetworkId.from)
+    const highestConfirmedNonce = getHighestNonce(confirmedTxs)
+
+    const pendingTxs = txList.filter((tx) => tx.status === 'submitted')
+    .filter((tx) => tx.txParams.from === txMeta.txParams.from)
+    .filter((tx) => tx.metamaskNetworkId.from === txMeta.metamaskNetworkId.from)
+    const highestContinuousNonce = getHighestContinuousFrom(pendingTxs, highestConfirmedNonce)
+
+    const maxNonce = Math.max(highestContinuousNonce, highestConfirmedNonce)
+
+    if (parseInt(txMeta.txParams.nonce, 16) > maxNonce + 1) {
+      txMeta.status = 'failed'
+      txMeta.err = {
+        message: 'nonce too high',
+        note: 'migration 019 custom error',
+      }
+    }
+    return txMeta
+  })
+  return newState
+}
+
+function getHighestContinuousFrom (txList, startPoint) {
+  const nonces = txList.map((txMeta) => {
+    const nonce = txMeta.txParams.nonce
+    return parseInt(nonce, 16)
+  })
+
+  let highest = startPoint
+  while (nonces.includes(highest)) {
+    highest++
+  }
+
+  return highest
+}
+
+function getHighestNonce (txList) {
+  const nonces = txList.map((txMeta) => {
+  const nonce = txMeta.txParams.nonce
+    return parseInt(nonce || '0x0', 16)
+  })
+  const highestNonce = Math.max.apply(null, nonces)
+  return highestNonce
+}
+
diff --git a/app/scripts/migrations/index.js b/app/scripts/migrations/index.js
index 6bfc622f7..e9cbd7b98 100644
--- a/app/scripts/migrations/index.js
+++ b/app/scripts/migrations/index.js
@@ -29,4 +29,5 @@ module.exports = [
   require('./016'),
   require('./017'),
   require('./018'),
+  require('./019'),
 ]