Merge remote-tracking branch 'upstream/development' into website

# Conflicts:
#	.gitignore
#	README.md
#	contracts/core/CHANGELOG.json
#	contracts/examples/package.json
#	contracts/extensions/package.json
#	contracts/interfaces/package.json
#	contracts/libs/package.json
#	contracts/multisig/package.json
#	contracts/protocol/CHANGELOG.json
#	contracts/protocol/DEPLOYS.json
#	contracts/protocol/package.json
#	contracts/test-utils/CHANGELOG.json
#	contracts/test-utils/CHANGELOG.md
#	contracts/test-utils/package.json
#	contracts/tokens/CHANGELOG.json
#	contracts/tokens/package.json
#	contracts/utils/package.json
#	packages/0x.js/CHANGELOG.json
#	packages/0x.js/CHANGELOG.md
#	packages/0x.js/package.json
#	packages/abi-gen-wrappers/CHANGELOG.json
#	packages/abi-gen-wrappers/CHANGELOG.md
#	packages/abi-gen-wrappers/package.json
#	packages/abi-gen/CHANGELOG.json
#	packages/abi-gen/CHANGELOG.md
#	packages/abi-gen/package.json
#	packages/assert/CHANGELOG.json
#	packages/assert/CHANGELOG.md
#	packages/assert/package.json
#	packages/asset-buyer/CHANGELOG.md
#	packages/asset-buyer/package.json
#	packages/base-contract/CHANGELOG.json
#	packages/base-contract/CHANGELOG.md
#	packages/base-contract/package.json
#	packages/connect/CHANGELOG.json
#	packages/connect/CHANGELOG.md
#	packages/connect/package.json
#	packages/contract-wrappers/CHANGELOG.json
#	packages/contract-wrappers/CHANGELOG.md
#	packages/contract-wrappers/package.json
#	packages/dev-tools-pages/package.json
#	packages/dev-utils/CHANGELOG.json
#	packages/dev-utils/CHANGELOG.md
#	packages/dev-utils/package.json
#	packages/ethereum-types/CHANGELOG.json
#	packages/ethereum-types/CHANGELOG.md
#	packages/ethereum-types/package.json
#	packages/fill-scenarios/CHANGELOG.json
#	packages/fill-scenarios/CHANGELOG.md
#	packages/fill-scenarios/package.json
#	packages/instant/package.json
#	packages/instant/src/types.ts
#	packages/json-schemas/CHANGELOG.json
#	packages/json-schemas/CHANGELOG.md
#	packages/json-schemas/package.json
#	packages/metacoin/package.json
#	packages/migrations/CHANGELOG.json
#	packages/migrations/CHANGELOG.md
#	packages/migrations/package.json
#	packages/order-utils/CHANGELOG.json
#	packages/order-utils/CHANGELOG.md
#	packages/order-utils/package.json
#	packages/order-watcher/CHANGELOG.json
#	packages/order-watcher/CHANGELOG.md
#	packages/order-watcher/package.json
#	packages/pipeline/package.json
#	packages/pipeline/src/ormconfig.ts
#	packages/pipeline/src/parsers/ddex_orders/index.ts
#	packages/pipeline/src/parsers/oasis_orders/index.ts
#	packages/pipeline/src/parsers/paradex_orders/index.ts
#	packages/pipeline/src/parsers/token_metadata/index.ts
#	packages/pipeline/src/scripts/pull_competing_dex_trades.ts
#	packages/pipeline/src/scripts/pull_ddex_orderbook_snapshots.ts
#	packages/pipeline/src/scripts/pull_idex_orderbook_snapshots.ts
#	packages/pipeline/src/scripts/pull_missing_blocks.ts
#	packages/pipeline/src/scripts/pull_oasis_orderbook_snapshots.ts
#	packages/pipeline/src/scripts/pull_paradex_orderbook_snapshots.ts
#	packages/pipeline/src/scripts/pull_trusted_tokens.ts
#	packages/pipeline/src/scripts/update_relayer_info.ts
#	packages/pipeline/src/utils/get_ohlcv_trading_pairs.ts
#	packages/pipeline/src/utils/index.ts
#	packages/pipeline/test/parsers/ddex_orders/index_test.ts
#	packages/pipeline/test/parsers/oasis_orders/index_test.ts
#	packages/pipeline/test/parsers/paradex_orders/index_test.ts
#	packages/react-docs/CHANGELOG.json
#	packages/react-docs/CHANGELOG.md
#	packages/react-docs/package.json
#	packages/react-shared/CHANGELOG.json
#	packages/react-shared/CHANGELOG.md
#	packages/react-shared/package.json
#	packages/sol-compiler/CHANGELOG.md
#	packages/sol-compiler/package.json
#	packages/sol-cov/CHANGELOG.json
#	packages/sol-cov/CHANGELOG.md
#	packages/sol-cov/package.json
#	packages/sol-doc/CHANGELOG.json
#	packages/sol-doc/CHANGELOG.md
#	packages/sol-doc/package.json
#	packages/sol-resolver/CHANGELOG.md
#	packages/sol-resolver/package.json
#	packages/sra-spec/CHANGELOG.json
#	packages/sra-spec/CHANGELOG.md
#	packages/sra-spec/package.json
#	packages/subproviders/CHANGELOG.json
#	packages/subproviders/CHANGELOG.md
#	packages/subproviders/package.json
#	packages/testnet-faucets/package.json
#	packages/types/CHANGELOG.md
#	packages/types/package.json
#	packages/typescript-typings/CHANGELOG.json
#	packages/typescript-typings/CHANGELOG.md
#	packages/typescript-typings/package.json
#	packages/utils/CHANGELOG.json
#	packages/utils/CHANGELOG.md
#	packages/utils/package.json
#	packages/web3-wrapper/CHANGELOG.json
#	packages/web3-wrapper/CHANGELOG.md
#	packages/web3-wrapper/package.json
#	packages/website/package.json
#	packages/website/public/index.html
#	packages/website/ts/pages/instant/fee_percentage_slider.tsx
#	yarn.lock

1 files changed, 34 insertions, 10 deletions

diff --git a/packages/pipeline/src/utils/get_ohlcv_trading_pairs.ts b/packages/pipeline/src/utils/get_ohlcv_trading_pairs.ts
index 9d3ef2fba..19f81344e 100644
--- a/packages/pipeline/src/utils/get_ohlcv_trading_pairs.ts
+++ b/packages/pipeline/src/utils/get_ohlcv_trading_pairs.ts
@@ -23,6 +23,18 @@ interface CryptoCompareCoin {
 const TO_CURRENCIES = ['USD', 'EUR', 'ETH', 'USDT'];
 const ETHEREUM_IDENTIFIER = '7605';
 const HTTP_OK_STATUS = 200;
+
+interface StaticPair {
+    fromSymbol: string;
+    toSymbol: string;
+}
+const SPECIAL_CASES: StaticPair[] = [
+    {
+        fromSymbol: 'ETH',
+        toSymbol: 'USD',
+    },
+];
+
 /**
  * Get trading pairs with latest scraped time for OHLCV records
  * @param conn a typeorm Connection to postgres
@@ -44,6 +56,7 @@ export async function fetchOHLCVTradingPairsAsync(
     FROM raw.ohlcv_external
     GROUP BY from_symbol, to_symbol;`);
 
+    // build addressable index: { fromsym: { tosym: time }}
     const latestTradingPairsIndex: { [fromSym: string]: { [toSym: string]: number } } = {};
     latestTradingPairs.forEach(pair => {
         const latestIndex: { [toSym: string]: number } = latestTradingPairsIndex[pair.from_symbol] || {};
@@ -51,6 +64,13 @@ export async function fetchOHLCVTradingPairsAsync(
         latestTradingPairsIndex[pair.from_symbol] = latestIndex;
     });
 
+    // match time to special cases
+    const specialCases: TradingPair[] = SPECIAL_CASES.map(pair => {
+        const latestSavedTime =
+            R.path<number>([pair.fromSymbol, pair.toSymbol], latestTradingPairsIndex) || earliestBackfillTime;
+        return R.assoc('latestSavedTime', latestSavedTime, pair);
+    });
+
     // get token symbols used by Crypto Compare
     const allCoinsResp = await fetchAsync(COINLIST_API);
     if (allCoinsResp.status !== HTTP_OK_STATUS) {
@@ -66,27 +86,31 @@ export async function fetchOHLCVTradingPairsAsync(
     });
 
     // fetch all tokens that are traded on 0x
-    const rawTokenAddresses: Array<{ tokenaddress: string }> = await conn.query(
+    const rawEventTokenAddresses: Array<{ tokenaddress: string }> = await conn.query(
         `SELECT DISTINCT(maker_token_address) as tokenaddress FROM raw.exchange_fill_events UNION
       SELECT DISTINCT(taker_token_address) as tokenaddress FROM raw.exchange_fill_events`,
     );
-    const tokenAddresses = R.pluck('tokenaddress', rawTokenAddresses);
+
+    // tslint:disable-next-line:no-unbound-method
+    const eventTokenAddresses = R.pluck('tokenaddress', rawEventTokenAddresses).map(R.toLower);
 
     // join token addresses with CC symbols
-    const allTokenSymbols: string[] = tokenAddresses
-        .map(tokenAddress => erc20CoinsIndex.get(tokenAddress.toLowerCase()) || '')
-        .filter(x => x);
+    const eventTokenSymbols: string[] = eventTokenAddresses
+        .filter(tokenAddress => erc20CoinsIndex.has(tokenAddress))
+        .map(tokenAddress => erc20CoinsIndex.get(tokenAddress) as string);
 
-    // generate list of all tokens with time of latest existing record OR default earliest time
-    const allTradingPairCombinations: TradingPair[] = R.chain(sym => {
+    // join traded tokens with fiat and latest backfill time
+    const eventTradingPairs: TradingPair[] = R.chain(sym => {
         return TO_CURRENCIES.map(fiat => {
-            return {
+            const pair = {
                 fromSymbol: sym,
                 toSymbol: fiat,
                 latestSavedTime: R.path<number>([sym, fiat], latestTradingPairsIndex) || earliestBackfillTime,
             };
+            return pair;
         });
-    }, allTokenSymbols);
+    }, eventTokenSymbols);
 
-    return allTradingPairCombinations;
+    // join with special cases
+    return R.concat(eventTradingPairs, specialCases);
 }