Merge branch 'development' into feature/website/0x-org

2 files changed, 55 insertions, 29 deletions

diff --git a/packages/pipeline/src/scripts/pull_erc20_events.ts b/packages/pipeline/src/scripts/pull_erc20_events.ts
index 0ad12c97a..bd520c610 100644
--- a/packages/pipeline/src/scripts/pull_erc20_events.ts
+++ b/packages/pipeline/src/scripts/pull_erc20_events.ts
@@ -1,10 +1,10 @@
-// tslint:disable:no-console
 import { getContractAddressesForNetworkOrThrow } from '@0x/contract-addresses';
 import { web3Factory } from '@0x/dev-utils';
 import { Web3ProviderEngine } from '@0x/subproviders';
+import { logUtils } from '@0x/utils';
 import { Web3Wrapper } from '@0x/web3-wrapper';
 import 'reflect-metadata';
-import { Connection, ConnectionOptions, createConnection, Repository } from 'typeorm';
+import { Connection, ConnectionOptions, createConnection } from 'typeorm';
 
 import { ERC20EventsSource } from '../data_sources/contract-wrappers/erc20_events';
 import { ERC20ApprovalEvent } from '../entities';
@@ -16,33 +16,63 @@ const NETWORK_ID = 1;
 const START_BLOCK_OFFSET = 100; // Number of blocks before the last known block to consider when updating fill events.
 const BATCH_SAVE_SIZE = 1000; // Number of events to save at once.
 const BLOCK_FINALITY_THRESHOLD = 10; // When to consider blocks as final. Used to compute default endBlock.
-const WETH_START_BLOCK = 4719568; // Block number when the WETH contract was deployed.
 
 let connection: Connection;
 
+interface Token {
+    // name is used for logging only.
+    name: string;
+    address: string;
+    defaultStartBlock: number;
+}
+
+const tokensToGetApprovalEvents: Token[] = [
+    {
+        name: 'WETH',
+        address: getContractAddressesForNetworkOrThrow(NETWORK_ID).etherToken,
+        defaultStartBlock: 4719568, // Block when the WETH contract was deployed.
+    },
+    {
+        name: 'ZRX',
+        address: getContractAddressesForNetworkOrThrow(NETWORK_ID).zrxToken,
+        defaultStartBlock: 4145415, // Block when the ZRX contract was deployed.
+    },
+    {
+        name: 'DAI',
+        address: '0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359',
+        defaultStartBlock: 4752008, // Block when the DAI contract was deployed.
+    },
+];
+
 (async () => {
     connection = await createConnection(ormConfig as ConnectionOptions);
     const provider = web3Factory.getRpcProvider({
         rpcUrl: INFURA_ROOT_URL,
     });
     const endBlock = await calculateEndBlockAsync(provider);
-    await getAndSaveWETHApprovalEventsAsync(provider, endBlock);
+    for (const token of tokensToGetApprovalEvents) {
+        await getAndSaveApprovalEventsAsync(provider, token, endBlock);
+    }
     process.exit(0);
 })().catch(handleError);
 
-async function getAndSaveWETHApprovalEventsAsync(provider: Web3ProviderEngine, endBlock: number): Promise<void> {
-    console.log('Checking existing approval events...');
+async function getAndSaveApprovalEventsAsync(
+    provider: Web3ProviderEngine,
+    token: Token,
+    endBlock: number,
+): Promise<void> {
+    logUtils.log(`Getting approval events for ${token.name}...`);
+    logUtils.log('Checking existing approval events...');
     const repository = connection.getRepository(ERC20ApprovalEvent);
-    const startBlock = (await getStartBlockAsync(repository)) || WETH_START_BLOCK;
+    const startBlock = (await getStartBlockAsync(token)) || token.defaultStartBlock;
 
-    console.log(`Getting WETH approval events starting at ${startBlock}...`);
-    const wethTokenAddress = getContractAddressesForNetworkOrThrow(NETWORK_ID).etherToken;
-    const eventsSource = new ERC20EventsSource(provider, NETWORK_ID, wethTokenAddress);
+    logUtils.log(`Getting approval events starting at ${startBlock}...`);
+    const eventsSource = new ERC20EventsSource(provider, NETWORK_ID, token.address);
     const eventLogs = await eventsSource.getApprovalEventsAsync(startBlock, endBlock);
 
-    console.log(`Parsing ${eventLogs.length} WETH approval events...`);
+    logUtils.log(`Parsing ${eventLogs.length} approval events...`);
     const events = parseERC20ApprovalEvents(eventLogs);
-    console.log(`Retrieved and parsed ${events.length} total WETH approval events.`);
+    logUtils.log(`Retrieved and parsed ${events.length} total approval events.`);
     await repository.save(events, { chunk: Math.ceil(events.length / BATCH_SAVE_SIZE) });
 }
 
@@ -52,15 +82,15 @@ async function calculateEndBlockAsync(provider: Web3ProviderEngine): Promise<num
     return currentBlock - BLOCK_FINALITY_THRESHOLD;
 }
 
-async function getStartBlockAsync(repository: Repository<ERC20ApprovalEvent>): Promise<number | null> {
-    const fillEventCount = await repository.count();
-    if (fillEventCount === 0) {
-        console.log(`No existing approval events found.`);
-        return null;
-    }
+async function getStartBlockAsync(token: Token): Promise<number | null> {
     const queryResult = await connection.query(
-        `SELECT block_number FROM raw.erc20_approval_events ORDER BY block_number DESC LIMIT 1`,
+        `SELECT block_number FROM raw.erc20_approval_events WHERE token_address = $1 ORDER BY block_number DESC LIMIT 1`,
+        [token.address],
     );
+    if (queryResult.length === 0) {
+        logUtils.log(`No existing approval events found for ${token.name}.`);
+        return null;
+    }
     const lastKnownBlock = queryResult[0].block_number;
     return lastKnownBlock - START_BLOCK_OFFSET;
 }
diff --git a/packages/pipeline/src/scripts/pull_missing_blocks.ts b/packages/pipeline/src/scripts/pull_missing_blocks.ts
index a5203824c..bb5385126 100644
--- a/packages/pipeline/src/scripts/pull_missing_blocks.ts
+++ b/packages/pipeline/src/scripts/pull_missing_blocks.ts
@@ -9,7 +9,7 @@ import { Web3Source } from '../data_sources/web3';
 import { Block } from '../entities';
 import * as ormConfig from '../ormconfig';
 import { parseBlock } from '../parsers/web3';
-import { EXCHANGE_START_BLOCK, handleError, INFURA_ROOT_URL } from '../utils';
+import { handleError, INFURA_ROOT_URL } from '../utils';
 
 // Number of blocks to save at once.
 const BATCH_SAVE_SIZE = 1000;
@@ -37,22 +37,19 @@ interface MissingBlocksResponse {
 
 async function getAllMissingBlocksAsync(web3Source: Web3Source): Promise<void> {
     const blocksRepository = connection.getRepository(Block);
-    let fromBlock = EXCHANGE_START_BLOCK;
     while (true) {
-        const blockNumbers = await getMissingBlockNumbersAsync(fromBlock);
+        const blockNumbers = await getMissingBlockNumbersAsync();
         if (blockNumbers.length === 0) {
             // There are no more missing blocks. We're done.
             break;
         }
         await getAndSaveBlocksAsync(web3Source, blocksRepository, blockNumbers);
-        fromBlock = Math.max(...blockNumbers) + 1;
     }
     const totalBlocks = await blocksRepository.count();
     console.log(`Done saving blocks. There are now ${totalBlocks} total blocks.`);
 }
 
-async function getMissingBlockNumbersAsync(fromBlock: number): Promise<number[]> {
-    console.log(`Checking for missing blocks starting at ${fromBlock}...`);
+async function getMissingBlockNumbersAsync(): Promise<number[]> {
     // Note(albrow): The easiest way to get all the blocks we need is to
     // consider all the events tables together in a single query. If this query
     // gets too slow, we should consider re-architecting so that we can work on
@@ -66,13 +63,12 @@ async function getMissingBlockNumbersAsync(fromBlock: number): Promise<number[]>
         )
         SELECT DISTINCT(block_number) FROM all_events
             WHERE block_number NOT IN (SELECT number FROM raw.blocks)
-                AND block_number >= $1
-            ORDER BY block_number ASC LIMIT $2`,
-        [fromBlock, MAX_BLOCKS_PER_QUERY],
+            ORDER BY block_number ASC LIMIT $1`,
+        [MAX_BLOCKS_PER_QUERY],
     )) as MissingBlocksResponse[];
     const blockNumberStrings = R.pluck('block_number', response);
     const blockNumbers = R.map(parseInt, blockNumberStrings);
-    console.log(`Found ${blockNumbers.length} missing blocks in the given range.`);
+    console.log(`Found ${blockNumbers.length} missing blocks.`);
     return blockNumbers;
 }