1 files changed, 23 insertions, 14 deletions

diff --git a/packages/pipeline/src/scripts/pull_missing_blocks.ts b/packages/pipeline/src/scripts/pull_missing_blocks.ts
index b7bd51f08..bb5385126 100644
--- a/packages/pipeline/src/scripts/pull_missing_blocks.ts
+++ b/packages/pipeline/src/scripts/pull_missing_blocks.ts
@@ -9,7 +9,7 @@ import { Web3Source } from '../data_sources/web3';
 import { Block } from '../entities';
 import * as ormConfig from '../ormconfig';
 import { parseBlock } from '../parsers/web3';
-import { EXCHANGE_START_BLOCK, handleError, INFURA_ROOT_URL } from '../utils';
+import { handleError, INFURA_ROOT_URL } from '../utils';
 
 // Number of blocks to save at once.
 const BATCH_SAVE_SIZE = 1000;
@@ -24,10 +24,10 @@ let connection: Connection;
 (async () => {
     connection = await createConnection(ormConfig as ConnectionOptions);
     const provider = web3Factory.getRpcProvider({
-        rpcUrl: `${INFURA_ROOT_URL}/${process.env.INFURA_API_KEY}`,
+        rpcUrl: INFURA_ROOT_URL,
     });
     const web3Source = new Web3Source(provider);
-    await getAllMissingBlocks(web3Source);
+    await getAllMissingBlocksAsync(web3Source);
     process.exit(0);
 })().catch(handleError);
 
@@ -35,35 +35,44 @@ interface MissingBlocksResponse {
     block_number: string;
 }
 
-async function getAllMissingBlocks(web3Source: Web3Source): Promise<void> {
+async function getAllMissingBlocksAsync(web3Source: Web3Source): Promise<void> {
     const blocksRepository = connection.getRepository(Block);
-    let fromBlock = EXCHANGE_START_BLOCK;
     while (true) {
-        const blockNumbers = await getMissingBlockNumbers(fromBlock);
+        const blockNumbers = await getMissingBlockNumbersAsync();
         if (blockNumbers.length === 0) {
             // There are no more missing blocks. We're done.
             break;
         }
-        await getAndSaveBlocks(web3Source, blocksRepository, blockNumbers);
-        fromBlock = Math.max(...blockNumbers) + 1;
+        await getAndSaveBlocksAsync(web3Source, blocksRepository, blockNumbers);
     }
     const totalBlocks = await blocksRepository.count();
     console.log(`Done saving blocks. There are now ${totalBlocks} total blocks.`);
 }
 
-async function getMissingBlockNumbers(fromBlock: number): Promise<number[]> {
-    console.log(`Checking for missing blocks starting at ${fromBlock}...`);
+async function getMissingBlockNumbersAsync(): Promise<number[]> {
+    // Note(albrow): The easiest way to get all the blocks we need is to
+    // consider all the events tables together in a single query. If this query
+    // gets too slow, we should consider re-architecting so that we can work on
+    // getting the blocks for one type of event at a time.
     const response = (await connection.query(
-        'SELECT DISTINCT(block_number) FROM raw.exchange_fill_events WHERE block_number NOT IN (SELECT number FROM raw.blocks) AND block_number >= $1 ORDER BY block_number ASC LIMIT $2',
-        [fromBlock, MAX_BLOCKS_PER_QUERY],
+        `WITH all_events AS (
+            SELECT block_number FROM raw.exchange_fill_events
+                UNION SELECT block_number FROM raw.exchange_cancel_events
+                UNION SELECT block_number FROM raw.exchange_cancel_up_to_events
+                UNION SELECT block_number FROM raw.erc20_approval_events
+        )
+        SELECT DISTINCT(block_number) FROM all_events
+            WHERE block_number NOT IN (SELECT number FROM raw.blocks)
+            ORDER BY block_number ASC LIMIT $1`,
+        [MAX_BLOCKS_PER_QUERY],
     )) as MissingBlocksResponse[];
     const blockNumberStrings = R.pluck('block_number', response);
     const blockNumbers = R.map(parseInt, blockNumberStrings);
-    console.log(`Found ${blockNumbers.length} missing blocks in the given range.`);
+    console.log(`Found ${blockNumbers.length} missing blocks.`);
     return blockNumbers;
 }
 
-async function getAndSaveBlocks(
+async function getAndSaveBlocksAsync(
     web3Source: Web3Source,
     blocksRepository: Repository<Block>,
     blockNumbers: number[],