diff options
Diffstat (limited to 'packages/pipeline/src/scripts/pull_missing_blocks.ts')
-rw-r--r-- | packages/pipeline/src/scripts/pull_missing_blocks.ts | 29 |
1 files changed, 21 insertions, 8 deletions
diff --git a/packages/pipeline/src/scripts/pull_missing_blocks.ts b/packages/pipeline/src/scripts/pull_missing_blocks.ts index b7bd51f08..a5203824c 100644 --- a/packages/pipeline/src/scripts/pull_missing_blocks.ts +++ b/packages/pipeline/src/scripts/pull_missing_blocks.ts @@ -24,10 +24,10 @@ let connection: Connection; (async () => { connection = await createConnection(ormConfig as ConnectionOptions); const provider = web3Factory.getRpcProvider({ - rpcUrl: `${INFURA_ROOT_URL}/${process.env.INFURA_API_KEY}`, + rpcUrl: INFURA_ROOT_URL, }); const web3Source = new Web3Source(provider); - await getAllMissingBlocks(web3Source); + await getAllMissingBlocksAsync(web3Source); process.exit(0); })().catch(handleError); @@ -35,26 +35,39 @@ interface MissingBlocksResponse { block_number: string; } -async function getAllMissingBlocks(web3Source: Web3Source): Promise<void> { +async function getAllMissingBlocksAsync(web3Source: Web3Source): Promise<void> { const blocksRepository = connection.getRepository(Block); let fromBlock = EXCHANGE_START_BLOCK; while (true) { - const blockNumbers = await getMissingBlockNumbers(fromBlock); + const blockNumbers = await getMissingBlockNumbersAsync(fromBlock); if (blockNumbers.length === 0) { // There are no more missing blocks. We're done. break; } - await getAndSaveBlocks(web3Source, blocksRepository, blockNumbers); + await getAndSaveBlocksAsync(web3Source, blocksRepository, blockNumbers); fromBlock = Math.max(...blockNumbers) + 1; } const totalBlocks = await blocksRepository.count(); console.log(`Done saving blocks. There are now ${totalBlocks} total blocks.`); } -async function getMissingBlockNumbers(fromBlock: number): Promise<number[]> { +async function getMissingBlockNumbersAsync(fromBlock: number): Promise<number[]> { console.log(`Checking for missing blocks starting at ${fromBlock}...`); + // Note(albrow): The easiest way to get all the blocks we need is to + // consider all the events tables together in a single query. If this query + // gets too slow, we should consider re-architecting so that we can work on + // getting the blocks for one type of event at a time. const response = (await connection.query( - 'SELECT DISTINCT(block_number) FROM raw.exchange_fill_events WHERE block_number NOT IN (SELECT number FROM raw.blocks) AND block_number >= $1 ORDER BY block_number ASC LIMIT $2', + `WITH all_events AS ( + SELECT block_number FROM raw.exchange_fill_events + UNION SELECT block_number FROM raw.exchange_cancel_events + UNION SELECT block_number FROM raw.exchange_cancel_up_to_events + UNION SELECT block_number FROM raw.erc20_approval_events + ) + SELECT DISTINCT(block_number) FROM all_events + WHERE block_number NOT IN (SELECT number FROM raw.blocks) + AND block_number >= $1 + ORDER BY block_number ASC LIMIT $2`, [fromBlock, MAX_BLOCKS_PER_QUERY], )) as MissingBlocksResponse[]; const blockNumberStrings = R.pluck('block_number', response); @@ -63,7 +76,7 @@ async function getMissingBlockNumbers(fromBlock: number): Promise<number[]> { return blockNumbers; } -async function getAndSaveBlocks( +async function getAndSaveBlocksAsync( web3Source: Web3Source, blocksRepository: Repository<Block>, blockNumbers: number[], |