Split index.ts into multiple scripts in scripts/ and detect last known block when pulling events

1 files changed, 60 insertions, 0 deletions

diff --git a/packages/pipeline/src/scripts/pull_missing_events.ts b/packages/pipeline/src/scripts/pull_missing_events.ts
new file mode 100644
index 000000000..1f71722a3
--- /dev/null
+++ b/packages/pipeline/src/scripts/pull_missing_events.ts
@@ -0,0 +1,60 @@
+import { web3Factory } from '@0x/dev-utils';
+import { Web3ProviderEngine } from '@0x/subproviders';
+import R = require('ramda');
+import 'reflect-metadata';
+import { Connection, createConnection, Repository } from 'typeorm';
+
+import { ExchangeEventsSource } from '../data_sources/contract-wrappers/exchange_events';
+import { ExchangeFillEvent } from '../entities/ExchangeFillEvent';
+import { deployConfig } from '../ormconfig';
+import { parseExchangeEvents } from '../parsers/events';
+
+const EXCHANGE_START_BLOCK = 6271590; // Block number when the Exchange contract was deployed to mainnet.
+const START_BLOCK_OFFSET = 1000; // Number of blocks before the last known block to consider when updating fill events.
+const BATCH_SAVE_SIZE = 1000; // Number of events to save at once.
+
+let connection: Connection;
+
+(async () => {
+    connection = await createConnection(deployConfig);
+    const provider = web3Factory.getRpcProvider({
+        rpcUrl: 'https://mainnet.infura.io',
+    });
+    await getExchangeEventsAsync(provider);
+    process.exit(0);
+})();
+
+async function getExchangeEventsAsync(provider: Web3ProviderEngine): Promise<void> {
+    console.log('Checking existing event logs...');
+    const eventsRepository = connection.getRepository(ExchangeFillEvent);
+    const startBlock = await getStartBlockAsync(eventsRepository);
+    console.log(`Getting event logs starting at ${startBlock}...`);
+    const exchangeEvents = new ExchangeEventsSource(provider, 1);
+    const eventLogs = await exchangeEvents.getFillEventsAsync(startBlock);
+    console.log('Parsing events...');
+    const events = parseExchangeEvents(eventLogs);
+    console.log(`Retrieved and parsed ${events.length} total events.`);
+    console.log('Saving events...');
+    // Split the events into batches of size BATCH_SAVE_SIZE and save each batch
+    // in a single request. This reduces round-trip latency to the DB. We need
+    // to batch this way because saving an extremely large number of events in a
+    // single request causes problems.
+    for (const eventsBatch of R.splitEvery(BATCH_SAVE_SIZE, events)) {
+        await eventsRepository.save(eventsBatch);
+    }
+    const totalEvents = await eventsRepository.count();
+    console.log(`Done saving events. There are now ${totalEvents} total events.`);
+}
+
+async function getStartBlockAsync(eventsRepository: Repository<ExchangeFillEvent>): Promise<number> {
+    const fillEventCount = await eventsRepository.count();
+    if (fillEventCount === 0) {
+        console.log('No existing fill events found.');
+        return EXCHANGE_START_BLOCK;
+    }
+    const queryResult = await connection.query(
+        'SELECT "blockNumber" FROM exchange_fill_event ORDER BY "blockNumber" DESC LIMIT 1',
+    );
+    const lastKnownBlock = queryResult[0].blockNumber;
+    return lastKnownBlock - START_BLOCK_OFFSET;
+}