aboutsummaryrefslogtreecommitdiffstats
path: root/packages/pipeline/src/scripts
diff options
context:
space:
mode:
authorAlex Browne <stephenalexbrowne@gmail.com>2018-11-17 04:55:54 +0800
committerAlex Browne <stephenalexbrowne@gmail.com>2018-12-05 06:24:48 +0800
commit24fd2d9730d58a58929f401674175ad8a5a7fbc1 (patch)
tree17994e0f4171a796294bfa69dd126ccffef7fa90 /packages/pipeline/src/scripts
parent0397ff8b2206ab4e9308a524d635ec37ec9dda99 (diff)
downloaddexon-sol-tools-24fd2d9730d58a58929f401674175ad8a5a7fbc1.tar
dexon-sol-tools-24fd2d9730d58a58929f401674175ad8a5a7fbc1.tar.gz
dexon-sol-tools-24fd2d9730d58a58929f401674175ad8a5a7fbc1.tar.bz2
dexon-sol-tools-24fd2d9730d58a58929f401674175ad8a5a7fbc1.tar.lz
dexon-sol-tools-24fd2d9730d58a58929f401674175ad8a5a7fbc1.tar.xz
dexon-sol-tools-24fd2d9730d58a58929f401674175ad8a5a7fbc1.tar.zst
dexon-sol-tools-24fd2d9730d58a58929f401674175ad8a5a7fbc1.zip
Add support for pulling Cancel and CancelUpTo events
Diffstat (limited to 'packages/pipeline/src/scripts')
-rw-r--r--packages/pipeline/src/scripts/pull_missing_events.ts123
1 files changed, 81 insertions, 42 deletions
diff --git a/packages/pipeline/src/scripts/pull_missing_events.ts b/packages/pipeline/src/scripts/pull_missing_events.ts
index bceed299c..b2a99e3c0 100644
--- a/packages/pipeline/src/scripts/pull_missing_events.ts
+++ b/packages/pipeline/src/scripts/pull_missing_events.ts
@@ -1,14 +1,13 @@
// tslint:disable:no-console
import { web3Factory } from '@0x/dev-utils';
-import { Web3ProviderEngine } from '@0x/subproviders';
import R = require('ramda');
import 'reflect-metadata';
import { Connection, ConnectionOptions, createConnection, Repository } from 'typeorm';
import { ExchangeEventsSource } from '../data_sources/contract-wrappers/exchange_events';
-import { ExchangeFillEvent } from '../entities';
+import { ExchangeCancelEvent, ExchangeCancelUpToEvent, ExchangeEvent, ExchangeFillEvent } from '../entities';
import * as ormConfig from '../ormconfig';
-import { ExchangeEventEntity, parseExchangeEvents } from '../parsers/events';
+import { parseExchangeCancelEvents, parseExchangeCancelUpToEvents, parseExchangeFillEvents } from '../parsers/events';
import { handleError } from '../utils';
const EXCHANGE_START_BLOCK = 6271590; // Block number when the Exchange contract was deployed to mainnet.
@@ -22,40 +21,88 @@ let connection: Connection;
const provider = web3Factory.getRpcProvider({
rpcUrl: 'https://mainnet.infura.io',
});
- await getExchangeEventsAsync(provider);
+ const eventsSource = new ExchangeEventsSource(provider, 1);
+ await getFillEventsAsync(eventsSource);
+ await getCancelEventsAsync(eventsSource);
+ await getCancelUpToEventsAsync(eventsSource);
process.exit(0);
})().catch(handleError);
-async function getExchangeEventsAsync(provider: Web3ProviderEngine): Promise<void> {
- console.log('Checking existing event logs...');
- const eventsRepository = connection.getRepository(ExchangeFillEvent);
- const manager = connection.createEntityManager();
- const startBlock = await getStartBlockAsync(eventsRepository);
- console.log(`Getting event logs starting at ${startBlock}...`);
- const exchangeEvents = new ExchangeEventsSource(provider, 1);
- const eventLogs = await exchangeEvents.getFillEventsAsync(startBlock);
- console.log('Parsing events...');
- const events = parseExchangeEvents(eventLogs);
- console.log(`Retrieved and parsed ${events.length} total events.`);
- console.log('Saving events...');
- if (startBlock === EXCHANGE_START_BLOCK) {
+async function getFillEventsAsync(eventsSource: ExchangeEventsSource): Promise<void> {
+ console.log('Checking existing fill events...');
+ const repository = connection.getRepository(ExchangeFillEvent);
+ const startBlock = await getStartBlockAsync(repository);
+ console.log(`Getting fill events starting at ${startBlock}...`);
+ const eventLogs = await eventsSource.getFillEventsAsync(startBlock);
+ console.log('Parsing fill events...');
+ const events = parseExchangeFillEvents(eventLogs);
+ console.log(`Retrieved and parsed ${events.length} total fill events.`);
+ await saveEventsAsync(startBlock === EXCHANGE_START_BLOCK, repository, events);
+}
+
+async function getCancelEventsAsync(eventsSource: ExchangeEventsSource): Promise<void> {
+ console.log('Checking existing cancel events...');
+ const repository = connection.getRepository(ExchangeCancelEvent);
+ const startBlock = await getStartBlockAsync(repository);
+ console.log(`Getting cancel events starting at ${startBlock}...`);
+ const eventLogs = await eventsSource.getCancelEventsAsync(startBlock);
+ console.log('Parsing cancel events...');
+ const events = parseExchangeCancelEvents(eventLogs);
+ console.log(`Retrieved and parsed ${events.length} total cancel events.`);
+ await saveEventsAsync(startBlock === EXCHANGE_START_BLOCK, repository, events);
+}
+
+async function getCancelUpToEventsAsync(eventsSource: ExchangeEventsSource): Promise<void> {
+ console.log('Checking existing CancelUpTo events...');
+ const repository = connection.getRepository(ExchangeCancelUpToEvent);
+ const startBlock = await getStartBlockAsync(repository);
+ console.log(`Getting CancelUpTo events starting at ${startBlock}...`);
+ const eventLogs = await eventsSource.getCancelUpToEventsAsync(startBlock);
+ console.log('Parsing CancelUpTo events...');
+ const events = parseExchangeCancelUpToEvents(eventLogs);
+ console.log(`Retrieved and parsed ${events.length} total CancelUpTo events.`);
+ await saveEventsAsync(startBlock === EXCHANGE_START_BLOCK, repository, events);
+}
+
+async function getStartBlockAsync<T extends ExchangeEvent>(repository: Repository<T>): Promise<number> {
+ const fillEventCount = await repository.count();
+ if (fillEventCount === 0) {
+ console.log(`No existing ${repository.metadata.name}s found.`);
+ return EXCHANGE_START_BLOCK;
+ }
+ const queryResult = await connection.query(
+ // TODO(albrow): Would prefer to use a prepared statement here to reduce
+ // surface area for SQL injections, but it doesn't appear to be working.
+ `SELECT block_number FROM raw.${repository.metadata.tableName} ORDER BY block_number DESC LIMIT 1`,
+ );
+ const lastKnownBlock = queryResult[0].block_number;
+ return lastKnownBlock - START_BLOCK_OFFSET;
+}
+
+async function saveEventsAsync<T extends ExchangeEvent>(
+ isInitialPull: boolean,
+ repository: Repository<T>,
+ events: T[],
+): Promise<void> {
+ console.log(`Saving ${repository.metadata.name}s...`);
+ if (isInitialPull) {
// Split data into numChunks pieces of maximum size BATCH_SAVE_SIZE
// each.
for (const eventsBatch of R.splitEvery(BATCH_SAVE_SIZE, events)) {
- await eventsRepository.insert(eventsBatch);
+ await repository.insert(eventsBatch);
}
} else {
// If we possibly have some overlap where we need to update some
// existing events, we need to use our workaround/fallback.
- await saveIndividuallyWithFallbackAsync(eventsRepository, events);
+ await saveIndividuallyWithFallbackAsync(repository, events);
}
- const totalEvents = await eventsRepository.count();
- console.log(`Done saving events. There are now ${totalEvents} total events.`);
+ const totalEvents = await repository.count();
+ console.log(`Done saving events. There are now ${totalEvents} total ${repository.metadata.name}s.`);
}
-async function saveIndividuallyWithFallbackAsync(
- eventsRepository: Repository<ExchangeFillEvent>,
- events: ExchangeEventEntity[],
+async function saveIndividuallyWithFallbackAsync<T extends ExchangeEvent>(
+ repository: Repository<T>,
+ events: T[],
): Promise<void> {
// Note(albrow): This is a temporary hack because `save` is not working as
// documented and is causing a foreign key constraint violation. Hopefully
@@ -63,32 +110,24 @@ async function saveIndividuallyWithFallbackAsync(
// on one event at a time and is therefore much slower.
for (const event of events) {
try {
- // First try and insert.
- await eventsRepository.insert(event);
+ // First try an insert.
+ await repository.insert(event);
} catch {
// If it fails, assume it was a foreign key constraint error and try
// doing an update instead.
- await eventsRepository.update(
+ // Note(albrow): Unfortunately the `as any` hack here seems
+ // required. I can't figure out how to convince the type-checker
+ // that the criteria and the entity itself are the correct type for
+ // the given repository. If we can remove the `save` hack then this
+ // will probably no longer be necessary.
+ await repository.update(
{
contractAddress: event.contractAddress,
blockNumber: event.blockNumber,
logIndex: event.logIndex,
- },
- event,
+ } as any,
+ event as any,
);
}
}
}
-
-async function getStartBlockAsync(eventsRepository: Repository<ExchangeFillEvent>): Promise<number> {
- const fillEventCount = await eventsRepository.count();
- if (fillEventCount === 0) {
- console.log('No existing fill events found.');
- return EXCHANGE_START_BLOCK;
- }
- const queryResult = await connection.query(
- 'SELECT block_number FROM raw.exchange_fill_events ORDER BY block_number DESC LIMIT 1',
- );
- const lastKnownBlock = queryResult[0].block_number;
- return lastKnownBlock - START_BLOCK_OFFSET;
-}