aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAlex Browne <stephenalexbrowne@gmail.com>2018-11-15 08:20:07 +0800
committerFred Carlsen <fred@sjelfull.no>2018-12-06 19:04:25 +0800
commitf07dc5ae6d4cc928e4d00708e3f90f7cf9b4836d (patch)
treead83d2ebbc93b058bc6c9c99099dac2f06d3814f
parent8cb6c2b51ba7c63951f4b87fac423a19861bf47f (diff)
downloaddexon-sol-tools-f07dc5ae6d4cc928e4d00708e3f90f7cf9b4836d.tar
dexon-sol-tools-f07dc5ae6d4cc928e4d00708e3f90f7cf9b4836d.tar.gz
dexon-sol-tools-f07dc5ae6d4cc928e4d00708e3f90f7cf9b4836d.tar.bz2
dexon-sol-tools-f07dc5ae6d4cc928e4d00708e3f90f7cf9b4836d.tar.lz
dexon-sol-tools-f07dc5ae6d4cc928e4d00708e3f90f7cf9b4836d.tar.xz
dexon-sol-tools-f07dc5ae6d4cc928e4d00708e3f90f7cf9b4836d.tar.zst
dexon-sol-tools-f07dc5ae6d4cc928e4d00708e3f90f7cf9b4836d.zip
Use built-in chunk feature of TypeORM save method
-rw-r--r--packages/pipeline/src/scripts/pull_missing_events.ts9
1 files changed, 2 insertions, 7 deletions
diff --git a/packages/pipeline/src/scripts/pull_missing_events.ts b/packages/pipeline/src/scripts/pull_missing_events.ts
index 1693bb59a..b1b8665dd 100644
--- a/packages/pipeline/src/scripts/pull_missing_events.ts
+++ b/packages/pipeline/src/scripts/pull_missing_events.ts
@@ -29,6 +29,7 @@ let connection: Connection;
async function getExchangeEventsAsync(provider: Web3ProviderEngine): Promise<void> {
console.log('Checking existing event logs...');
const eventsRepository = connection.getRepository(ExchangeFillEvent);
+ const manager = connection.createEntityManager();
const startBlock = await getStartBlockAsync(eventsRepository);
console.log(`Getting event logs starting at ${startBlock}...`);
const exchangeEvents = new ExchangeEventsSource(provider, 1);
@@ -37,13 +38,7 @@ async function getExchangeEventsAsync(provider: Web3ProviderEngine): Promise<voi
const events = parseExchangeEvents(eventLogs);
console.log(`Retrieved and parsed ${events.length} total events.`);
console.log('Saving events...');
- // Split the events into batches of size BATCH_SAVE_SIZE and save each batch
- // in a single request. This reduces round-trip latency to the DB. We need
- // to batch this way because saving an extremely large number of events in a
- // single request causes problems.
- for (const eventsBatch of R.splitEvery(BATCH_SAVE_SIZE, events)) {
- await eventsRepository.save(eventsBatch);
- }
+ await eventsRepository.save(events, { chunk: BATCH_SAVE_SIZE });
const totalEvents = await eventsRepository.count();
console.log(`Done saving events. There are now ${totalEvents} total events.`);
}