aboutsummaryrefslogtreecommitdiffstats
path: root/packages/pipeline
diff options
context:
space:
mode:
authorAlex Browne <stephenalexbrowne@gmail.com>2018-11-15 08:20:07 +0800
committerAlex Browne <stephenalexbrowne@gmail.com>2018-12-05 06:24:48 +0800
commitb0a2c10e11fa41e2800d4fe67d8008b5828128a3 (patch)
treeda61c8682a5dba1e5eed7566aacfa89f2e0218f5 /packages/pipeline
parent303bbc42f4322448998f3fde202574335d1190e6 (diff)
downloaddexon-sol-tools-b0a2c10e11fa41e2800d4fe67d8008b5828128a3.tar
dexon-sol-tools-b0a2c10e11fa41e2800d4fe67d8008b5828128a3.tar.gz
dexon-sol-tools-b0a2c10e11fa41e2800d4fe67d8008b5828128a3.tar.bz2
dexon-sol-tools-b0a2c10e11fa41e2800d4fe67d8008b5828128a3.tar.lz
dexon-sol-tools-b0a2c10e11fa41e2800d4fe67d8008b5828128a3.tar.xz
dexon-sol-tools-b0a2c10e11fa41e2800d4fe67d8008b5828128a3.tar.zst
dexon-sol-tools-b0a2c10e11fa41e2800d4fe67d8008b5828128a3.zip
Use built-in chunk feature of TypeORM save method
Diffstat (limited to 'packages/pipeline')
-rw-r--r--packages/pipeline/src/scripts/pull_missing_events.ts9
1 files changed, 2 insertions, 7 deletions
diff --git a/packages/pipeline/src/scripts/pull_missing_events.ts b/packages/pipeline/src/scripts/pull_missing_events.ts
index 1693bb59a..b1b8665dd 100644
--- a/packages/pipeline/src/scripts/pull_missing_events.ts
+++ b/packages/pipeline/src/scripts/pull_missing_events.ts
@@ -29,6 +29,7 @@ let connection: Connection;
async function getExchangeEventsAsync(provider: Web3ProviderEngine): Promise<void> {
console.log('Checking existing event logs...');
const eventsRepository = connection.getRepository(ExchangeFillEvent);
+ const manager = connection.createEntityManager();
const startBlock = await getStartBlockAsync(eventsRepository);
console.log(`Getting event logs starting at ${startBlock}...`);
const exchangeEvents = new ExchangeEventsSource(provider, 1);
@@ -37,13 +38,7 @@ async function getExchangeEventsAsync(provider: Web3ProviderEngine): Promise<voi
const events = parseExchangeEvents(eventLogs);
console.log(`Retrieved and parsed ${events.length} total events.`);
console.log('Saving events...');
- // Split the events into batches of size BATCH_SAVE_SIZE and save each batch
- // in a single request. This reduces round-trip latency to the DB. We need
- // to batch this way because saving an extremely large number of events in a
- // single request causes problems.
- for (const eventsBatch of R.splitEvery(BATCH_SAVE_SIZE, events)) {
- await eventsRepository.save(eventsBatch);
- }
+ await eventsRepository.save(events, { chunk: BATCH_SAVE_SIZE });
const totalEvents = await eventsRepository.count();
console.log(`Done saving events. There are now ${totalEvents} total events.`);
}