diff options
| author | Alex Browne <stephenalexbrowne@gmail.com> | 2018-09-18 02:27:38 +0800 |
|---|---|---|
| committer | Alex Browne <stephenalexbrowne@gmail.com> | 2018-12-05 06:21:42 +0800 |
| commit | 57e7119c0d4f1ab7dd1d4c0118e72dc1706e2151 (patch) | |
| tree | 68596be050d6f84e3b7e6f51f91d09a542ac3129 /packages/pipeline/src/run_jobs.ts | |
| parent | c43ba6b3c7082544f8aa8f1c297eb271ca079ea4 (diff) | |
| download | dexon-sol-tools-57e7119c0d4f1ab7dd1d4c0118e72dc1706e2151.tar dexon-sol-tools-57e7119c0d4f1ab7dd1d4c0118e72dc1706e2151.tar.gz dexon-sol-tools-57e7119c0d4f1ab7dd1d4c0118e72dc1706e2151.tar.bz2 dexon-sol-tools-57e7119c0d4f1ab7dd1d4c0118e72dc1706e2151.tar.lz dexon-sol-tools-57e7119c0d4f1ab7dd1d4c0118e72dc1706e2151.tar.xz dexon-sol-tools-57e7119c0d4f1ab7dd1d4c0118e72dc1706e2151.tar.zst dexon-sol-tools-57e7119c0d4f1ab7dd1d4c0118e72dc1706e2151.zip | |
Rebase pipeline branch off development
Diffstat (limited to 'packages/pipeline/src/run_jobs.ts')
| -rw-r--r-- | packages/pipeline/src/run_jobs.ts | 87 |
1 files changed, 87 insertions, 0 deletions
diff --git a/packages/pipeline/src/run_jobs.ts b/packages/pipeline/src/run_jobs.ts new file mode 100644 index 000000000..4d82d4e2d --- /dev/null +++ b/packages/pipeline/src/run_jobs.ts @@ -0,0 +1,87 @@ +import { exec } from 'child_process'; + +import { postgresClient } from './postgres.js'; +import { dataFetchingQueries } from './scripts/query_data.js'; +import { web3, zrx } from './zrx.js'; +const CUR_BLOCK_OFFSET = 20; +postgresClient.query(dataFetchingQueries.get_max_block, []).then((data: any) => { + const maxBlockNumber = data.rows[0].max; + const safeCurBlockNumber = web3.eth.blockNumber - CUR_BLOCK_OFFSET; + console.log('Scraping ' + maxBlockNumber + ' to ' + safeCurBlockNumber); + exec( + 'node ./lib/scripts/scrape_data --type events --from ' + maxBlockNumber + ' --to ' + safeCurBlockNumber, + (error, stdout, stderr) => { + if (error) { + console.log(error); + return; + } + console.log('Scraped events'); + console.log('Scraping blocks'); + exec( + 'node ./lib/scripts/scrape_data --type blocks --from ' + maxBlockNumber + ' --to ' + safeCurBlockNumber, + (error, stdout, stderr) => { + if (error) { + console.log(error); + return; + } + console.log('Scraped blocks'); + console.log('Scraping transactions'); + exec( + 'node ./lib/scripts/scrape_data --type transactions --from ' + + maxBlockNumber + + ' --to ' + + safeCurBlockNumber, + (error, stdout, stderr) => { + if (error) { + console.log(error); + return; + } + console.log('Scraped transactions'); + console.log('Joining events_staging'); + exec( + 'node ./lib/scripts/join_tables --name events_staging --from ' + + maxBlockNumber + + ' --to ' + + safeCurBlockNumber, + (error, stdout, stderr) => { + if (error) { + console.log(error); + return; + } + console.log('Joined events_staging'); + console.log('Joining events'); + exec( + 'node ./lib/scripts/join_tables --name events --from ' + + maxBlockNumber + + ' --to ' + + safeCurBlockNumber, + (error, stdout, stderr) => { + if (error) { + console.log(error); + return; + } + console.log('Joined events'); + console.log('Joining events_full'); + exec( + 'node ./lib/scripts/join_tables --name events_full --from ' + + maxBlockNumber + + ' --to ' + + safeCurBlockNumber, + (error, stdout, stderr) => { + if (error) { + console.log(error); + return; + } + }, + ); + }, + ); + }, + ); + }, + ); + }, + ); + }, + ); +}); |