diff options
author | Fabio Berger <me@fabioberger.com> | 2018-12-19 00:59:15 +0800 |
---|---|---|
committer | Fabio Berger <me@fabioberger.com> | 2018-12-19 00:59:15 +0800 |
commit | 622b9f662e74d571da745047ede097c7a392d09e (patch) | |
tree | 68a9517882b04a3d428f6996e0790ceb82e89be4 /packages/pipeline/src/scripts/pull_idex_orderbook_snapshots.ts | |
parent | e295eeb8938468b1527d5d81f212766cef40bc81 (diff) | |
parent | 67df5a433d68a2af1a3a03a8bf431629a534dc97 (diff) | |
download | dexon-sol-tools-622b9f662e74d571da745047ede097c7a392d09e.tar dexon-sol-tools-622b9f662e74d571da745047ede097c7a392d09e.tar.gz dexon-sol-tools-622b9f662e74d571da745047ede097c7a392d09e.tar.bz2 dexon-sol-tools-622b9f662e74d571da745047ede097c7a392d09e.tar.lz dexon-sol-tools-622b9f662e74d571da745047ede097c7a392d09e.tar.xz dexon-sol-tools-622b9f662e74d571da745047ede097c7a392d09e.tar.zst dexon-sol-tools-622b9f662e74d571da745047ede097c7a392d09e.zip |
Merge branch 'development' into features/orderwatcher_ws
* development: (107 commits)
Fix OrderWatcher title to fix sidebar top
Fix version picker so it doesn't overflow onto two lines
Fix bug in pull_missing_blocks with incorrect start block (#1438)
Pull approval events for ZRX and DAI (#1430)
fix semicolon and apply prettier
Fix dex order quote/base asset assigning (#1432)
Apply prettier
Publish
Updated CHANGELOGS
Rename contracts CHANGELOGs to DEPLOYs
Move Forwarder CHANGELOG entries to extensions CHANGELOG
Make contracts packages not private
Publish
Updated CHANGELOGS
Show @ price in light grey
Updated CHANGELOGS
typeof -> isString
add special case to scrape OHLCV for eth/usd (#1428)
run linter
simplify scaling input logic
...
Diffstat (limited to 'packages/pipeline/src/scripts/pull_idex_orderbook_snapshots.ts')
-rw-r--r-- | packages/pipeline/src/scripts/pull_idex_orderbook_snapshots.ts | 63 |
1 files changed, 63 insertions, 0 deletions
diff --git a/packages/pipeline/src/scripts/pull_idex_orderbook_snapshots.ts b/packages/pipeline/src/scripts/pull_idex_orderbook_snapshots.ts new file mode 100644 index 000000000..490b17766 --- /dev/null +++ b/packages/pipeline/src/scripts/pull_idex_orderbook_snapshots.ts @@ -0,0 +1,63 @@ +import { logUtils } from '@0x/utils'; +import * as R from 'ramda'; +import { Connection, ConnectionOptions, createConnection } from 'typeorm'; + +import { IDEX_SOURCE, IdexSource } from '../data_sources/idex'; +import { TokenOrderbookSnapshot as TokenOrder } from '../entities'; +import * as ormConfig from '../ormconfig'; +import { parseIdexOrders } from '../parsers/idex_orders'; +import { handleError } from '../utils'; + +// Number of orders to save at once. +const BATCH_SAVE_SIZE = 1000; + +// Number of markets to retrieve orderbooks for at once. +const MARKET_ORDERBOOK_REQUEST_BATCH_SIZE = 100; + +// Delay between market orderbook requests. +const MILLISEC_MARKET_ORDERBOOK_REQUEST_DELAY = 2000; + +let connection: Connection; + +(async () => { + connection = await createConnection(ormConfig as ConnectionOptions); + const idexSource = new IdexSource(); + logUtils.log('Getting all IDEX markets'); + const markets = await idexSource.getMarketsAsync(); + logUtils.log(`Got ${markets.length} markets.`); + for (const marketsChunk of R.splitEvery(MARKET_ORDERBOOK_REQUEST_BATCH_SIZE, markets)) { + await Promise.all( + marketsChunk.map(async (marketId: string) => getAndSaveMarketOrderbookAsync(idexSource, marketId)), + ); + await new Promise<void>(resolve => setTimeout(resolve, MILLISEC_MARKET_ORDERBOOK_REQUEST_DELAY)); + } + process.exit(0); +})().catch(handleError); + +/** + * Retrieve orderbook from Idex API for a given market. Parse orders and insert + * them into our database. + * @param idexSource Data source which can query Idex API. + * @param marketId String representing market of interest, eg. 'ETH_TIC'. + */ +async function getAndSaveMarketOrderbookAsync(idexSource: IdexSource, marketId: string): Promise<void> { + logUtils.log(`${marketId}: Retrieving orderbook.`); + const orderBook = await idexSource.getMarketOrderbookAsync(marketId); + const observedTimestamp = Date.now(); + + if (!R.has('bids', orderBook) || !R.has('asks', orderBook)) { + logUtils.warn(`${marketId}: Orderbook faulty.`); + return; + } + + logUtils.log(`${marketId}: Parsing orders.`); + const orders = parseIdexOrders(orderBook, observedTimestamp, IDEX_SOURCE); + + if (orders.length > 0) { + logUtils.log(`${marketId}: Saving ${orders.length} orders.`); + const TokenOrderRepository = connection.getRepository(TokenOrder); + await TokenOrderRepository.save(orders, { chunk: Math.ceil(orders.length / BATCH_SAVE_SIZE) }); + } else { + logUtils.log(`${marketId}: 0 orders to save.`); + } +} |