diff options
author | Alex Browne <stephenalexbrowne@gmail.com> | 2018-12-06 03:08:19 +0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-12-06 03:08:19 +0800 |
commit | b2dd5495bcf13a9ea71498b5def12c75589b0156 (patch) | |
tree | 0e0d728d540e747c32a083d604d7916a35ea95cf /packages/pipeline/src/scripts/pull_ddex_orderbook_snapshots.ts | |
parent | 72a30260d88e722a6b076134693360c573f6c70f (diff) | |
parent | e0348f9c044b4909260e4864398b4f50232da620 (diff) | |
download | dexon-sol-tools-b2dd5495bcf13a9ea71498b5def12c75589b0156.tar dexon-sol-tools-b2dd5495bcf13a9ea71498b5def12c75589b0156.tar.gz dexon-sol-tools-b2dd5495bcf13a9ea71498b5def12c75589b0156.tar.bz2 dexon-sol-tools-b2dd5495bcf13a9ea71498b5def12c75589b0156.tar.lz dexon-sol-tools-b2dd5495bcf13a9ea71498b5def12c75589b0156.tar.xz dexon-sol-tools-b2dd5495bcf13a9ea71498b5def12c75589b0156.tar.zst dexon-sol-tools-b2dd5495bcf13a9ea71498b5def12c75589b0156.zip |
Merge pull request #1377 from 0xProject/feature/pipeline-cleanup-mega-rebase
Merge all pipeline code into development
Diffstat (limited to 'packages/pipeline/src/scripts/pull_ddex_orderbook_snapshots.ts')
-rw-r--r-- | packages/pipeline/src/scripts/pull_ddex_orderbook_snapshots.ts | 55 |
1 files changed, 55 insertions, 0 deletions
diff --git a/packages/pipeline/src/scripts/pull_ddex_orderbook_snapshots.ts b/packages/pipeline/src/scripts/pull_ddex_orderbook_snapshots.ts new file mode 100644 index 000000000..7868e9c5a --- /dev/null +++ b/packages/pipeline/src/scripts/pull_ddex_orderbook_snapshots.ts @@ -0,0 +1,55 @@ +import { logUtils } from '@0x/utils'; +import * as R from 'ramda'; +import { Connection, ConnectionOptions, createConnection } from 'typeorm'; + +import { DDEX_SOURCE, DdexMarket, DdexSource } from '../data_sources/ddex'; +import { TokenOrderbookSnapshot as TokenOrder } from '../entities'; +import * as ormConfig from '../ormconfig'; +import { parseDdexOrders } from '../parsers/ddex_orders'; +import { handleError } from '../utils'; + +// Number of orders to save at once. +const BATCH_SAVE_SIZE = 1000; + +// Number of markets to retrieve orderbooks for at once. +const MARKET_ORDERBOOK_REQUEST_BATCH_SIZE = 50; + +// Delay between market orderbook requests. +const MILLISEC_MARKET_ORDERBOOK_REQUEST_DELAY = 5000; + +let connection: Connection; + +(async () => { + connection = await createConnection(ormConfig as ConnectionOptions); + const ddexSource = new DdexSource(); + const markets = await ddexSource.getActiveMarketsAsync(); + for (const marketsChunk of R.splitEvery(MARKET_ORDERBOOK_REQUEST_BATCH_SIZE, markets)) { + await Promise.all( + marketsChunk.map(async (market: DdexMarket) => getAndSaveMarketOrderbook(ddexSource, market)), + ); + await new Promise<void>(resolve => setTimeout(resolve, MILLISEC_MARKET_ORDERBOOK_REQUEST_DELAY)); + } + process.exit(0); +})().catch(handleError); + +/** + * Retrieve orderbook from Ddex API for a given market. Parse orders and insert + * them into our database. + * @param ddexSource Data source which can query Ddex API. + * @param market Object from Ddex API containing market data. + */ +async function getAndSaveMarketOrderbook(ddexSource: DdexSource, market: DdexMarket): Promise<void> { + const orderBook = await ddexSource.getMarketOrderbookAsync(market.id); + const observedTimestamp = Date.now(); + + logUtils.log(`${market.id}: Parsing orders.`); + const orders = parseDdexOrders(orderBook, market, observedTimestamp, DDEX_SOURCE); + + if (orders.length > 0) { + logUtils.log(`${market.id}: Saving ${orders.length} orders.`); + const TokenOrderRepository = connection.getRepository(TokenOrder); + await TokenOrderRepository.save(orders, { chunk: Math.ceil(orders.length / BATCH_SAVE_SIZE) }); + } else { + logUtils.log(`${market.id}: 0 orders to save.`); + } +} |