1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
|
import { logUtils } from '@0x/utils';
import * as R from 'ramda';
import { Connection, ConnectionOptions, createConnection } from 'typeorm';
import { IDEX_SOURCE, IdexSource } from '../data_sources/idex';
import { TokenOrderbookSnapshot as TokenOrder } from '../entities';
import * as ormConfig from '../ormconfig';
import { parseIdexOrders } from '../parsers/idex_orders';
import { handleError } from '../utils';
// Number of orders to save at once.
const BATCH_SAVE_SIZE = 1000;
// Number of markets to retrieve orderbooks for at once.
const MARKET_ORDERBOOK_REQUEST_BATCH_SIZE = 100;
// Delay between market orderbook requests.
const MILLISEC_MARKET_ORDERBOOK_REQUEST_DELAY = 2000;
let connection: Connection;
(async () => {
connection = await createConnection(ormConfig as ConnectionOptions);
const idexSource = new IdexSource();
logUtils.log('Getting all IDEX markets');
const markets = await idexSource.getMarketsAsync();
logUtils.log(`Got ${markets.length} markets.`);
for (const marketsChunk of R.splitEvery(MARKET_ORDERBOOK_REQUEST_BATCH_SIZE, markets)) {
await Promise.all(
marketsChunk.map(async (marketId: string) => getAndSaveMarketOrderbookAsync(idexSource, marketId)),
);
await new Promise<void>(resolve => setTimeout(resolve, MILLISEC_MARKET_ORDERBOOK_REQUEST_DELAY));
}
process.exit(0);
})().catch(handleError);
/**
* Retrieve orderbook from Idex API for a given market. Parse orders and insert
* them into our database.
* @param idexSource Data source which can query Idex API.
* @param marketId String representing market of interest, eg. 'ETH_TIC'.
*/
async function getAndSaveMarketOrderbookAsync(idexSource: IdexSource, marketId: string): Promise<void> {
logUtils.log(`${marketId}: Retrieving orderbook.`);
const orderBook = await idexSource.getMarketOrderbookAsync(marketId);
const observedTimestamp = Date.now();
if (!R.has('bids', orderBook) || !R.has('asks', orderBook)) {
logUtils.warn(`${marketId}: Orderbook faulty.`);
return;
}
logUtils.log(`${marketId}: Parsing orders.`);
const orders = parseIdexOrders(orderBook, observedTimestamp, IDEX_SOURCE);
if (orders.length > 0) {
logUtils.log(`${marketId}: Saving ${orders.length} orders.`);
const TokenOrderRepository = connection.getRepository(TokenOrder);
await TokenOrderRepository.save(orders, { chunk: Math.ceil(orders.length / BATCH_SAVE_SIZE) });
} else {
logUtils.log(`${marketId}: 0 orders to save.`);
}
}
|
