Token_orderbook_snapshots for Ddex and Paradex(#1354)

* Implements the TokenOrderbookSnapshot Table

* Scripts, Data Sources and Entities to pull Ddex and Paradex API data.

2 files changed, 142 insertions, 0 deletions

diff --git a/packages/pipeline/src/scripts/pull_ddex_orderbook_snapshots.ts b/packages/pipeline/src/scripts/pull_ddex_orderbook_snapshots.ts
new file mode 100644
index 000000000..b02468e9b
--- /dev/null
+++ b/packages/pipeline/src/scripts/pull_ddex_orderbook_snapshots.ts
@@ -0,0 +1,55 @@
+import { logUtils } from '@0x/utils';
+import * as R from 'ramda';
+import { Connection, ConnectionOptions, createConnection } from 'typeorm';
+
+import { DDEX_SOURCE, DdexMarket, DdexSource } from '../data_sources/ddex';
+import { TokenOrderbookSnapshot as TokenOrder } from '../entities';
+import * as ormConfig from '../ormconfig';
+import { parseDdexOrders } from '../parsers/ddex_orders';
+import { handleError } from '../utils';
+
+// Number of orders to save at once.
+const BATCH_SAVE_SIZE = 1000;
+
+// Number of markets to retrieve orderbooks for at once.
+const MARKET_ORDERBOOK_REQUEST_BATCH_SIZE = 50;
+
+// Delay between market orderbook requests.
+const MILLISEC_MARKET_ORDERBOOK_REQUEST_DELAY = 5000;
+
+let connection: Connection;
+
+(async () => {
+    connection = await createConnection(ormConfig as ConnectionOptions);
+    const ddexSource = new DdexSource();
+    const markets = await ddexSource.getActiveMarketsAsync();
+    for (const marketsChunk of R.splitEvery(MARKET_ORDERBOOK_REQUEST_BATCH_SIZE, markets)) {
+        await Promise.all(
+            marketsChunk.map(async (market: DdexMarket) => getAndSaveMarketOrderbook(ddexSource, market)),
+        );
+        await new Promise(resolve => setTimeout(resolve, MILLISEC_MARKET_ORDERBOOK_REQUEST_DELAY));
+    }
+    process.exit(0);
+})().catch(handleError);
+
+/**
+ * Retrieve orderbook from Ddex API for a given market. Parse orders and insert
+ * them into our database.
+ * @param ddexSource Data source which can query Ddex API.
+ * @param market Object from Ddex API containing market data.
+ */
+async function getAndSaveMarketOrderbook(ddexSource: DdexSource, market: DdexMarket): Promise<void> {
+    const orderBook = await ddexSource.getMarketOrderbookAsync(market.id);
+    const observedTimestamp = Date.now();
+
+    logUtils.log(`${market.id}: Parsing orders.`);
+    const orders = parseDdexOrders(orderBook, market, observedTimestamp, DDEX_SOURCE);
+
+    if (orders.length > 0) {
+        logUtils.log(`${market.id}: Saving ${orders.length} orders.`);
+        const TokenOrderRepository = connection.getRepository(TokenOrder);
+        await TokenOrderRepository.save(orders, { chunk: Math.ceil(orders.length / BATCH_SAVE_SIZE) });
+    } else {
+        logUtils.log(`${market.id}: 0 orders to save.`);
+    }
+}
diff --git a/packages/pipeline/src/scripts/pull_paradex_orderbook_snapshots.ts b/packages/pipeline/src/scripts/pull_paradex_orderbook_snapshots.ts
new file mode 100644
index 000000000..bae1fbede
--- /dev/null
+++ b/packages/pipeline/src/scripts/pull_paradex_orderbook_snapshots.ts
@@ -0,0 +1,87 @@
+import { logUtils } from '@0x/utils';
+import { Connection, ConnectionOptions, createConnection } from 'typeorm';
+
+import {
+    PARADEX_SOURCE,
+    ParadexActiveMarketsResponse,
+    ParadexMarket,
+    ParadexSource,
+    ParadexTokenInfoResponse,
+} from '../data_sources/paradex';
+import { TokenOrderbookSnapshot as TokenOrder } from '../entities';
+import * as ormConfig from '../ormconfig';
+import { parseParadexOrders } from '../parsers/paradex_orders';
+import { handleError } from '../utils';
+
+// Number of orders to save at once.
+const BATCH_SAVE_SIZE = 1000;
+
+let connection: Connection;
+
+(async () => {
+    connection = await createConnection(ormConfig as ConnectionOptions);
+    const apiKey = process.env.PARADEX_DATA_PIPELINE_API_KEY;
+    if (apiKey === undefined) {
+        throw new Error('Missing required env var: PARADEX_DATA_PIPELINE_API_KEY');
+    }
+    const paradexSource = new ParadexSource(apiKey);
+    const markets = await paradexSource.getActiveMarketsAsync();
+    const tokenInfoResponse = await paradexSource.getTokenInfoAsync();
+    const extendedMarkets = addTokenAddresses(markets, tokenInfoResponse);
+    await Promise.all(
+        extendedMarkets.map(async (market: ParadexMarket) => getAndSaveMarketOrderbook(paradexSource, market)),
+    );
+    process.exit(0);
+})().catch(handleError);
+
+/**
+ * Extend the default ParadexMarket objects with token addresses.
+ * @param markets An array of ParadexMarket objects.
+ * @param tokenInfoResponse An array of ParadexTokenInfo containing the addresses.
+ */
+function addTokenAddresses(
+    markets: ParadexActiveMarketsResponse,
+    tokenInfoResponse: ParadexTokenInfoResponse,
+): ParadexMarket[] {
+    const symbolAddressMapping = new Map<string, string>();
+    tokenInfoResponse.forEach(tokenInfo => symbolAddressMapping.set(tokenInfo.symbol, tokenInfo.address));
+
+    markets.forEach((market: ParadexMarket) => {
+        if (symbolAddressMapping.has(market.baseToken)) {
+            market.baseTokenAddress = symbolAddressMapping.get(market.baseToken);
+        } else {
+            market.quoteTokenAddress = '';
+            logUtils.warn(`${market.baseToken}: No address found.`);
+        }
+
+        if (symbolAddressMapping.has(market.quoteToken)) {
+            market.quoteTokenAddress = symbolAddressMapping.get(market.quoteToken);
+        } else {
+            market.quoteTokenAddress = '';
+            logUtils.warn(`${market.quoteToken}: No address found.`);
+        }
+    });
+    return markets;
+}
+
+/**
+ * Retrieve orderbook from Paradex API for a given market. Parse orders and insert
+ * them into our database.
+ * @param paradexSource Data source which can query the Paradex API.
+ * @param market Object from the Paradex API with information about the market in question.
+ */
+async function getAndSaveMarketOrderbook(paradexSource: ParadexSource, market: ParadexMarket): Promise<void> {
+    const paradexOrderbookResponse = await paradexSource.getMarketOrderbookAsync(market.symbol);
+    const observedTimestamp = Date.now();
+
+    logUtils.log(`${market.symbol}: Parsing orders.`);
+    const orders = parseParadexOrders(paradexOrderbookResponse, market, observedTimestamp, PARADEX_SOURCE);
+
+    if (orders.length > 0) {
+        logUtils.log(`${market.symbol}: Saving ${orders.length} orders.`);
+        const tokenOrderRepository = connection.getRepository(TokenOrder);
+        await tokenOrderRepository.save(orders, { chunk: Math.ceil(orders.length / BATCH_SAVE_SIZE) });
+    } else {
+        logUtils.log(`${market.symbol}: 0 orders to save.`);
+    }
+}