aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAlex Browne <stephenalexbrowne@gmail.com>2018-11-14 06:05:49 +0800
committerFred Carlsen <fred@sjelfull.no>2018-12-06 19:04:25 +0800
commit5c655eeda75f8a859378a0511d9e19438c149a60 (patch)
treec594874eb53a7c8771ea7ca33aef8e1858bdcd3f
parentd9324485637ce96112eb02150cb99e993b37a694 (diff)
downloaddexon-sol-tools-5c655eeda75f8a859378a0511d9e19438c149a60.tar
dexon-sol-tools-5c655eeda75f8a859378a0511d9e19438c149a60.tar.gz
dexon-sol-tools-5c655eeda75f8a859378a0511d9e19438c149a60.tar.bz2
dexon-sol-tools-5c655eeda75f8a859378a0511d9e19438c149a60.tar.lz
dexon-sol-tools-5c655eeda75f8a859378a0511d9e19438c149a60.tar.xz
dexon-sol-tools-5c655eeda75f8a859378a0511d9e19438c149a60.tar.zst
dexon-sol-tools-5c655eeda75f8a859378a0511d9e19438c149a60.zip
Preliminary work for adding RR order book scraping
-rw-r--r--packages/pipeline/src/scripts/pull_radar_relay_orders.ts52
1 files changed, 52 insertions, 0 deletions
diff --git a/packages/pipeline/src/scripts/pull_radar_relay_orders.ts b/packages/pipeline/src/scripts/pull_radar_relay_orders.ts
new file mode 100644
index 000000000..c4d3f7095
--- /dev/null
+++ b/packages/pipeline/src/scripts/pull_radar_relay_orders.ts
@@ -0,0 +1,52 @@
+// tslint:disable:no-console
+import { HttpClient } from '@0x/connect';
+import * as R from 'ramda';
+import 'reflect-metadata';
+import { Connection, ConnectionOptions, createConnection } from 'typeorm';
+
+import { SraOrder } from '../entities';
+import * as ormConfig from '../ormconfig';
+import { parseSraOrders } from '../parsers/sra_orders';
+import { handleError } from '../utils';
+
+const RADAR_RELAY_URL = 'https://api.radarrelay.com/0x/v2';
+const BATCH_SAVE_SIZE = 1000; // Number of orders to save at once.
+const ORDERS_PER_PAGE = 10000; // Number of orders to get per request.
+
+let connection: Connection;
+
+(async () => {
+ connection = await createConnection(ormConfig as ConnectionOptions);
+ await getOrderbook();
+ process.exit(0);
+})().catch(handleError);
+
+async function getOrderbook(): Promise<void> {
+ console.log('Getting all orders...');
+ const connectClient = new HttpClient(RADAR_RELAY_URL);
+ const rawOrders = await connectClient.getOrdersAsync({
+ perPage: ORDERS_PER_PAGE,
+ });
+ console.log(`Got ${rawOrders.records.length} orders.`);
+ console.log('Parsing orders...');
+ const orders = R.pipe(parseSraOrders, R.map(setSourceUrl(RADAR_RELAY_URL)))(rawOrders);
+ const ordersRepository = connection.getRepository(SraOrder);
+ // TODO(albrow): Move batch saving to a utility function to reduce
+ // duplicated code.
+ for (const ordersBatch of R.splitEvery(BATCH_SAVE_SIZE, orders)) {
+ await ordersRepository.save(ordersBatch);
+ }
+}
+
+const sourceUrlProp = R.lensProp('sourceUrl');
+
+const setSourceUrl = R.curry((sourceURL: string, order: SraOrder): SraOrder => {
+ return R.set(sourceUrlProp, sourceURL, order);
+});
+
+const firstSeenProp = R.lensProp('firstSeenTimestamp');
+const lastUpdatedProp = R.lensProp('lastUpdatedTimestamp');
+
+const setFirstSeen = R.curry((sourceURL: string, order: SraOrder): SraOrder => {
+ return R.set(firstSeenTimestampProp, sourceURL, order);
+});