Add preliminary support for scraping orders from SRA endpoints (no pagination, only RR support for now)

8 files changed, 134 insertions, 12 deletions

diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json
index 6670938f2..c01b7f448 100644
--- a/packages/pipeline/package.json
+++ b/packages/pipeline/package.json
@@ -39,6 +39,7 @@
         "typescript": "3.0.1"
     },
     "dependencies": {
+        "@0xproject/connect": "^2.0.4",
         "@0xproject/contract-wrappers": "^1.0.1",
         "@0xproject/order-utils": "^1.0.2",
         "@0xproject/subproviders": "^2.0.2",
diff --git a/packages/pipeline/src/data_types/events/exchange_events.ts b/packages/pipeline/src/data_types/events/exchange_events.ts
index 2d4059a0e..a406d27fb 100644
--- a/packages/pipeline/src/data_types/events/exchange_events.ts
+++ b/packages/pipeline/src/data_types/events/exchange_events.ts
@@ -6,7 +6,6 @@ import {
 } from '@0xproject/contract-wrappers';
 import { assetDataUtils } from '@0xproject/order-utils';
 import { AssetProxyId, ERC721AssetData } from '@0xproject/types';
-import { BigNumber } from '@0xproject/utils';
 import { LogWithDecodedArgs } from 'ethereum-types';
 import * as R from 'ramda';
 
@@ -15,6 +14,7 @@ import { EventsResponse } from '../../data_sources/etherscan';
 import { ExchangeCancelEvent } from '../../entities/ExchangeCancelEvent';
 import { ExchangeCancelUpToEvent } from '../../entities/ExchangeCancelUpToEvent';
 import { ExchangeFillEvent } from '../../entities/ExchangeFillEvent';
+import { bigNumbertoStringOrNull } from '../../utils';
 
 import { convertResponseToLogEntry, decodeLogEntry } from './event_utils';
 
@@ -130,10 +130,3 @@ export function _convertToExchangeCancelUpToEvent(
     exchangeCancelUpToEvent.orderEpoch = eventLog.args.orderEpoch.toString();
     return exchangeCancelUpToEvent;
 }
-
-function bigNumbertoStringOrNull(n: BigNumber): string | null {
-    if (n == null) {
-        return null;
-    }
-    return n.toString();
-}
diff --git a/packages/pipeline/src/data_types/sra_order/index.ts b/packages/pipeline/src/data_types/sra_order/index.ts
new file mode 100644
index 000000000..b6364415c
--- /dev/null
+++ b/packages/pipeline/src/data_types/sra_order/index.ts
@@ -0,0 +1,54 @@
+import { APIOrder, OrdersResponse } from '@0xproject/connect';
+import { assetDataUtils, orderHashUtils } from '@0xproject/order-utils';
+import { AssetProxyId, ERC721AssetData } from '@0xproject/types';
+import * as R from 'ramda';
+
+import { SraOrder } from '../../entities/SraOrder';
+import { bigNumbertoStringOrNull } from '../../utils';
+
+export function parseSraOrders(rawOrdersResponse: OrdersResponse): SraOrder[] {
+    return R.map(_convertToEntity, rawOrdersResponse.records);
+}
+
+export function _convertToEntity(apiOrder: APIOrder): SraOrder {
+    // TODO(albrow): refactor out common asset data decoding code.
+    const makerAssetData = assetDataUtils.decodeAssetDataOrThrow(apiOrder.order.makerAssetData);
+    const makerAssetType = makerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721';
+    const takerAssetData = assetDataUtils.decodeAssetDataOrThrow(apiOrder.order.takerAssetData);
+    const takerAssetType = takerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721';
+
+    const sraOrder = new SraOrder();
+    sraOrder.exchangeAddress = apiOrder.order.exchangeAddress;
+    sraOrder.orderHashHex = orderHashUtils.getOrderHashHex(apiOrder.order);
+
+    // TODO(albrow): Set these fields to the correct values upstack.
+    sraOrder.lastUpdatedTimestamp = Date.now();
+    sraOrder.firstSeenTimestamp = Date.now();
+
+    sraOrder.makerAddress = apiOrder.order.makerAddress;
+    sraOrder.takerAddress = apiOrder.order.takerAddress;
+    sraOrder.feeRecipientAddress = apiOrder.order.feeRecipientAddress;
+    sraOrder.senderAddress = apiOrder.order.senderAddress;
+    sraOrder.makerAssetAmount = apiOrder.order.makerAssetAmount.toString();
+    sraOrder.takerAssetAmount = apiOrder.order.takerAssetAmount.toString();
+    sraOrder.makerFee = apiOrder.order.makerFee.toString();
+    sraOrder.takerFee = apiOrder.order.takerFee.toString();
+    sraOrder.expirationTimeSeconds = apiOrder.order.expirationTimeSeconds.toString();
+    sraOrder.salt = apiOrder.order.salt.toString();
+    sraOrder.signature = apiOrder.order.signature;
+
+    sraOrder.rawMakerAssetData = apiOrder.order.makerAssetData;
+    sraOrder.makerAssetType = makerAssetType;
+    sraOrder.makerAssetProxyId = makerAssetData.assetProxyId;
+    sraOrder.makerTokenAddress = makerAssetData.tokenAddress;
+    sraOrder.makerTokenId = bigNumbertoStringOrNull((makerAssetData as ERC721AssetData).tokenId);
+    sraOrder.rawTakerAssetData = apiOrder.order.takerAssetData;
+    sraOrder.takerAssetType = takerAssetType;
+    sraOrder.takerAssetProxyId = takerAssetData.assetProxyId;
+    sraOrder.takerTokenAddress = takerAssetData.tokenAddress;
+    sraOrder.takerTokenId = bigNumbertoStringOrNull((takerAssetData as ERC721AssetData).tokenId);
+
+    sraOrder.metaDataJson = JSON.stringify(apiOrder.metaData);
+
+    return sraOrder;
+}
diff --git a/packages/pipeline/src/entities/ExchangeCancelEvent.ts b/packages/pipeline/src/entities/ExchangeCancelEvent.ts
index d0188c2f5..7010ab9f2 100644
--- a/packages/pipeline/src/entities/ExchangeCancelEvent.ts
+++ b/packages/pipeline/src/entities/ExchangeCancelEvent.ts
@@ -16,6 +16,7 @@ export class ExchangeCancelEvent extends BaseEntity {
     @Column() public feeRecepientAddress!: string;
     @Column() public senderAddress!: string;
     @Column() public orderHash!: string;
+
     @Column() public rawMakerAssetData!: string;
     @Column() public makerAssetType!: AssetType;
     @Column() public makerAssetProxyId!: string;
@@ -28,5 +29,6 @@ export class ExchangeCancelEvent extends BaseEntity {
     @Column() public takerTokenAddress!: string;
     @Column({ nullable: true, type: String })
     public takerTokenId!: string | null;
+
     // TODO(albrow): Include topics?
 }
diff --git a/packages/pipeline/src/entities/ExchangeFillEvent.ts b/packages/pipeline/src/entities/ExchangeFillEvent.ts
index abd73191a..5eafa7449 100644
--- a/packages/pipeline/src/entities/ExchangeFillEvent.ts
+++ b/packages/pipeline/src/entities/ExchangeFillEvent.ts
@@ -19,6 +19,7 @@ export class ExchangeFillEvent extends BaseEntity {
     @Column() public makerFeePaid!: string;
     @Column() public takerFeePaid!: string;
     @Column() public orderHash!: string;
+
     @Column() public rawMakerAssetData!: string;
     @Column() public makerAssetType!: AssetType;
     @Column() public makerAssetProxyId!: string;
@@ -31,5 +32,6 @@ export class ExchangeFillEvent extends BaseEntity {
     @Column() public takerTokenAddress!: string;
     @Column({ nullable: true, type: String })
     public takerTokenId!: string | null;
+
     // TODO(albrow): Include topics?
 }
diff --git a/packages/pipeline/src/entities/SraOrder.ts b/packages/pipeline/src/entities/SraOrder.ts
new file mode 100644
index 000000000..c9a1f926d
--- /dev/null
+++ b/packages/pipeline/src/entities/SraOrder.ts
@@ -0,0 +1,40 @@
+import { BaseEntity, Column, Entity, PrimaryColumn } from 'typeorm';
+
+import { AssetType } from '../types';
+
+@Entity()
+export class SraOrder extends BaseEntity {
+    @PrimaryColumn() public exchangeAddress!: string;
+    @PrimaryColumn() public orderHashHex!: string;
+
+    @Column() public lastUpdatedTimestamp!: number;
+    @Column() public firstSeenTimestamp!: number;
+
+    @Column() public makerAddress!: string;
+    @Column() public takerAddress!: string;
+    @Column() public feeRecipientAddress!: string;
+    @Column() public senderAddress!: string;
+    @Column() public makerAssetAmount!: string;
+    @Column() public takerAssetAmount!: string;
+    @Column() public makerFee!: string;
+    @Column() public takerFee!: string;
+    @Column() public expirationTimeSeconds!: string;
+    @Column() public salt!: string;
+    @Column() public signature!: string;
+
+    @Column() public rawMakerAssetData!: string;
+    @Column() public makerAssetType!: AssetType;
+    @Column() public makerAssetProxyId!: string;
+    @Column() public makerTokenAddress!: string;
+    @Column({ nullable: true, type: String })
+    public makerTokenId!: string | null;
+    @Column() public rawTakerAssetData!: string;
+    @Column() public takerAssetType!: AssetType;
+    @Column() public takerAssetProxyId!: string;
+    @Column() public takerTokenAddress!: string;
+    @Column({ nullable: true, type: String })
+    public takerTokenId!: string | null;
+
+    // TODO(albrow): Make this optional?
+    @Column() public metaDataJson!: string;
+}
diff --git a/packages/pipeline/src/index.ts b/packages/pipeline/src/index.ts
index 07ab1d991..3e8434e3d 100644
--- a/packages/pipeline/src/index.ts
+++ b/packages/pipeline/src/index.ts
@@ -1,19 +1,28 @@
-import * as R from 'ramda';
+import { HttpClient } from '@0xproject/connect';
 import 'reflect-metadata';
-import { createConnection } from 'typeorm';
+import { Connection, createConnection } from 'typeorm';
 
 import { Etherscan } from './data_sources/etherscan';
 import { parseExchangeEvents } from './data_types/events/exchange_events';
+import { parseSraOrders } from './data_types/sra_order';
 import { ExchangeCancelEvent } from './entities/ExchangeCancelEvent';
 import { ExchangeCancelUpToEvent } from './entities/ExchangeCancelUpToEvent';
 import { ExchangeFillEvent } from './entities/ExchangeFillEvent';
+import { SraOrder } from './entities/SraOrder';
 import { config } from './ormconfig';
 
 const etherscan = new Etherscan(process.env.ETHERSCAN_API_KEY as string);
 const EXCHANGE_ADDRESS = '0x4f833a24e1f95d70f028921e27040ca56e09ab0b';
 
+let connection: Connection;
+
 (async () => {
-    const connection = await createConnection(config);
+    connection = await createConnection(config);
+    await getExchangeEventsAsync();
+    await getSraOrdersAsync();
+})();
+
+async function getExchangeEventsAsync(): Promise<void> {
     const fillRepository = connection.getRepository(ExchangeFillEvent);
     const cancelRepository = connection.getRepository(ExchangeCancelEvent);
     const cancelUpToRepository = connection.getRepository(ExchangeCancelUpToEvent);
@@ -32,4 +41,17 @@ const EXCHANGE_ADDRESS = '0x4f833a24e1f95d70f028921e27040ca56e09ab0b';
             (await cancelRepository.count()) +
             (await cancelUpToRepository.count())} total events`,
     );
-})();
+}
+
+async function getSraOrdersAsync(): Promise<void> {
+    const orderRepository = connection.getRepository(SraOrder);
+    console.log(`found ${await orderRepository.count()} existing orders`);
+
+    const connect = new HttpClient('https://api.radarrelay.com/0x/v2');
+    const rawOrders = await connect.getOrdersAsync();
+    const orders = parseSraOrders(rawOrders);
+    for (const order of orders) {
+        order.save();
+    }
+    console.log(`now there are ${await orderRepository.count()} total orders`);
+}
diff --git a/packages/pipeline/src/utils/index.ts b/packages/pipeline/src/utils/index.ts
new file mode 100644
index 000000000..8fe7f9685
--- /dev/null
+++ b/packages/pipeline/src/utils/index.ts
@@ -0,0 +1,8 @@
+import { BigNumber } from '@0xproject/utils';
+
+export function bigNumbertoStringOrNull(n: BigNumber): string | null {
+    if (n == null) {
+        return null;
+    }
+    return n.toString();
+}