aboutsummaryrefslogtreecommitdiffstats
path: root/packages/pipeline/src
diff options
context:
space:
mode:
authorAlex Browne <stephenalexbrowne@gmail.com>2018-09-27 07:42:48 +0800
committerAlex Browne <stephenalexbrowne@gmail.com>2018-12-05 06:24:06 +0800
commit9ab55ccec0478e65f9d605aa2da7ed1fa13e01ac (patch)
tree85afc226768cf0d2c0568b2db263b950831e7032 /packages/pipeline/src
parent71a015f2aa32fedd3d3c5e9d5a3fa4eb272b2852 (diff)
downloaddexon-sol-tools-9ab55ccec0478e65f9d605aa2da7ed1fa13e01ac.tar
dexon-sol-tools-9ab55ccec0478e65f9d605aa2da7ed1fa13e01ac.tar.gz
dexon-sol-tools-9ab55ccec0478e65f9d605aa2da7ed1fa13e01ac.tar.bz2
dexon-sol-tools-9ab55ccec0478e65f9d605aa2da7ed1fa13e01ac.tar.lz
dexon-sol-tools-9ab55ccec0478e65f9d605aa2da7ed1fa13e01ac.tar.xz
dexon-sol-tools-9ab55ccec0478e65f9d605aa2da7ed1fa13e01ac.tar.zst
dexon-sol-tools-9ab55ccec0478e65f9d605aa2da7ed1fa13e01ac.zip
Add preliminary support for scraping orders from SRA endpoints (no pagination, only RR support for now)
Diffstat (limited to 'packages/pipeline/src')
-rw-r--r--packages/pipeline/src/data_types/events/exchange_events.ts9
-rw-r--r--packages/pipeline/src/data_types/sra_order/index.ts54
-rw-r--r--packages/pipeline/src/entities/ExchangeCancelEvent.ts2
-rw-r--r--packages/pipeline/src/entities/ExchangeFillEvent.ts2
-rw-r--r--packages/pipeline/src/entities/SraOrder.ts40
-rw-r--r--packages/pipeline/src/index.ts30
-rw-r--r--packages/pipeline/src/utils/index.ts8
7 files changed, 133 insertions, 12 deletions
diff --git a/packages/pipeline/src/data_types/events/exchange_events.ts b/packages/pipeline/src/data_types/events/exchange_events.ts
index 2d4059a0e..a406d27fb 100644
--- a/packages/pipeline/src/data_types/events/exchange_events.ts
+++ b/packages/pipeline/src/data_types/events/exchange_events.ts
@@ -6,7 +6,6 @@ import {
} from '@0xproject/contract-wrappers';
import { assetDataUtils } from '@0xproject/order-utils';
import { AssetProxyId, ERC721AssetData } from '@0xproject/types';
-import { BigNumber } from '@0xproject/utils';
import { LogWithDecodedArgs } from 'ethereum-types';
import * as R from 'ramda';
@@ -15,6 +14,7 @@ import { EventsResponse } from '../../data_sources/etherscan';
import { ExchangeCancelEvent } from '../../entities/ExchangeCancelEvent';
import { ExchangeCancelUpToEvent } from '../../entities/ExchangeCancelUpToEvent';
import { ExchangeFillEvent } from '../../entities/ExchangeFillEvent';
+import { bigNumbertoStringOrNull } from '../../utils';
import { convertResponseToLogEntry, decodeLogEntry } from './event_utils';
@@ -130,10 +130,3 @@ export function _convertToExchangeCancelUpToEvent(
exchangeCancelUpToEvent.orderEpoch = eventLog.args.orderEpoch.toString();
return exchangeCancelUpToEvent;
}
-
-function bigNumbertoStringOrNull(n: BigNumber): string | null {
- if (n == null) {
- return null;
- }
- return n.toString();
-}
diff --git a/packages/pipeline/src/data_types/sra_order/index.ts b/packages/pipeline/src/data_types/sra_order/index.ts
new file mode 100644
index 000000000..b6364415c
--- /dev/null
+++ b/packages/pipeline/src/data_types/sra_order/index.ts
@@ -0,0 +1,54 @@
+import { APIOrder, OrdersResponse } from '@0xproject/connect';
+import { assetDataUtils, orderHashUtils } from '@0xproject/order-utils';
+import { AssetProxyId, ERC721AssetData } from '@0xproject/types';
+import * as R from 'ramda';
+
+import { SraOrder } from '../../entities/SraOrder';
+import { bigNumbertoStringOrNull } from '../../utils';
+
+export function parseSraOrders(rawOrdersResponse: OrdersResponse): SraOrder[] {
+ return R.map(_convertToEntity, rawOrdersResponse.records);
+}
+
+export function _convertToEntity(apiOrder: APIOrder): SraOrder {
+ // TODO(albrow): refactor out common asset data decoding code.
+ const makerAssetData = assetDataUtils.decodeAssetDataOrThrow(apiOrder.order.makerAssetData);
+ const makerAssetType = makerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721';
+ const takerAssetData = assetDataUtils.decodeAssetDataOrThrow(apiOrder.order.takerAssetData);
+ const takerAssetType = takerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721';
+
+ const sraOrder = new SraOrder();
+ sraOrder.exchangeAddress = apiOrder.order.exchangeAddress;
+ sraOrder.orderHashHex = orderHashUtils.getOrderHashHex(apiOrder.order);
+
+ // TODO(albrow): Set these fields to the correct values upstack.
+ sraOrder.lastUpdatedTimestamp = Date.now();
+ sraOrder.firstSeenTimestamp = Date.now();
+
+ sraOrder.makerAddress = apiOrder.order.makerAddress;
+ sraOrder.takerAddress = apiOrder.order.takerAddress;
+ sraOrder.feeRecipientAddress = apiOrder.order.feeRecipientAddress;
+ sraOrder.senderAddress = apiOrder.order.senderAddress;
+ sraOrder.makerAssetAmount = apiOrder.order.makerAssetAmount.toString();
+ sraOrder.takerAssetAmount = apiOrder.order.takerAssetAmount.toString();
+ sraOrder.makerFee = apiOrder.order.makerFee.toString();
+ sraOrder.takerFee = apiOrder.order.takerFee.toString();
+ sraOrder.expirationTimeSeconds = apiOrder.order.expirationTimeSeconds.toString();
+ sraOrder.salt = apiOrder.order.salt.toString();
+ sraOrder.signature = apiOrder.order.signature;
+
+ sraOrder.rawMakerAssetData = apiOrder.order.makerAssetData;
+ sraOrder.makerAssetType = makerAssetType;
+ sraOrder.makerAssetProxyId = makerAssetData.assetProxyId;
+ sraOrder.makerTokenAddress = makerAssetData.tokenAddress;
+ sraOrder.makerTokenId = bigNumbertoStringOrNull((makerAssetData as ERC721AssetData).tokenId);
+ sraOrder.rawTakerAssetData = apiOrder.order.takerAssetData;
+ sraOrder.takerAssetType = takerAssetType;
+ sraOrder.takerAssetProxyId = takerAssetData.assetProxyId;
+ sraOrder.takerTokenAddress = takerAssetData.tokenAddress;
+ sraOrder.takerTokenId = bigNumbertoStringOrNull((takerAssetData as ERC721AssetData).tokenId);
+
+ sraOrder.metaDataJson = JSON.stringify(apiOrder.metaData);
+
+ return sraOrder;
+}
diff --git a/packages/pipeline/src/entities/ExchangeCancelEvent.ts b/packages/pipeline/src/entities/ExchangeCancelEvent.ts
index d0188c2f5..7010ab9f2 100644
--- a/packages/pipeline/src/entities/ExchangeCancelEvent.ts
+++ b/packages/pipeline/src/entities/ExchangeCancelEvent.ts
@@ -16,6 +16,7 @@ export class ExchangeCancelEvent extends BaseEntity {
@Column() public feeRecepientAddress!: string;
@Column() public senderAddress!: string;
@Column() public orderHash!: string;
+
@Column() public rawMakerAssetData!: string;
@Column() public makerAssetType!: AssetType;
@Column() public makerAssetProxyId!: string;
@@ -28,5 +29,6 @@ export class ExchangeCancelEvent extends BaseEntity {
@Column() public takerTokenAddress!: string;
@Column({ nullable: true, type: String })
public takerTokenId!: string | null;
+
// TODO(albrow): Include topics?
}
diff --git a/packages/pipeline/src/entities/ExchangeFillEvent.ts b/packages/pipeline/src/entities/ExchangeFillEvent.ts
index abd73191a..5eafa7449 100644
--- a/packages/pipeline/src/entities/ExchangeFillEvent.ts
+++ b/packages/pipeline/src/entities/ExchangeFillEvent.ts
@@ -19,6 +19,7 @@ export class ExchangeFillEvent extends BaseEntity {
@Column() public makerFeePaid!: string;
@Column() public takerFeePaid!: string;
@Column() public orderHash!: string;
+
@Column() public rawMakerAssetData!: string;
@Column() public makerAssetType!: AssetType;
@Column() public makerAssetProxyId!: string;
@@ -31,5 +32,6 @@ export class ExchangeFillEvent extends BaseEntity {
@Column() public takerTokenAddress!: string;
@Column({ nullable: true, type: String })
public takerTokenId!: string | null;
+
// TODO(albrow): Include topics?
}
diff --git a/packages/pipeline/src/entities/SraOrder.ts b/packages/pipeline/src/entities/SraOrder.ts
new file mode 100644
index 000000000..c9a1f926d
--- /dev/null
+++ b/packages/pipeline/src/entities/SraOrder.ts
@@ -0,0 +1,40 @@
+import { BaseEntity, Column, Entity, PrimaryColumn } from 'typeorm';
+
+import { AssetType } from '../types';
+
+@Entity()
+export class SraOrder extends BaseEntity {
+ @PrimaryColumn() public exchangeAddress!: string;
+ @PrimaryColumn() public orderHashHex!: string;
+
+ @Column() public lastUpdatedTimestamp!: number;
+ @Column() public firstSeenTimestamp!: number;
+
+ @Column() public makerAddress!: string;
+ @Column() public takerAddress!: string;
+ @Column() public feeRecipientAddress!: string;
+ @Column() public senderAddress!: string;
+ @Column() public makerAssetAmount!: string;
+ @Column() public takerAssetAmount!: string;
+ @Column() public makerFee!: string;
+ @Column() public takerFee!: string;
+ @Column() public expirationTimeSeconds!: string;
+ @Column() public salt!: string;
+ @Column() public signature!: string;
+
+ @Column() public rawMakerAssetData!: string;
+ @Column() public makerAssetType!: AssetType;
+ @Column() public makerAssetProxyId!: string;
+ @Column() public makerTokenAddress!: string;
+ @Column({ nullable: true, type: String })
+ public makerTokenId!: string | null;
+ @Column() public rawTakerAssetData!: string;
+ @Column() public takerAssetType!: AssetType;
+ @Column() public takerAssetProxyId!: string;
+ @Column() public takerTokenAddress!: string;
+ @Column({ nullable: true, type: String })
+ public takerTokenId!: string | null;
+
+ // TODO(albrow): Make this optional?
+ @Column() public metaDataJson!: string;
+}
diff --git a/packages/pipeline/src/index.ts b/packages/pipeline/src/index.ts
index 07ab1d991..3e8434e3d 100644
--- a/packages/pipeline/src/index.ts
+++ b/packages/pipeline/src/index.ts
@@ -1,19 +1,28 @@
-import * as R from 'ramda';
+import { HttpClient } from '@0xproject/connect';
import 'reflect-metadata';
-import { createConnection } from 'typeorm';
+import { Connection, createConnection } from 'typeorm';
import { Etherscan } from './data_sources/etherscan';
import { parseExchangeEvents } from './data_types/events/exchange_events';
+import { parseSraOrders } from './data_types/sra_order';
import { ExchangeCancelEvent } from './entities/ExchangeCancelEvent';
import { ExchangeCancelUpToEvent } from './entities/ExchangeCancelUpToEvent';
import { ExchangeFillEvent } from './entities/ExchangeFillEvent';
+import { SraOrder } from './entities/SraOrder';
import { config } from './ormconfig';
const etherscan = new Etherscan(process.env.ETHERSCAN_API_KEY as string);
const EXCHANGE_ADDRESS = '0x4f833a24e1f95d70f028921e27040ca56e09ab0b';
+let connection: Connection;
+
(async () => {
- const connection = await createConnection(config);
+ connection = await createConnection(config);
+ await getExchangeEventsAsync();
+ await getSraOrdersAsync();
+})();
+
+async function getExchangeEventsAsync(): Promise<void> {
const fillRepository = connection.getRepository(ExchangeFillEvent);
const cancelRepository = connection.getRepository(ExchangeCancelEvent);
const cancelUpToRepository = connection.getRepository(ExchangeCancelUpToEvent);
@@ -32,4 +41,17 @@ const EXCHANGE_ADDRESS = '0x4f833a24e1f95d70f028921e27040ca56e09ab0b';
(await cancelRepository.count()) +
(await cancelUpToRepository.count())} total events`,
);
-})();
+}
+
+async function getSraOrdersAsync(): Promise<void> {
+ const orderRepository = connection.getRepository(SraOrder);
+ console.log(`found ${await orderRepository.count()} existing orders`);
+
+ const connect = new HttpClient('https://api.radarrelay.com/0x/v2');
+ const rawOrders = await connect.getOrdersAsync();
+ const orders = parseSraOrders(rawOrders);
+ for (const order of orders) {
+ order.save();
+ }
+ console.log(`now there are ${await orderRepository.count()} total orders`);
+}
diff --git a/packages/pipeline/src/utils/index.ts b/packages/pipeline/src/utils/index.ts
new file mode 100644
index 000000000..8fe7f9685
--- /dev/null
+++ b/packages/pipeline/src/utils/index.ts
@@ -0,0 +1,8 @@
+import { BigNumber } from '@0xproject/utils';
+
+export function bigNumbertoStringOrNull(n: BigNumber): string | null {
+ if (n == null) {
+ return null;
+ }
+ return n.toString();
+}