From 9ab55ccec0478e65f9d605aa2da7ed1fa13e01ac Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Wed, 26 Sep 2018 16:42:48 -0700 Subject: Add preliminary support for scraping orders from SRA endpoints (no pagination, only RR support for now) --- packages/pipeline/package.json | 1 + .../src/data_types/events/exchange_events.ts | 9 +--- .../pipeline/src/data_types/sra_order/index.ts | 54 ++++++++++++++++++++++ .../pipeline/src/entities/ExchangeCancelEvent.ts | 2 + .../pipeline/src/entities/ExchangeFillEvent.ts | 2 + packages/pipeline/src/entities/SraOrder.ts | 40 ++++++++++++++++ packages/pipeline/src/index.ts | 30 ++++++++++-- packages/pipeline/src/utils/index.ts | 8 ++++ 8 files changed, 134 insertions(+), 12 deletions(-) create mode 100644 packages/pipeline/src/data_types/sra_order/index.ts create mode 100644 packages/pipeline/src/entities/SraOrder.ts create mode 100644 packages/pipeline/src/utils/index.ts (limited to 'packages') diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index 6670938f2..c01b7f448 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -39,6 +39,7 @@ "typescript": "3.0.1" }, "dependencies": { + "@0xproject/connect": "^2.0.4", "@0xproject/contract-wrappers": "^1.0.1", "@0xproject/order-utils": "^1.0.2", "@0xproject/subproviders": "^2.0.2", diff --git a/packages/pipeline/src/data_types/events/exchange_events.ts b/packages/pipeline/src/data_types/events/exchange_events.ts index 2d4059a0e..a406d27fb 100644 --- a/packages/pipeline/src/data_types/events/exchange_events.ts +++ b/packages/pipeline/src/data_types/events/exchange_events.ts @@ -6,7 +6,6 @@ import { } from '@0xproject/contract-wrappers'; import { assetDataUtils } from '@0xproject/order-utils'; import { AssetProxyId, ERC721AssetData } from '@0xproject/types'; -import { BigNumber } from '@0xproject/utils'; import { LogWithDecodedArgs } from 'ethereum-types'; import * as R from 'ramda'; @@ -15,6 +14,7 @@ import { EventsResponse } from '../../data_sources/etherscan'; import { ExchangeCancelEvent } from '../../entities/ExchangeCancelEvent'; import { ExchangeCancelUpToEvent } from '../../entities/ExchangeCancelUpToEvent'; import { ExchangeFillEvent } from '../../entities/ExchangeFillEvent'; +import { bigNumbertoStringOrNull } from '../../utils'; import { convertResponseToLogEntry, decodeLogEntry } from './event_utils'; @@ -130,10 +130,3 @@ export function _convertToExchangeCancelUpToEvent( exchangeCancelUpToEvent.orderEpoch = eventLog.args.orderEpoch.toString(); return exchangeCancelUpToEvent; } - -function bigNumbertoStringOrNull(n: BigNumber): string | null { - if (n == null) { - return null; - } - return n.toString(); -} diff --git a/packages/pipeline/src/data_types/sra_order/index.ts b/packages/pipeline/src/data_types/sra_order/index.ts new file mode 100644 index 000000000..b6364415c --- /dev/null +++ b/packages/pipeline/src/data_types/sra_order/index.ts @@ -0,0 +1,54 @@ +import { APIOrder, OrdersResponse } from '@0xproject/connect'; +import { assetDataUtils, orderHashUtils } from '@0xproject/order-utils'; +import { AssetProxyId, ERC721AssetData } from '@0xproject/types'; +import * as R from 'ramda'; + +import { SraOrder } from '../../entities/SraOrder'; +import { bigNumbertoStringOrNull } from '../../utils'; + +export function parseSraOrders(rawOrdersResponse: OrdersResponse): SraOrder[] { + return R.map(_convertToEntity, rawOrdersResponse.records); +} + +export function _convertToEntity(apiOrder: APIOrder): SraOrder { + // TODO(albrow): refactor out common asset data decoding code. + const makerAssetData = assetDataUtils.decodeAssetDataOrThrow(apiOrder.order.makerAssetData); + const makerAssetType = makerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721'; + const takerAssetData = assetDataUtils.decodeAssetDataOrThrow(apiOrder.order.takerAssetData); + const takerAssetType = takerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721'; + + const sraOrder = new SraOrder(); + sraOrder.exchangeAddress = apiOrder.order.exchangeAddress; + sraOrder.orderHashHex = orderHashUtils.getOrderHashHex(apiOrder.order); + + // TODO(albrow): Set these fields to the correct values upstack. + sraOrder.lastUpdatedTimestamp = Date.now(); + sraOrder.firstSeenTimestamp = Date.now(); + + sraOrder.makerAddress = apiOrder.order.makerAddress; + sraOrder.takerAddress = apiOrder.order.takerAddress; + sraOrder.feeRecipientAddress = apiOrder.order.feeRecipientAddress; + sraOrder.senderAddress = apiOrder.order.senderAddress; + sraOrder.makerAssetAmount = apiOrder.order.makerAssetAmount.toString(); + sraOrder.takerAssetAmount = apiOrder.order.takerAssetAmount.toString(); + sraOrder.makerFee = apiOrder.order.makerFee.toString(); + sraOrder.takerFee = apiOrder.order.takerFee.toString(); + sraOrder.expirationTimeSeconds = apiOrder.order.expirationTimeSeconds.toString(); + sraOrder.salt = apiOrder.order.salt.toString(); + sraOrder.signature = apiOrder.order.signature; + + sraOrder.rawMakerAssetData = apiOrder.order.makerAssetData; + sraOrder.makerAssetType = makerAssetType; + sraOrder.makerAssetProxyId = makerAssetData.assetProxyId; + sraOrder.makerTokenAddress = makerAssetData.tokenAddress; + sraOrder.makerTokenId = bigNumbertoStringOrNull((makerAssetData as ERC721AssetData).tokenId); + sraOrder.rawTakerAssetData = apiOrder.order.takerAssetData; + sraOrder.takerAssetType = takerAssetType; + sraOrder.takerAssetProxyId = takerAssetData.assetProxyId; + sraOrder.takerTokenAddress = takerAssetData.tokenAddress; + sraOrder.takerTokenId = bigNumbertoStringOrNull((takerAssetData as ERC721AssetData).tokenId); + + sraOrder.metaDataJson = JSON.stringify(apiOrder.metaData); + + return sraOrder; +} diff --git a/packages/pipeline/src/entities/ExchangeCancelEvent.ts b/packages/pipeline/src/entities/ExchangeCancelEvent.ts index d0188c2f5..7010ab9f2 100644 --- a/packages/pipeline/src/entities/ExchangeCancelEvent.ts +++ b/packages/pipeline/src/entities/ExchangeCancelEvent.ts @@ -16,6 +16,7 @@ export class ExchangeCancelEvent extends BaseEntity { @Column() public feeRecepientAddress!: string; @Column() public senderAddress!: string; @Column() public orderHash!: string; + @Column() public rawMakerAssetData!: string; @Column() public makerAssetType!: AssetType; @Column() public makerAssetProxyId!: string; @@ -28,5 +29,6 @@ export class ExchangeCancelEvent extends BaseEntity { @Column() public takerTokenAddress!: string; @Column({ nullable: true, type: String }) public takerTokenId!: string | null; + // TODO(albrow): Include topics? } diff --git a/packages/pipeline/src/entities/ExchangeFillEvent.ts b/packages/pipeline/src/entities/ExchangeFillEvent.ts index abd73191a..5eafa7449 100644 --- a/packages/pipeline/src/entities/ExchangeFillEvent.ts +++ b/packages/pipeline/src/entities/ExchangeFillEvent.ts @@ -19,6 +19,7 @@ export class ExchangeFillEvent extends BaseEntity { @Column() public makerFeePaid!: string; @Column() public takerFeePaid!: string; @Column() public orderHash!: string; + @Column() public rawMakerAssetData!: string; @Column() public makerAssetType!: AssetType; @Column() public makerAssetProxyId!: string; @@ -31,5 +32,6 @@ export class ExchangeFillEvent extends BaseEntity { @Column() public takerTokenAddress!: string; @Column({ nullable: true, type: String }) public takerTokenId!: string | null; + // TODO(albrow): Include topics? } diff --git a/packages/pipeline/src/entities/SraOrder.ts b/packages/pipeline/src/entities/SraOrder.ts new file mode 100644 index 000000000..c9a1f926d --- /dev/null +++ b/packages/pipeline/src/entities/SraOrder.ts @@ -0,0 +1,40 @@ +import { BaseEntity, Column, Entity, PrimaryColumn } from 'typeorm'; + +import { AssetType } from '../types'; + +@Entity() +export class SraOrder extends BaseEntity { + @PrimaryColumn() public exchangeAddress!: string; + @PrimaryColumn() public orderHashHex!: string; + + @Column() public lastUpdatedTimestamp!: number; + @Column() public firstSeenTimestamp!: number; + + @Column() public makerAddress!: string; + @Column() public takerAddress!: string; + @Column() public feeRecipientAddress!: string; + @Column() public senderAddress!: string; + @Column() public makerAssetAmount!: string; + @Column() public takerAssetAmount!: string; + @Column() public makerFee!: string; + @Column() public takerFee!: string; + @Column() public expirationTimeSeconds!: string; + @Column() public salt!: string; + @Column() public signature!: string; + + @Column() public rawMakerAssetData!: string; + @Column() public makerAssetType!: AssetType; + @Column() public makerAssetProxyId!: string; + @Column() public makerTokenAddress!: string; + @Column({ nullable: true, type: String }) + public makerTokenId!: string | null; + @Column() public rawTakerAssetData!: string; + @Column() public takerAssetType!: AssetType; + @Column() public takerAssetProxyId!: string; + @Column() public takerTokenAddress!: string; + @Column({ nullable: true, type: String }) + public takerTokenId!: string | null; + + // TODO(albrow): Make this optional? + @Column() public metaDataJson!: string; +} diff --git a/packages/pipeline/src/index.ts b/packages/pipeline/src/index.ts index 07ab1d991..3e8434e3d 100644 --- a/packages/pipeline/src/index.ts +++ b/packages/pipeline/src/index.ts @@ -1,19 +1,28 @@ -import * as R from 'ramda'; +import { HttpClient } from '@0xproject/connect'; import 'reflect-metadata'; -import { createConnection } from 'typeorm'; +import { Connection, createConnection } from 'typeorm'; import { Etherscan } from './data_sources/etherscan'; import { parseExchangeEvents } from './data_types/events/exchange_events'; +import { parseSraOrders } from './data_types/sra_order'; import { ExchangeCancelEvent } from './entities/ExchangeCancelEvent'; import { ExchangeCancelUpToEvent } from './entities/ExchangeCancelUpToEvent'; import { ExchangeFillEvent } from './entities/ExchangeFillEvent'; +import { SraOrder } from './entities/SraOrder'; import { config } from './ormconfig'; const etherscan = new Etherscan(process.env.ETHERSCAN_API_KEY as string); const EXCHANGE_ADDRESS = '0x4f833a24e1f95d70f028921e27040ca56e09ab0b'; +let connection: Connection; + (async () => { - const connection = await createConnection(config); + connection = await createConnection(config); + await getExchangeEventsAsync(); + await getSraOrdersAsync(); +})(); + +async function getExchangeEventsAsync(): Promise { const fillRepository = connection.getRepository(ExchangeFillEvent); const cancelRepository = connection.getRepository(ExchangeCancelEvent); const cancelUpToRepository = connection.getRepository(ExchangeCancelUpToEvent); @@ -32,4 +41,17 @@ const EXCHANGE_ADDRESS = '0x4f833a24e1f95d70f028921e27040ca56e09ab0b'; (await cancelRepository.count()) + (await cancelUpToRepository.count())} total events`, ); -})(); +} + +async function getSraOrdersAsync(): Promise { + const orderRepository = connection.getRepository(SraOrder); + console.log(`found ${await orderRepository.count()} existing orders`); + + const connect = new HttpClient('https://api.radarrelay.com/0x/v2'); + const rawOrders = await connect.getOrdersAsync(); + const orders = parseSraOrders(rawOrders); + for (const order of orders) { + order.save(); + } + console.log(`now there are ${await orderRepository.count()} total orders`); +} diff --git a/packages/pipeline/src/utils/index.ts b/packages/pipeline/src/utils/index.ts new file mode 100644 index 000000000..8fe7f9685 --- /dev/null +++ b/packages/pipeline/src/utils/index.ts @@ -0,0 +1,8 @@ +import { BigNumber } from '@0xproject/utils'; + +export function bigNumbertoStringOrNull(n: BigNumber): string | null { + if (n == null) { + return null; + } + return n.toString(); +} -- cgit v1.2.3