From 9e9104578c8526ff48ecdda8b87d61ccb3d66a2d Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Thu, 20 Sep 2018 17:25:48 -0700 Subject: Add support for decoding asset data --- packages/pipeline/package.json | 2 + .../pipeline/src/data-sources/etherscan/events.ts | 78 ++++++++++++++++++++-- .../pipeline/src/data-sources/etherscan/index.ts | 6 +- .../pipeline/src/entities/ExchangeFillEvent.ts | 17 ++++- packages/pipeline/src/index.ts | 27 ++------ 5 files changed, 94 insertions(+), 36 deletions(-) (limited to 'packages/pipeline') diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index d81ea2668..6670938f2 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -40,7 +40,9 @@ }, "dependencies": { "@0xproject/contract-wrappers": "^1.0.1", + "@0xproject/order-utils": "^1.0.2", "@0xproject/subproviders": "^2.0.2", + "@0xproject/types": "^1.0.1", "@0xproject/utils": "^1.0.8", "@types/ramda": "^0.25.38", "axios": "^0.18.0", diff --git a/packages/pipeline/src/data-sources/etherscan/events.ts b/packages/pipeline/src/data-sources/etherscan/events.ts index edc8cde7b..a828af527 100644 --- a/packages/pipeline/src/data-sources/etherscan/events.ts +++ b/packages/pipeline/src/data-sources/etherscan/events.ts @@ -1,8 +1,15 @@ -import { ExchangeEventArgs } from '@0xproject/contract-wrappers'; -import { AbiDecoder } from '@0xproject/utils'; +import { ExchangeEventArgs, ExchangeFillEventArgs } from '@0xproject/contract-wrappers'; +import { assetDataUtils } from '@0xproject/order-utils'; +import { AssetProxyId, ERC721AssetData } from '@0xproject/types'; +import { AbiDecoder, BigNumber } from '@0xproject/utils'; import { AbiDefinition, LogEntry, LogWithDecodedArgs } from 'ethereum-types'; import * as R from 'ramda'; +import { ExchangeFillEvent } from '../../entities/ExchangeFillEvent'; + +// TODO(albrow): Union with other exchange event entity types +export type ExchangeEventEntity = ExchangeFillEvent; + // Raw events response from etherescan.io export interface EventsResponse { status: string; @@ -56,17 +63,74 @@ export const _decodeLogEntry = R.curry((contractAbi: AbiDefinition[], log: LogEn return logWithDecodedArgs as LogWithDecodedArgs; }); +export function _convertToEntity(eventLog: LogWithDecodedArgs): ExchangeEventEntity { + switch (eventLog.event) { + case 'Fill': + return _convertToExchangeFillEvent(eventLog as LogWithDecodedArgs); + default: + throw new Error('unexpected eventLog.event type: ' + eventLog.event); + } +} + +export function _convertToExchangeFillEvent(eventLog: LogWithDecodedArgs): ExchangeFillEvent { + const makerAssetData = assetDataUtils.decodeAssetDataOrThrow(eventLog.args.makerAssetData); + const makerAssetType = makerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721'; + const takerAssetData = assetDataUtils.decodeAssetDataOrThrow(eventLog.args.takerAssetData); + const takerAssetType = takerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721'; + const exchangeFillEvent = new ExchangeFillEvent(); + exchangeFillEvent.logIndex = eventLog.logIndex as number; + exchangeFillEvent.address = eventLog.address as string; + exchangeFillEvent.rawData = eventLog.data as string; + exchangeFillEvent.blockNumber = eventLog.blockNumber as number; + exchangeFillEvent.makerAddress = eventLog.args.makerAddress.toString(); + exchangeFillEvent.takerAddress = eventLog.args.takerAddress.toString(); + exchangeFillEvent.feeRecepientAddress = eventLog.args.feeRecipientAddress; + exchangeFillEvent.senderAddress = eventLog.args.senderAddress; + exchangeFillEvent.makerAssetFilledAmount = eventLog.args.makerAssetFilledAmount.toString(); + exchangeFillEvent.takerAssetFilledAmount = eventLog.args.takerAssetFilledAmount.toString(); + exchangeFillEvent.makerFeePaid = eventLog.args.makerFeePaid.toString(); + exchangeFillEvent.takerFeePaid = eventLog.args.takerFeePaid.toString(); + exchangeFillEvent.orderHash = eventLog.args.orderHash; + exchangeFillEvent.rawMakerAssetData = eventLog.args.makerAssetData; + exchangeFillEvent.makerAssetType = makerAssetType; + exchangeFillEvent.makerAssetProxyId = makerAssetData.assetProxyId; + exchangeFillEvent.makerTokenAddress = makerAssetData.tokenAddress; + exchangeFillEvent.makerTokenId = bigNumbertoStringOrNull((makerAssetData as ERC721AssetData).tokenId); + exchangeFillEvent.rawTakerAssetData = eventLog.args.takerAssetData; + exchangeFillEvent.takerAssetType = takerAssetType; + exchangeFillEvent.takerAssetProxyId = takerAssetData.assetProxyId; + exchangeFillEvent.takerTokenAddress = takerAssetData.tokenAddress; + exchangeFillEvent.takerTokenId = bigNumbertoStringOrNull((takerAssetData as ERC721AssetData).tokenId); + return exchangeFillEvent; +} + +function bigNumbertoStringOrNull(n: BigNumber): string | null { + if (n == null) { + return null; + } + return n.toString(); +} + +function filterEventLogs( + eventLogs: Array>, +): Array> { + return R.filter(eventLog => eventLog.event === 'Fill', eventLogs); +} + /** * Parses and abi-decodes the raw events response from etherscan.io. * @param contractAbi The ABI for the contract that the events where emited from. * @param rawEventsResponse The raw events response from etherescan.io. - * @returns Parsed and decoded events. + * @returns Parsed and decoded event entities, ready to be saved to database. */ export function parseRawEventsResponse( contractAbi: AbiDefinition[], rawEventsResponse: EventsResponse, -): Array> { - return R.pipe(R.map(_convertResponseToLogEntry), R.map(_decodeLogEntry(contractAbi)))(rawEventsResponse.result); +): ExchangeEventEntity[] { + return R.pipe( + R.map(_convertResponseToLogEntry), + R.map(_decodeLogEntry(contractAbi)), + filterEventLogs, + R.map(_convertToEntity), + )(rawEventsResponse.result); } - -// export const parseRawEventsResponse = R.pipe(R.map(_convertResponseToLogEntry), R.map(_decodeLogEntry)); diff --git a/packages/pipeline/src/data-sources/etherscan/index.ts b/packages/pipeline/src/data-sources/etherscan/index.ts index 66b6b1a8d..c5eb2b9c6 100644 --- a/packages/pipeline/src/data-sources/etherscan/index.ts +++ b/packages/pipeline/src/data-sources/etherscan/index.ts @@ -1,7 +1,7 @@ import { default as axios } from 'axios'; -import { AbiDefinition, BlockParam, BlockParamLiteral, DecodedLogArgs, LogWithDecodedArgs } from 'ethereum-types'; +import { AbiDefinition, BlockParam, BlockParamLiteral } from 'ethereum-types'; -import { EventsResponse, parseRawEventsResponse } from './events'; +import { EventsResponse, ExchangeEventEntity, parseRawEventsResponse } from './events'; const ETHERSCAN_URL = 'https://api.etherscan.io/api'; @@ -24,7 +24,7 @@ export class Etherscan { contractAbi: AbiDefinition[], fromBlock: BlockParam = BlockParamLiteral.Earliest, toBlock: BlockParam = BlockParamLiteral.Latest, - ): Promise>> { + ): Promise { const fullURL = `${ETHERSCAN_URL}?module=logs&action=getLogs&address=${contractAddress}&fromBlock=${fromBlock}&toBlock=${toBlock}&apikey=${ this._apiKey }`; diff --git a/packages/pipeline/src/entities/ExchangeFillEvent.ts b/packages/pipeline/src/entities/ExchangeFillEvent.ts index 1e9e8d986..1716c60d1 100644 --- a/packages/pipeline/src/entities/ExchangeFillEvent.ts +++ b/packages/pipeline/src/entities/ExchangeFillEvent.ts @@ -1,5 +1,7 @@ import { Column, Entity, PrimaryColumn } from 'typeorm'; +export type ExchangeFillEventAssetType = 'erc20' | 'erc721'; + @Entity() export class ExchangeFillEvent { @PrimaryColumn() public logIndex!: number; @@ -17,8 +19,17 @@ export class ExchangeFillEvent { @Column() public makerFeePaid!: string; @Column() public takerFeePaid!: string; @Column() public orderHash!: string; - // TODO(albrow): Decode asset data. - @Column() public makerAssetData!: string; - @Column() public takerAssetData!: string; + @Column() public rawMakerAssetData!: string; + @Column() public makerAssetType!: ExchangeFillEventAssetType; + @Column() public makerAssetProxyId!: string; + @Column() public makerTokenAddress!: string; + @Column({ nullable: true, type: String }) + public makerTokenId!: string | null; + @Column() public rawTakerAssetData!: string; + @Column() public takerAssetType!: ExchangeFillEventAssetType; + @Column() public takerAssetProxyId!: string; + @Column() public takerTokenAddress!: string; + @Column({ nullable: true, type: String }) + public takerTokenId!: string | null; // TODO(albrow): Include topics? } diff --git a/packages/pipeline/src/index.ts b/packages/pipeline/src/index.ts index d70ec3e3e..db26343e0 100644 --- a/packages/pipeline/src/index.ts +++ b/packages/pipeline/src/index.ts @@ -1,4 +1,5 @@ import { ExchangeFillEventArgs } from '@0xproject/contract-wrappers'; +import { assetDataUtils } from '@0xproject/order-utils'; import { LogWithDecodedArgs } from 'ethereum-types'; import 'reflect-metadata'; import { createConnection } from 'typeorm'; @@ -14,32 +15,12 @@ const etherscan = new Etherscan(process.env.ETHERSCAN_API_KEY as string); const connection = await createConnection(config); const repository = connection.getRepository(ExchangeFillEvent); console.log(`found ${await repository.count()} existing fill events`); - const eventLogs = await etherscan.getContractEventsAsync( + const events = await etherscan.getContractEventsAsync( '0x4f833a24e1f95d70f028921e27040ca56e09ab0b', artifacts.Exchange.compilerOutput.abi, ); - for (const eventLog of eventLogs) { - if (eventLog.event !== 'Fill') { - continue; - } - const fillEventLog = eventLog as LogWithDecodedArgs; - const exchangeFillEvent = new ExchangeFillEvent(); - exchangeFillEvent.logIndex = fillEventLog.logIndex as number; - exchangeFillEvent.address = fillEventLog.address as string; - exchangeFillEvent.rawData = fillEventLog.data as string; - exchangeFillEvent.blockNumber = fillEventLog.blockNumber as number; - exchangeFillEvent.makerAddress = fillEventLog.args.makerAddress.toString(); - exchangeFillEvent.takerAddress = fillEventLog.args.takerAddress.toString(); - exchangeFillEvent.feeRecepientAddress = fillEventLog.args.feeRecipientAddress; - exchangeFillEvent.senderAddress = fillEventLog.args.senderAddress; - exchangeFillEvent.makerAssetFilledAmount = fillEventLog.args.makerAssetFilledAmount.toString(); - exchangeFillEvent.takerAssetFilledAmount = fillEventLog.args.takerAssetFilledAmount.toString(); - exchangeFillEvent.makerFeePaid = fillEventLog.args.makerFeePaid.toString(); - exchangeFillEvent.takerFeePaid = fillEventLog.args.takerFeePaid.toString(); - exchangeFillEvent.orderHash = fillEventLog.args.orderHash; - exchangeFillEvent.makerAssetData = fillEventLog.args.makerAssetData; - exchangeFillEvent.takerAssetData = fillEventLog.args.takerAssetData; - await repository.save(exchangeFillEvent); + for (const event of events) { + await repository.save(event); } console.log(`now ${await repository.count()} total fill events`); })(); -- cgit v1.2.3