From b797a45d4a67ffd3fa21678e9bcce93e774bfd66 Mon Sep 17 00:00:00 2001 From: Amir Bandeali Date: Sat, 22 Dec 2018 18:11:40 -0800 Subject: Do not attempt to parse tokenAddress if decoding data from multiAssetProxy --- .../pipeline/src/parsers/events/exchange_events.ts | 40 ++++++++++++++-------- packages/pipeline/src/parsers/sra_orders/index.ts | 22 +++++++----- packages/pipeline/src/types.ts | 2 +- .../src/utils/transformers/asset_proxy_id_types.ts | 16 +++++++++ packages/pipeline/src/utils/transformers/index.ts | 1 + 5 files changed, 58 insertions(+), 23 deletions(-) create mode 100644 packages/pipeline/src/utils/transformers/asset_proxy_id_types.ts (limited to 'packages/pipeline') diff --git a/packages/pipeline/src/parsers/events/exchange_events.ts b/packages/pipeline/src/parsers/events/exchange_events.ts index e18106c75..16b05260b 100644 --- a/packages/pipeline/src/parsers/events/exchange_events.ts +++ b/packages/pipeline/src/parsers/events/exchange_events.ts @@ -1,11 +1,11 @@ import { ExchangeCancelEventArgs, ExchangeCancelUpToEventArgs, ExchangeFillEventArgs } from '@0x/contract-wrappers'; import { assetDataUtils } from '@0x/order-utils'; -import { AssetProxyId, ERC721AssetData } from '@0x/types'; +import { AssetProxyId, ERC20AssetData, ERC721AssetData } from '@0x/types'; import { LogWithDecodedArgs } from 'ethereum-types'; import * as R from 'ramda'; import { ExchangeCancelEvent, ExchangeCancelUpToEvent, ExchangeFillEvent } from '../../entities'; -import { bigNumbertoStringOrNull } from '../../utils'; +import { bigNumbertoStringOrNull, convertAssetProxyIdToType } from '../../utils'; /** * Parses raw event logs for a fill event and returns an array of @@ -40,9 +40,7 @@ export const parseExchangeCancelUpToEvents: ( */ export function _convertToExchangeFillEvent(eventLog: LogWithDecodedArgs): ExchangeFillEvent { const makerAssetData = assetDataUtils.decodeAssetDataOrThrow(eventLog.args.makerAssetData); - const makerAssetType = makerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721'; const takerAssetData = assetDataUtils.decodeAssetDataOrThrow(eventLog.args.takerAssetData); - const takerAssetType = takerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721'; const exchangeFillEvent = new ExchangeFillEvent(); exchangeFillEvent.contractAddress = eventLog.address as string; exchangeFillEvent.blockNumber = eventLog.blockNumber as number; @@ -59,16 +57,24 @@ export function _convertToExchangeFillEvent(eventLog: LogWithDecodedArgs, ): ExchangeCancelEvent { const makerAssetData = assetDataUtils.decodeAssetDataOrThrow(eventLog.args.makerAssetData); - const makerAssetType = makerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721'; const takerAssetData = assetDataUtils.decodeAssetDataOrThrow(eventLog.args.takerAssetData); - const takerAssetType = takerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721'; const exchangeCancelEvent = new ExchangeCancelEvent(); exchangeCancelEvent.contractAddress = eventLog.address as string; exchangeCancelEvent.blockNumber = eventLog.blockNumber as number; @@ -98,15 +102,23 @@ export function _convertToExchangeCancelEvent( exchangeCancelEvent.senderAddress = eventLog.args.senderAddress; exchangeCancelEvent.orderHash = eventLog.args.orderHash; exchangeCancelEvent.rawMakerAssetData = eventLog.args.makerAssetData; - exchangeCancelEvent.makerAssetType = makerAssetType; + exchangeCancelEvent.makerAssetType = convertAssetProxyIdToType(makerAssetData.assetProxyId as AssetProxyId); exchangeCancelEvent.makerAssetProxyId = makerAssetData.assetProxyId; - exchangeCancelEvent.makerTokenAddress = makerAssetData.tokenAddress; + // HACK(abandeali1): this event schema currently does not support multiple maker/taker assets, so we store a null byte array when decoding assetData from the MultiAssetProxy + exchangeCancelEvent.makerTokenAddress = + makerAssetData.assetProxyId === AssetProxyId.MultiAsset + ? '0x' + : (makerAssetData as ERC20AssetData | ERC721AssetData).tokenAddress; // tslint:disable-next-line:no-unnecessary-type-assertion exchangeCancelEvent.makerTokenId = bigNumbertoStringOrNull((makerAssetData as ERC721AssetData).tokenId); exchangeCancelEvent.rawTakerAssetData = eventLog.args.takerAssetData; - exchangeCancelEvent.takerAssetType = takerAssetType; + exchangeCancelEvent.takerAssetType = convertAssetProxyIdToType(takerAssetData.assetProxyId as AssetProxyId); exchangeCancelEvent.takerAssetProxyId = takerAssetData.assetProxyId; - exchangeCancelEvent.takerTokenAddress = takerAssetData.tokenAddress; + // HACK(abandeali1): this event schema currently does not support multiple maker/taker assets, so we store a null byte array when decoding assetData from the MultiAssetProxy + exchangeCancelEvent.takerTokenAddress = + makerAssetData.assetProxyId === AssetProxyId.MultiAsset + ? '0x' + : (takerAssetData as ERC20AssetData | ERC721AssetData).tokenAddress; // tslint:disable-next-line:no-unnecessary-type-assertion exchangeCancelEvent.takerTokenId = bigNumbertoStringOrNull((takerAssetData as ERC721AssetData).tokenId); return exchangeCancelEvent; diff --git a/packages/pipeline/src/parsers/sra_orders/index.ts b/packages/pipeline/src/parsers/sra_orders/index.ts index ef8901e40..2d3e5381d 100644 --- a/packages/pipeline/src/parsers/sra_orders/index.ts +++ b/packages/pipeline/src/parsers/sra_orders/index.ts @@ -1,10 +1,10 @@ import { APIOrder, OrdersResponse } from '@0x/connect'; import { assetDataUtils, orderHashUtils } from '@0x/order-utils'; -import { AssetProxyId, ERC721AssetData } from '@0x/types'; +import { AssetProxyId, ERC20AssetData, ERC721AssetData } from '@0x/types'; import * as R from 'ramda'; import { SraOrder } from '../../entities'; -import { bigNumbertoStringOrNull } from '../../utils'; +import { bigNumbertoStringOrNull, convertAssetProxyIdToType } from '../../utils'; /** * Parses a raw order response from an SRA endpoint and returns an array of @@ -22,9 +22,7 @@ export function parseSraOrders(rawOrdersResponse: OrdersResponse): SraOrder[] { export function _convertToEntity(apiOrder: APIOrder): SraOrder { // TODO(albrow): refactor out common asset data decoding code. const makerAssetData = assetDataUtils.decodeAssetDataOrThrow(apiOrder.order.makerAssetData); - const makerAssetType = makerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721'; const takerAssetData = assetDataUtils.decodeAssetDataOrThrow(apiOrder.order.takerAssetData); - const takerAssetType = takerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721'; const sraOrder = new SraOrder(); sraOrder.exchangeAddress = apiOrder.order.exchangeAddress; @@ -43,16 +41,24 @@ export function _convertToEntity(apiOrder: APIOrder): SraOrder { sraOrder.signature = apiOrder.order.signature; sraOrder.rawMakerAssetData = apiOrder.order.makerAssetData; - sraOrder.makerAssetType = makerAssetType; + sraOrder.makerAssetType = convertAssetProxyIdToType(makerAssetData.assetProxyId as AssetProxyId); sraOrder.makerAssetProxyId = makerAssetData.assetProxyId; - sraOrder.makerTokenAddress = makerAssetData.tokenAddress; + // HACK(abandeali1): this event schema currently does not support multiple maker/taker assets, so we store a null byte array when decoding assetData from the MultiAssetProxy + sraOrder.makerTokenAddress = + makerAssetData.assetProxyId === AssetProxyId.MultiAsset + ? '0x' + : (makerAssetData as ERC20AssetData | ERC721AssetData).tokenAddress; // tslint has a false positive here. Type assertion is required. // tslint:disable-next-line:no-unnecessary-type-assertion sraOrder.makerTokenId = bigNumbertoStringOrNull((makerAssetData as ERC721AssetData).tokenId); sraOrder.rawTakerAssetData = apiOrder.order.takerAssetData; - sraOrder.takerAssetType = takerAssetType; + sraOrder.takerAssetType = convertAssetProxyIdToType(takerAssetData.assetProxyId as AssetProxyId); sraOrder.takerAssetProxyId = takerAssetData.assetProxyId; - sraOrder.takerTokenAddress = takerAssetData.tokenAddress; + // HACK(abandeali1): this event schema currently does not support multiple maker/taker assets, so we store a null byte array when decoding assetData from the MultiAssetProxy + sraOrder.takerTokenAddress = + makerAssetData.assetProxyId === AssetProxyId.MultiAsset + ? '0x' + : (takerAssetData as ERC20AssetData | ERC721AssetData).tokenAddress; // tslint:disable-next-line:no-unnecessary-type-assertion sraOrder.takerTokenId = bigNumbertoStringOrNull((takerAssetData as ERC721AssetData).tokenId); diff --git a/packages/pipeline/src/types.ts b/packages/pipeline/src/types.ts index e02b42a40..cd649426b 100644 --- a/packages/pipeline/src/types.ts +++ b/packages/pipeline/src/types.ts @@ -1,2 +1,2 @@ -export type AssetType = 'erc20' | 'erc721'; +export type AssetType = 'erc20' | 'erc721' | 'multiAsset'; export type OrderType = 'bid' | 'ask'; diff --git a/packages/pipeline/src/utils/transformers/asset_proxy_id_types.ts b/packages/pipeline/src/utils/transformers/asset_proxy_id_types.ts new file mode 100644 index 000000000..bc83ad283 --- /dev/null +++ b/packages/pipeline/src/utils/transformers/asset_proxy_id_types.ts @@ -0,0 +1,16 @@ +import { AssetProxyId } from '@0x/types'; + +import { AssetType } from '../../types'; + +export function convertAssetProxyIdToType(assetProxyId: AssetProxyId): AssetType { + switch (assetProxyId) { + case AssetProxyId.ERC20: + return 'erc20'; + case AssetProxyId.ERC721: + return 'erc721'; + case AssetProxyId.MultiAsset: + return 'multiAsset'; + default: + throw new Error(`${assetProxyId} not a supported assetProxyId`); + } +} diff --git a/packages/pipeline/src/utils/transformers/index.ts b/packages/pipeline/src/utils/transformers/index.ts index 232c1c5de..31a4c9223 100644 --- a/packages/pipeline/src/utils/transformers/index.ts +++ b/packages/pipeline/src/utils/transformers/index.ts @@ -1,2 +1,3 @@ export * from './big_number'; export * from './number_to_bigint'; +export * from './asset_proxy_id_types'; -- cgit v1.2.3 From 1dda10b4f34eb19b731e58c26f13ecc08e367bbe Mon Sep 17 00:00:00 2001 From: Amir Bandeali Date: Wed, 2 Jan 2019 16:52:15 -0800 Subject: Add recursive MultiAsset decoding, store first address in pipeline --- packages/pipeline/package.json | 2 +- .../pipeline/src/parsers/events/exchange_events.ts | 26 +++++++++++++--------- packages/pipeline/src/parsers/sra_orders/index.ts | 14 +++++++----- 3 files changed, 24 insertions(+), 18 deletions(-) (limited to 'packages/pipeline') diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index a40f3d21c..ab73642ec 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -44,7 +44,7 @@ "@0x/contract-artifacts": "^1.0.1", "@0x/contract-wrappers": "^3.0.0", "@0x/dev-utils": "^1.0.21", - "@0x/order-utils": "^2.0.0", + "@0x/order-utils": "^3.0.7", "@0x/subproviders": "^2.1.8", "@0x/types": "^1.4.1", "@0x/utils": "^2.0.8", diff --git a/packages/pipeline/src/parsers/events/exchange_events.ts b/packages/pipeline/src/parsers/events/exchange_events.ts index 16b05260b..50fad7e00 100644 --- a/packages/pipeline/src/parsers/events/exchange_events.ts +++ b/packages/pipeline/src/parsers/events/exchange_events.ts @@ -1,6 +1,6 @@ import { ExchangeCancelEventArgs, ExchangeCancelUpToEventArgs, ExchangeFillEventArgs } from '@0x/contract-wrappers'; import { assetDataUtils } from '@0x/order-utils'; -import { AssetProxyId, ERC20AssetData, ERC721AssetData } from '@0x/types'; +import { AssetProxyId, ERC721AssetData, SingleAssetData } from '@0x/types'; import { LogWithDecodedArgs } from 'ethereum-types'; import * as R from 'ramda'; @@ -62,8 +62,9 @@ export function _convertToExchangeFillEvent(eventLog: LogWithDecodedArgs Date: Wed, 2 Jan 2019 17:05:32 -0800 Subject: FIx linting errors --- packages/pipeline/src/parsers/events/exchange_events.ts | 2 ++ packages/pipeline/src/parsers/sra_orders/index.ts | 2 ++ packages/pipeline/src/utils/transformers/asset_proxy_id_types.ts | 4 ++++ 3 files changed, 8 insertions(+) (limited to 'packages/pipeline') diff --git a/packages/pipeline/src/parsers/events/exchange_events.ts b/packages/pipeline/src/parsers/events/exchange_events.ts index 50fad7e00..8d2e7dec7 100644 --- a/packages/pipeline/src/parsers/events/exchange_events.ts +++ b/packages/pipeline/src/parsers/events/exchange_events.ts @@ -7,6 +7,8 @@ import * as R from 'ramda'; import { ExchangeCancelEvent, ExchangeCancelUpToEvent, ExchangeFillEvent } from '../../entities'; import { bigNumbertoStringOrNull, convertAssetProxyIdToType } from '../../utils'; +// tslint:disable:no-unnecessary-type-assertion + /** * Parses raw event logs for a fill event and returns an array of * ExchangeFillEvent entities. diff --git a/packages/pipeline/src/parsers/sra_orders/index.ts b/packages/pipeline/src/parsers/sra_orders/index.ts index 663b8f509..10546921d 100644 --- a/packages/pipeline/src/parsers/sra_orders/index.ts +++ b/packages/pipeline/src/parsers/sra_orders/index.ts @@ -6,6 +6,8 @@ import * as R from 'ramda'; import { SraOrder } from '../../entities'; import { bigNumbertoStringOrNull, convertAssetProxyIdToType } from '../../utils'; +// tslint:disable:no-unnecessary-type-assertion + /** * Parses a raw order response from an SRA endpoint and returns an array of * SraOrder entities. diff --git a/packages/pipeline/src/utils/transformers/asset_proxy_id_types.ts b/packages/pipeline/src/utils/transformers/asset_proxy_id_types.ts index bc83ad283..ffeaaca52 100644 --- a/packages/pipeline/src/utils/transformers/asset_proxy_id_types.ts +++ b/packages/pipeline/src/utils/transformers/asset_proxy_id_types.ts @@ -2,6 +2,10 @@ import { AssetProxyId } from '@0x/types'; import { AssetType } from '../../types'; +/** + * Converts an assetProxyId to its string equivalent + * @param assetProxyId Id of AssetProxy + */ export function convertAssetProxyIdToType(assetProxyId: AssetProxyId): AssetType { switch (assetProxyId) { case AssetProxyId.ERC20: -- cgit v1.2.3 From 24564b986daa703f66e54f85abf4782d99a40f94 Mon Sep 17 00:00:00 2001 From: Amir Bandeali Date: Fri, 4 Jan 2019 14:31:25 -0800 Subject: Minimize unnecessary type assertions --- .../pipeline/src/parsers/events/exchange_events.ts | 48 ++++++++++------------ packages/pipeline/src/parsers/sra_orders/index.ts | 26 +++++------- 2 files changed, 32 insertions(+), 42 deletions(-) (limited to 'packages/pipeline') diff --git a/packages/pipeline/src/parsers/events/exchange_events.ts b/packages/pipeline/src/parsers/events/exchange_events.ts index 8d2e7dec7..9c4a5f89a 100644 --- a/packages/pipeline/src/parsers/events/exchange_events.ts +++ b/packages/pipeline/src/parsers/events/exchange_events.ts @@ -1,14 +1,12 @@ import { ExchangeCancelEventArgs, ExchangeCancelUpToEventArgs, ExchangeFillEventArgs } from '@0x/contract-wrappers'; import { assetDataUtils } from '@0x/order-utils'; -import { AssetProxyId, ERC721AssetData, SingleAssetData } from '@0x/types'; +import { AssetProxyId, ERC721AssetData } from '@0x/types'; import { LogWithDecodedArgs } from 'ethereum-types'; import * as R from 'ramda'; import { ExchangeCancelEvent, ExchangeCancelUpToEvent, ExchangeFillEvent } from '../../entities'; import { bigNumbertoStringOrNull, convertAssetProxyIdToType } from '../../utils'; -// tslint:disable:no-unnecessary-type-assertion - /** * Parses raw event logs for a fill event and returns an array of * ExchangeFillEvent entities. @@ -59,26 +57,24 @@ export function _convertToExchangeFillEvent(eventLog: LogWithDecodedArgs Date: Mon, 7 Jan 2019 11:11:19 -0800 Subject: Use enums in pipeline --- packages/pipeline/src/entities/token_order.ts | 3 +-- packages/pipeline/src/parsers/ddex_orders/index.ts | 8 ++++++-- packages/pipeline/src/parsers/idex_orders/index.ts | 12 ++++++++---- packages/pipeline/src/parsers/oasis_orders/index.ts | 8 ++++---- packages/pipeline/src/parsers/paradex_orders/index.ts | 4 ++-- packages/pipeline/src/types.ts | 11 +++++++++-- .../pipeline/src/utils/transformers/asset_proxy_id_types.ts | 6 +++--- packages/pipeline/test/parsers/ddex_orders/index_test.ts | 4 ++-- .../pipeline/test/parsers/events/exchange_events_test.ts | 5 +++-- packages/pipeline/test/parsers/idex_orders/index_test.ts | 8 ++++---- packages/pipeline/test/parsers/oasis_orders/index_test.ts | 4 ++-- packages/pipeline/test/parsers/paradex_orders/index_test.ts | 4 ++-- packages/pipeline/test/parsers/sra_orders/index_test.ts | 5 +++-- 13 files changed, 49 insertions(+), 33 deletions(-) (limited to 'packages/pipeline') diff --git a/packages/pipeline/src/entities/token_order.ts b/packages/pipeline/src/entities/token_order.ts index 4b8f0abc3..2709747cb 100644 --- a/packages/pipeline/src/entities/token_order.ts +++ b/packages/pipeline/src/entities/token_order.ts @@ -1,7 +1,6 @@ import { BigNumber } from '@0x/utils'; import { Column, Entity, PrimaryColumn } from 'typeorm'; -import { OrderType } from '../types'; import { bigNumberTransformer, numberToBigIntTransformer } from '../utils'; @Entity({ name: 'token_orderbook_snapshots', schema: 'raw' }) @@ -11,7 +10,7 @@ export class TokenOrderbookSnapshot { @PrimaryColumn({ name: 'source' }) public source!: string; @PrimaryColumn({ name: 'order_type' }) - public orderType!: OrderType; + public orderType!: string; @PrimaryColumn({ name: 'price', type: 'numeric', transformer: bigNumberTransformer }) public price!: BigNumber; @PrimaryColumn({ name: 'base_asset_symbol' }) diff --git a/packages/pipeline/src/parsers/ddex_orders/index.ts b/packages/pipeline/src/parsers/ddex_orders/index.ts index d7b97efbe..eeb9c9d5b 100644 --- a/packages/pipeline/src/parsers/ddex_orders/index.ts +++ b/packages/pipeline/src/parsers/ddex_orders/index.ts @@ -23,8 +23,12 @@ export function parseDdexOrders( ): TokenOrder[] { const aggregatedBids = aggregateOrders(ddexOrderbook.bids); const aggregatedAsks = aggregateOrders(ddexOrderbook.asks); - const parsedBids = aggregatedBids.map(order => parseDdexOrder(ddexMarket, observedTimestamp, 'bid', source, order)); - const parsedAsks = aggregatedAsks.map(order => parseDdexOrder(ddexMarket, observedTimestamp, 'ask', source, order)); + const parsedBids = aggregatedBids.map(order => + parseDdexOrder(ddexMarket, observedTimestamp, OrderType.Bid, source, order), + ); + const parsedAsks = aggregatedAsks.map(order => + parseDdexOrder(ddexMarket, observedTimestamp, OrderType.Ask, source, order), + ); return parsedBids.concat(parsedAsks); } diff --git a/packages/pipeline/src/parsers/idex_orders/index.ts b/packages/pipeline/src/parsers/idex_orders/index.ts index dfe27455c..14b871195 100644 --- a/packages/pipeline/src/parsers/idex_orders/index.ts +++ b/packages/pipeline/src/parsers/idex_orders/index.ts @@ -2,7 +2,7 @@ import { BigNumber } from '@0x/utils'; import { aggregateOrders } from '../utils'; -import { IdexOrder, IdexOrderbook, IdexOrderParam } from '../../data_sources/idex'; +import { IdexOrderbook, IdexOrderParam } from '../../data_sources/idex'; import { TokenOrderbookSnapshot as TokenOrder } from '../../entities'; import { OrderType } from '../../types'; @@ -21,7 +21,9 @@ export function parseIdexOrders(idexOrderbook: IdexOrderbook, observedTimestamp: const idexBidOrder = idexOrderbook.bids[0]; const parsedBids = aggregatedBids.length > 0 - ? aggregatedBids.map(order => parseIdexOrder(idexBidOrder.params, observedTimestamp, 'bid', source, order)) + ? aggregatedBids.map(order => + parseIdexOrder(idexBidOrder.params, observedTimestamp, OrderType.Bid, source, order), + ) : []; const aggregatedAsks = aggregateOrders(idexOrderbook.asks); @@ -29,7 +31,9 @@ export function parseIdexOrders(idexOrderbook: IdexOrderbook, observedTimestamp: const idexAskOrder = idexOrderbook.asks[0]; const parsedAsks = aggregatedAsks.length > 0 - ? aggregatedAsks.map(order => parseIdexOrder(idexAskOrder.params, observedTimestamp, 'ask', source, order)) + ? aggregatedAsks.map(order => + parseIdexOrder(idexAskOrder.params, observedTimestamp, OrderType.Ask, source, order), + ) : []; return parsedBids.concat(parsedAsks); } @@ -62,7 +66,7 @@ export function parseIdexOrder( tokenOrder.baseVolume = amount; tokenOrder.quoteVolume = price.times(amount); - if (orderType === 'bid') { + if (orderType === OrderType.Bid) { tokenOrder.baseAssetSymbol = idexOrderParam.buySymbol; tokenOrder.baseAssetAddress = idexOrderParam.tokenBuy; tokenOrder.quoteAssetSymbol = idexOrderParam.sellSymbol; diff --git a/packages/pipeline/src/parsers/oasis_orders/index.ts b/packages/pipeline/src/parsers/oasis_orders/index.ts index 13997f31b..b71fb65b9 100644 --- a/packages/pipeline/src/parsers/oasis_orders/index.ts +++ b/packages/pipeline/src/parsers/oasis_orders/index.ts @@ -23,13 +23,13 @@ export function parseOasisOrders( observedTimestamp: number, source: string, ): TokenOrder[] { - const aggregatedBids = aggregateOrders(R.filter(R.propEq('act', 'bid'), oasisOrderbook)); - const aggregatedAsks = aggregateOrders(R.filter(R.propEq('act', 'ask'), oasisOrderbook)); + const aggregatedBids = aggregateOrders(R.filter(R.propEq('act', OrderType.Bid), oasisOrderbook)); + const aggregatedAsks = aggregateOrders(R.filter(R.propEq('act', OrderType.Ask), oasisOrderbook)); const parsedBids = aggregatedBids.map(order => - parseOasisOrder(oasisMarket, observedTimestamp, 'bid', source, order), + parseOasisOrder(oasisMarket, observedTimestamp, OrderType.Bid, source, order), ); const parsedAsks = aggregatedAsks.map(order => - parseOasisOrder(oasisMarket, observedTimestamp, 'ask', source, order), + parseOasisOrder(oasisMarket, observedTimestamp, OrderType.Ask, source, order), ); return parsedBids.concat(parsedAsks); } diff --git a/packages/pipeline/src/parsers/paradex_orders/index.ts b/packages/pipeline/src/parsers/paradex_orders/index.ts index 5ceeb64a4..85990dae4 100644 --- a/packages/pipeline/src/parsers/paradex_orders/index.ts +++ b/packages/pipeline/src/parsers/paradex_orders/index.ts @@ -21,10 +21,10 @@ export function parseParadexOrders( source: string, ): TokenOrder[] { const parsedBids = paradexOrderbookResponse.bids.map(order => - parseParadexOrder(paradexMarket, observedTimestamp, 'bid', source, order), + parseParadexOrder(paradexMarket, observedTimestamp, OrderType.Bid, source, order), ); const parsedAsks = paradexOrderbookResponse.asks.map(order => - parseParadexOrder(paradexMarket, observedTimestamp, 'ask', source, order), + parseParadexOrder(paradexMarket, observedTimestamp, OrderType.Ask, source, order), ); return parsedBids.concat(parsedAsks); } diff --git a/packages/pipeline/src/types.ts b/packages/pipeline/src/types.ts index cd649426b..5f2121807 100644 --- a/packages/pipeline/src/types.ts +++ b/packages/pipeline/src/types.ts @@ -1,2 +1,9 @@ -export type AssetType = 'erc20' | 'erc721' | 'multiAsset'; -export type OrderType = 'bid' | 'ask'; +export enum AssetType { + ERC20 = 'erc20', + ERC721 = 'erc721', + MultiAsset = 'multiAsset', +} +export enum OrderType { + Bid = 'bid', + Ask = 'ask', +} diff --git a/packages/pipeline/src/utils/transformers/asset_proxy_id_types.ts b/packages/pipeline/src/utils/transformers/asset_proxy_id_types.ts index ffeaaca52..2cd05a616 100644 --- a/packages/pipeline/src/utils/transformers/asset_proxy_id_types.ts +++ b/packages/pipeline/src/utils/transformers/asset_proxy_id_types.ts @@ -9,11 +9,11 @@ import { AssetType } from '../../types'; export function convertAssetProxyIdToType(assetProxyId: AssetProxyId): AssetType { switch (assetProxyId) { case AssetProxyId.ERC20: - return 'erc20'; + return AssetType.ERC20; case AssetProxyId.ERC721: - return 'erc721'; + return AssetType.ERC721; case AssetProxyId.MultiAsset: - return 'multiAsset'; + return AssetType.MultiAsset; default: throw new Error(`${assetProxyId} not a supported assetProxyId`); } diff --git a/packages/pipeline/test/parsers/ddex_orders/index_test.ts b/packages/pipeline/test/parsers/ddex_orders/index_test.ts index 4a4a86bf8..f30e86b02 100644 --- a/packages/pipeline/test/parsers/ddex_orders/index_test.ts +++ b/packages/pipeline/test/parsers/ddex_orders/index_test.ts @@ -31,13 +31,13 @@ describe('ddex_orders', () => { amountDecimals: 0, }; const observedTimestamp: number = Date.now(); - const orderType: OrderType = 'bid'; + const orderType: OrderType = OrderType.Bid; const source: string = 'ddex'; const expected = new TokenOrder(); expected.source = 'ddex'; expected.observedTimestamp = observedTimestamp; - expected.orderType = 'bid'; + expected.orderType = OrderType.Bid; expected.price = new BigNumber(0.5); // ddex currently confuses base and quote assets. // Switch them to maintain our internal consistency. diff --git a/packages/pipeline/test/parsers/events/exchange_events_test.ts b/packages/pipeline/test/parsers/events/exchange_events_test.ts index 5d4b185a5..956ad9ef8 100644 --- a/packages/pipeline/test/parsers/events/exchange_events_test.ts +++ b/packages/pipeline/test/parsers/events/exchange_events_test.ts @@ -6,6 +6,7 @@ import 'mocha'; import { ExchangeFillEvent } from '../../../src/entities'; import { _convertToExchangeFillEvent } from '../../../src/parsers/events/exchange_events'; +import { AssetType } from '../../../src/types'; import { chaiSetup } from '../../utils/chai_setup'; chaiSetup.configure(); @@ -62,12 +63,12 @@ describe('exchange_events', () => { expected.takerFeePaid = new BigNumber('12345'); expected.orderHash = '0xab12ed2cbaa5615ab690b9da75a46e53ddfcf3f1a68655b5fe0d94c75a1aac4a'; expected.rawMakerAssetData = '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2'; - expected.makerAssetType = 'erc20'; + expected.makerAssetType = AssetType.ERC20; expected.makerAssetProxyId = '0xf47261b0'; expected.makerTokenAddress = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2'; expected.makerTokenId = null; expected.rawTakerAssetData = '0xf47261b0000000000000000000000000e41d2489571d322189246dafa5ebde1f4699f498'; - expected.takerAssetType = 'erc20'; + expected.takerAssetType = AssetType.ERC20; expected.takerAssetProxyId = '0xf47261b0'; expected.takerTokenAddress = '0xe41d2489571d322189246dafa5ebde1f4699f498'; expected.takerTokenId = null; diff --git a/packages/pipeline/test/parsers/idex_orders/index_test.ts b/packages/pipeline/test/parsers/idex_orders/index_test.ts index d54ecb9a8..48b019732 100644 --- a/packages/pipeline/test/parsers/idex_orders/index_test.ts +++ b/packages/pipeline/test/parsers/idex_orders/index_test.ts @@ -31,13 +31,13 @@ describe('idex_orders', () => { user: '0x212345667543456435324564345643453453333', }; const observedTimestamp: number = Date.now(); - const orderType: OrderType = 'bid'; + const orderType: OrderType = OrderType.Bid; const source: string = 'idex'; const expected = new TokenOrder(); expected.source = 'idex'; expected.observedTimestamp = observedTimestamp; - expected.orderType = 'bid'; + expected.orderType = OrderType.Bid; expected.price = new BigNumber(0.5); expected.baseAssetSymbol = 'ABC'; expected.baseAssetAddress = '0x0000000000000000000000000000000000000000'; @@ -65,13 +65,13 @@ describe('idex_orders', () => { user: '0x212345667543456435324564345643453453333', }; const observedTimestamp: number = Date.now(); - const orderType: OrderType = 'ask'; + const orderType: OrderType = OrderType.Ask; const source: string = 'idex'; const expected = new TokenOrder(); expected.source = 'idex'; expected.observedTimestamp = observedTimestamp; - expected.orderType = 'ask'; + expected.orderType = OrderType.Ask; expected.price = new BigNumber(0.5); expected.baseAssetSymbol = 'ABC'; expected.baseAssetAddress = '0x0000000000000000000000000000000000000000'; diff --git a/packages/pipeline/test/parsers/oasis_orders/index_test.ts b/packages/pipeline/test/parsers/oasis_orders/index_test.ts index 433bfb665..401fedff8 100644 --- a/packages/pipeline/test/parsers/oasis_orders/index_test.ts +++ b/packages/pipeline/test/parsers/oasis_orders/index_test.ts @@ -27,13 +27,13 @@ describe('oasis_orders', () => { low: 0, }; const observedTimestamp: number = Date.now(); - const orderType: OrderType = 'bid'; + const orderType: OrderType = OrderType.Bid; const source: string = 'oasis'; const expected = new TokenOrder(); expected.source = 'oasis'; expected.observedTimestamp = observedTimestamp; - expected.orderType = 'bid'; + expected.orderType = OrderType.Bid; expected.price = new BigNumber(0.5); expected.baseAssetSymbol = 'DEF'; expected.baseAssetAddress = null; diff --git a/packages/pipeline/test/parsers/paradex_orders/index_test.ts b/packages/pipeline/test/parsers/paradex_orders/index_test.ts index 6b811b90d..c5dd8751b 100644 --- a/packages/pipeline/test/parsers/paradex_orders/index_test.ts +++ b/packages/pipeline/test/parsers/paradex_orders/index_test.ts @@ -32,13 +32,13 @@ describe('paradex_orders', () => { quoteTokenAddress: '0x0000000000000000000000000000000000000000', }; const observedTimestamp: number = Date.now(); - const orderType: OrderType = 'bid'; + const orderType: OrderType = OrderType.Bid; const source: string = 'paradex'; const expected = new TokenOrder(); expected.source = 'paradex'; expected.observedTimestamp = observedTimestamp; - expected.orderType = 'bid'; + expected.orderType = OrderType.Bid; expected.price = new BigNumber(0.1245); expected.baseAssetSymbol = 'DEF'; expected.baseAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81'; diff --git a/packages/pipeline/test/parsers/sra_orders/index_test.ts b/packages/pipeline/test/parsers/sra_orders/index_test.ts index ee2842ef3..838171a72 100644 --- a/packages/pipeline/test/parsers/sra_orders/index_test.ts +++ b/packages/pipeline/test/parsers/sra_orders/index_test.ts @@ -5,6 +5,7 @@ import 'mocha'; import { SraOrder } from '../../../src/entities'; import { _convertToEntity } from '../../../src/parsers/sra_orders'; +import { AssetType } from '../../../src/types'; import { chaiSetup } from '../../utils/chai_setup'; chaiSetup.configure(); @@ -50,12 +51,12 @@ describe('sra_orders', () => { expected.signature = '0x1b5a5d672b0d647b5797387ccbb89d822d5d2e873346b014f4ff816ff0783f2a7a0d2824d2d7042ec8ea375bc7f870963e1cb8248f1db03ddf125e27b5963aa11f03'; expected.rawMakerAssetData = '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2'; - expected.makerAssetType = 'erc20'; + expected.makerAssetType = AssetType.ERC20; expected.makerAssetProxyId = '0xf47261b0'; expected.makerTokenAddress = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2'; expected.makerTokenId = null; expected.rawTakerAssetData = '0xf47261b000000000000000000000000042d6622dece394b54999fbd73d108123806f6a18'; - expected.takerAssetType = 'erc20'; + expected.takerAssetType = AssetType.ERC20; expected.takerAssetProxyId = '0xf47261b0'; expected.takerTokenAddress = '0x42d6622dece394b54999fbd73d108123806f6a18'; expected.takerTokenId = null; -- cgit v1.2.3 From 7dda953bc929e218121c331fedb3884b24555855 Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Mon, 7 Jan 2019 15:04:00 -0800 Subject: Optimize SQL queries in pull_missing_blocks (#1458) * Optimize SQL queries in pull_missing_blocks * Update comment in pull_missing_blocks --- .../pipeline/src/scripts/pull_missing_blocks.ts | 39 +++++++++++----------- 1 file changed, 20 insertions(+), 19 deletions(-) (limited to 'packages/pipeline') diff --git a/packages/pipeline/src/scripts/pull_missing_blocks.ts b/packages/pipeline/src/scripts/pull_missing_blocks.ts index bb5385126..ced9d99eb 100644 --- a/packages/pipeline/src/scripts/pull_missing_blocks.ts +++ b/packages/pipeline/src/scripts/pull_missing_blocks.ts @@ -14,20 +14,29 @@ import { handleError, INFURA_ROOT_URL } from '../utils'; // Number of blocks to save at once. const BATCH_SAVE_SIZE = 1000; // Maximum number of requests to send at once. -const MAX_CONCURRENCY = 10; +const MAX_CONCURRENCY = 20; // Maximum number of blocks to query for at once. This is also the maximum // number of blocks we will hold in memory prior to being saved to the database. const MAX_BLOCKS_PER_QUERY = 1000; let connection: Connection; +const tablesWithMissingBlocks = [ + 'raw.exchange_fill_events', + 'raw.exchange_cancel_events', + 'raw.exchange_cancel_up_to_events', + 'raw.erc20_approval_events', +]; + (async () => { connection = await createConnection(ormConfig as ConnectionOptions); const provider = web3Factory.getRpcProvider({ rpcUrl: INFURA_ROOT_URL, }); const web3Source = new Web3Source(provider); - await getAllMissingBlocksAsync(web3Source); + for (const tableName of tablesWithMissingBlocks) { + await getAllMissingBlocksAsync(web3Source, tableName); + } process.exit(0); })().catch(handleError); @@ -35,10 +44,11 @@ interface MissingBlocksResponse { block_number: string; } -async function getAllMissingBlocksAsync(web3Source: Web3Source): Promise { +async function getAllMissingBlocksAsync(web3Source: Web3Source, tableName: string): Promise { const blocksRepository = connection.getRepository(Block); while (true) { - const blockNumbers = await getMissingBlockNumbersAsync(); + console.log(`Checking for missing blocks in ${tableName}...`); + const blockNumbers = await getMissingBlockNumbersAsync(tableName); if (blockNumbers.length === 0) { // There are no more missing blocks. We're done. break; @@ -46,24 +56,14 @@ async function getAllMissingBlocksAsync(web3Source: Web3Source): Promise { await getAndSaveBlocksAsync(web3Source, blocksRepository, blockNumbers); } const totalBlocks = await blocksRepository.count(); - console.log(`Done saving blocks. There are now ${totalBlocks} total blocks.`); + console.log(`Done saving blocks for ${tableName}. There are now ${totalBlocks} total blocks.`); } -async function getMissingBlockNumbersAsync(): Promise { - // Note(albrow): The easiest way to get all the blocks we need is to - // consider all the events tables together in a single query. If this query - // gets too slow, we should consider re-architecting so that we can work on - // getting the blocks for one type of event at a time. +async function getMissingBlockNumbersAsync(tableName: string): Promise { + // This query returns up to `MAX_BLOCKS_PER_QUERY` distinct block numbers + // which are present in `tableName` but not in `raw.blocks`. const response = (await connection.query( - `WITH all_events AS ( - SELECT block_number FROM raw.exchange_fill_events - UNION SELECT block_number FROM raw.exchange_cancel_events - UNION SELECT block_number FROM raw.exchange_cancel_up_to_events - UNION SELECT block_number FROM raw.erc20_approval_events - ) - SELECT DISTINCT(block_number) FROM all_events - WHERE block_number NOT IN (SELECT number FROM raw.blocks) - ORDER BY block_number ASC LIMIT $1`, + `SELECT DISTINCT(block_number) FROM ${tableName} LEFT JOIN raw.blocks ON ${tableName}.block_number = raw.blocks.number WHERE number IS NULL LIMIT $1;`, [MAX_BLOCKS_PER_QUERY], )) as MissingBlocksResponse[]; const blockNumberStrings = R.pluck('block_number', response); @@ -86,4 +86,5 @@ async function getAndSaveBlocksAsync( const blocks = R.map(parseBlock, rawBlocks); console.log(`Saving ${blocks.length} blocks...`); await blocksRepository.save(blocks, { chunk: Math.ceil(blocks.length / BATCH_SAVE_SIZE) }); + console.log('Done saving this batch of blocks'); } -- cgit v1.2.3