aboutsummaryrefslogtreecommitdiffstats
path: root/packages/pipeline/src
diff options
context:
space:
mode:
authorFabio Berger <me@fabioberger.com>2019-01-08 21:30:38 +0800
committerFabio Berger <me@fabioberger.com>2019-01-08 21:30:38 +0800
commit1631031fa74894143cb6835030b7dcd44d7c3c6b (patch)
tree06dea01cc64fb42905a5f95c95f4b3e16ecfe744 /packages/pipeline/src
parent0bcb81d3a918fbcf71d68f42fa661d884d5d74cf (diff)
parent0ac36cef288deecd36caa601c53d13517eef5ca8 (diff)
downloaddexon-sol-tools-1631031fa74894143cb6835030b7dcd44d7c3c6b.tar
dexon-sol-tools-1631031fa74894143cb6835030b7dcd44d7c3c6b.tar.gz
dexon-sol-tools-1631031fa74894143cb6835030b7dcd44d7c3c6b.tar.bz2
dexon-sol-tools-1631031fa74894143cb6835030b7dcd44d7c3c6b.tar.lz
dexon-sol-tools-1631031fa74894143cb6835030b7dcd44d7c3c6b.tar.xz
dexon-sol-tools-1631031fa74894143cb6835030b7dcd44d7c3c6b.tar.zst
dexon-sol-tools-1631031fa74894143cb6835030b7dcd44d7c3c6b.zip
Merge branch 'development' into feature/order-watcher/dockerize
* development: (898 commits) Fixed merge conflict from development Ran prettier Doc generation working for changes by dutch auction wrapper added changelog entry for monorepo-scripts Hide dutch auction wrapper from docs -- hopefully this will prevent the "must export Web3Wrapper" error from doc generation relaxed version on contract-extension dependencies Added NetworkID 50 address for dutch auction wrapper removed manual updte of package.json version export dutch auction wrapper types from 0x.js Export dutch auction wrapper in 0x.js ran prettier Minor documentation updates to dutch auction wrapper `afterAuctionDetails` -> `auctionDetails` Added @todo for including dutch auction addresses once deployed Ran prettier & linter Removed redundant assignment removed needless newline on contract-wrappers changelog removed timestamp from changelog for abi-gen-wrappers added dutch auction address for testnets removed .only ...
Diffstat (limited to 'packages/pipeline/src')
-rw-r--r--packages/pipeline/src/entities/token_order.ts3
-rw-r--r--packages/pipeline/src/parsers/ddex_orders/index.ts8
-rw-r--r--packages/pipeline/src/parsers/events/exchange_events.ts38
-rw-r--r--packages/pipeline/src/parsers/idex_orders/index.ts12
-rw-r--r--packages/pipeline/src/parsers/oasis_orders/index.ts8
-rw-r--r--packages/pipeline/src/parsers/paradex_orders/index.ts4
-rw-r--r--packages/pipeline/src/parsers/sra_orders/index.ts20
-rw-r--r--packages/pipeline/src/scripts/pull_missing_blocks.ts39
-rw-r--r--packages/pipeline/src/types.ts11
-rw-r--r--packages/pipeline/src/utils/transformers/asset_proxy_id_types.ts20
-rw-r--r--packages/pipeline/src/utils/transformers/index.ts1
11 files changed, 109 insertions, 55 deletions
diff --git a/packages/pipeline/src/entities/token_order.ts b/packages/pipeline/src/entities/token_order.ts
index 4b8f0abc3..2709747cb 100644
--- a/packages/pipeline/src/entities/token_order.ts
+++ b/packages/pipeline/src/entities/token_order.ts
@@ -1,7 +1,6 @@
import { BigNumber } from '@0x/utils';
import { Column, Entity, PrimaryColumn } from 'typeorm';
-import { OrderType } from '../types';
import { bigNumberTransformer, numberToBigIntTransformer } from '../utils';
@Entity({ name: 'token_orderbook_snapshots', schema: 'raw' })
@@ -11,7 +10,7 @@ export class TokenOrderbookSnapshot {
@PrimaryColumn({ name: 'source' })
public source!: string;
@PrimaryColumn({ name: 'order_type' })
- public orderType!: OrderType;
+ public orderType!: string;
@PrimaryColumn({ name: 'price', type: 'numeric', transformer: bigNumberTransformer })
public price!: BigNumber;
@PrimaryColumn({ name: 'base_asset_symbol' })
diff --git a/packages/pipeline/src/parsers/ddex_orders/index.ts b/packages/pipeline/src/parsers/ddex_orders/index.ts
index d7b97efbe..eeb9c9d5b 100644
--- a/packages/pipeline/src/parsers/ddex_orders/index.ts
+++ b/packages/pipeline/src/parsers/ddex_orders/index.ts
@@ -23,8 +23,12 @@ export function parseDdexOrders(
): TokenOrder[] {
const aggregatedBids = aggregateOrders(ddexOrderbook.bids);
const aggregatedAsks = aggregateOrders(ddexOrderbook.asks);
- const parsedBids = aggregatedBids.map(order => parseDdexOrder(ddexMarket, observedTimestamp, 'bid', source, order));
- const parsedAsks = aggregatedAsks.map(order => parseDdexOrder(ddexMarket, observedTimestamp, 'ask', source, order));
+ const parsedBids = aggregatedBids.map(order =>
+ parseDdexOrder(ddexMarket, observedTimestamp, OrderType.Bid, source, order),
+ );
+ const parsedAsks = aggregatedAsks.map(order =>
+ parseDdexOrder(ddexMarket, observedTimestamp, OrderType.Ask, source, order),
+ );
return parsedBids.concat(parsedAsks);
}
diff --git a/packages/pipeline/src/parsers/events/exchange_events.ts b/packages/pipeline/src/parsers/events/exchange_events.ts
index e18106c75..9c4a5f89a 100644
--- a/packages/pipeline/src/parsers/events/exchange_events.ts
+++ b/packages/pipeline/src/parsers/events/exchange_events.ts
@@ -5,7 +5,7 @@ import { LogWithDecodedArgs } from 'ethereum-types';
import * as R from 'ramda';
import { ExchangeCancelEvent, ExchangeCancelUpToEvent, ExchangeFillEvent } from '../../entities';
-import { bigNumbertoStringOrNull } from '../../utils';
+import { bigNumbertoStringOrNull, convertAssetProxyIdToType } from '../../utils';
/**
* Parses raw event logs for a fill event and returns an array of
@@ -40,9 +40,7 @@ export const parseExchangeCancelUpToEvents: (
*/
export function _convertToExchangeFillEvent(eventLog: LogWithDecodedArgs<ExchangeFillEventArgs>): ExchangeFillEvent {
const makerAssetData = assetDataUtils.decodeAssetDataOrThrow(eventLog.args.makerAssetData);
- const makerAssetType = makerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721';
const takerAssetData = assetDataUtils.decodeAssetDataOrThrow(eventLog.args.takerAssetData);
- const takerAssetType = takerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721';
const exchangeFillEvent = new ExchangeFillEvent();
exchangeFillEvent.contractAddress = eventLog.address as string;
exchangeFillEvent.blockNumber = eventLog.blockNumber as number;
@@ -59,16 +57,24 @@ export function _convertToExchangeFillEvent(eventLog: LogWithDecodedArgs<Exchang
exchangeFillEvent.takerFeePaid = eventLog.args.takerFeePaid;
exchangeFillEvent.orderHash = eventLog.args.orderHash;
exchangeFillEvent.rawMakerAssetData = eventLog.args.makerAssetData;
- exchangeFillEvent.makerAssetType = makerAssetType;
+ // tslint:disable-next-line:no-unnecessary-type-assertion
+ exchangeFillEvent.makerAssetType = convertAssetProxyIdToType(makerAssetData.assetProxyId as AssetProxyId);
exchangeFillEvent.makerAssetProxyId = makerAssetData.assetProxyId;
- exchangeFillEvent.makerTokenAddress = makerAssetData.tokenAddress;
+ // HACK(abandeali1): this event schema currently does not support multiple maker/taker assets, so we store the first token address from the MultiAssetProxy assetData
+ exchangeFillEvent.makerTokenAddress = assetDataUtils.isMultiAssetData(makerAssetData)
+ ? assetDataUtils.decodeMultiAssetDataRecursively(eventLog.args.makerAssetData).nestedAssetData[0].tokenAddress
+ : makerAssetData.tokenAddress;
// tslint has a false positive here. Type assertion is required.
// tslint:disable-next-line:no-unnecessary-type-assertion
exchangeFillEvent.makerTokenId = bigNumbertoStringOrNull((makerAssetData as ERC721AssetData).tokenId);
exchangeFillEvent.rawTakerAssetData = eventLog.args.takerAssetData;
- exchangeFillEvent.takerAssetType = takerAssetType;
+ // tslint:disable-next-line:no-unnecessary-type-assertion
+ exchangeFillEvent.takerAssetType = convertAssetProxyIdToType(takerAssetData.assetProxyId as AssetProxyId);
exchangeFillEvent.takerAssetProxyId = takerAssetData.assetProxyId;
- exchangeFillEvent.takerTokenAddress = takerAssetData.tokenAddress;
+ // HACK(abandeali1): this event schema currently does not support multiple maker/taker assets, so we store the first token address from the MultiAssetProxy assetData
+ exchangeFillEvent.takerTokenAddress = assetDataUtils.isMultiAssetData(takerAssetData)
+ ? assetDataUtils.decodeMultiAssetDataRecursively(eventLog.args.takerAssetData).nestedAssetData[0].tokenAddress
+ : takerAssetData.tokenAddress;
// tslint:disable-next-line:no-unnecessary-type-assertion
exchangeFillEvent.takerTokenId = bigNumbertoStringOrNull((takerAssetData as ERC721AssetData).tokenId);
return exchangeFillEvent;
@@ -83,9 +89,7 @@ export function _convertToExchangeCancelEvent(
eventLog: LogWithDecodedArgs<ExchangeCancelEventArgs>,
): ExchangeCancelEvent {
const makerAssetData = assetDataUtils.decodeAssetDataOrThrow(eventLog.args.makerAssetData);
- const makerAssetType = makerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721';
const takerAssetData = assetDataUtils.decodeAssetDataOrThrow(eventLog.args.takerAssetData);
- const takerAssetType = takerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721';
const exchangeCancelEvent = new ExchangeCancelEvent();
exchangeCancelEvent.contractAddress = eventLog.address as string;
exchangeCancelEvent.blockNumber = eventLog.blockNumber as number;
@@ -98,15 +102,23 @@ export function _convertToExchangeCancelEvent(
exchangeCancelEvent.senderAddress = eventLog.args.senderAddress;
exchangeCancelEvent.orderHash = eventLog.args.orderHash;
exchangeCancelEvent.rawMakerAssetData = eventLog.args.makerAssetData;
- exchangeCancelEvent.makerAssetType = makerAssetType;
+ // tslint:disable-next-line:no-unnecessary-type-assertion
+ exchangeCancelEvent.makerAssetType = convertAssetProxyIdToType(makerAssetData.assetProxyId as AssetProxyId);
exchangeCancelEvent.makerAssetProxyId = makerAssetData.assetProxyId;
- exchangeCancelEvent.makerTokenAddress = makerAssetData.tokenAddress;
+ // HACK(abandeali1): this event schema currently does not support multiple maker/taker assets, so we store the first token address from the MultiAssetProxy assetData
+ exchangeCancelEvent.makerTokenAddress = assetDataUtils.isMultiAssetData(makerAssetData)
+ ? assetDataUtils.decodeMultiAssetDataRecursively(eventLog.args.makerAssetData).nestedAssetData[0].tokenAddress
+ : makerAssetData.tokenAddress;
// tslint:disable-next-line:no-unnecessary-type-assertion
exchangeCancelEvent.makerTokenId = bigNumbertoStringOrNull((makerAssetData as ERC721AssetData).tokenId);
exchangeCancelEvent.rawTakerAssetData = eventLog.args.takerAssetData;
- exchangeCancelEvent.takerAssetType = takerAssetType;
+ // tslint:disable-next-line:no-unnecessary-type-assertion
+ exchangeCancelEvent.takerAssetType = convertAssetProxyIdToType(takerAssetData.assetProxyId as AssetProxyId);
exchangeCancelEvent.takerAssetProxyId = takerAssetData.assetProxyId;
- exchangeCancelEvent.takerTokenAddress = takerAssetData.tokenAddress;
+ // HACK(abandeali1): this event schema currently does not support multiple maker/taker assets, so we store the first token address from the MultiAssetProxy assetData
+ exchangeCancelEvent.takerTokenAddress = assetDataUtils.isMultiAssetData(takerAssetData)
+ ? assetDataUtils.decodeMultiAssetDataRecursively(eventLog.args.takerAssetData).nestedAssetData[0].tokenAddress
+ : takerAssetData.tokenAddress;
// tslint:disable-next-line:no-unnecessary-type-assertion
exchangeCancelEvent.takerTokenId = bigNumbertoStringOrNull((takerAssetData as ERC721AssetData).tokenId);
return exchangeCancelEvent;
diff --git a/packages/pipeline/src/parsers/idex_orders/index.ts b/packages/pipeline/src/parsers/idex_orders/index.ts
index dfe27455c..14b871195 100644
--- a/packages/pipeline/src/parsers/idex_orders/index.ts
+++ b/packages/pipeline/src/parsers/idex_orders/index.ts
@@ -2,7 +2,7 @@ import { BigNumber } from '@0x/utils';
import { aggregateOrders } from '../utils';
-import { IdexOrder, IdexOrderbook, IdexOrderParam } from '../../data_sources/idex';
+import { IdexOrderbook, IdexOrderParam } from '../../data_sources/idex';
import { TokenOrderbookSnapshot as TokenOrder } from '../../entities';
import { OrderType } from '../../types';
@@ -21,7 +21,9 @@ export function parseIdexOrders(idexOrderbook: IdexOrderbook, observedTimestamp:
const idexBidOrder = idexOrderbook.bids[0];
const parsedBids =
aggregatedBids.length > 0
- ? aggregatedBids.map(order => parseIdexOrder(idexBidOrder.params, observedTimestamp, 'bid', source, order))
+ ? aggregatedBids.map(order =>
+ parseIdexOrder(idexBidOrder.params, observedTimestamp, OrderType.Bid, source, order),
+ )
: [];
const aggregatedAsks = aggregateOrders(idexOrderbook.asks);
@@ -29,7 +31,9 @@ export function parseIdexOrders(idexOrderbook: IdexOrderbook, observedTimestamp:
const idexAskOrder = idexOrderbook.asks[0];
const parsedAsks =
aggregatedAsks.length > 0
- ? aggregatedAsks.map(order => parseIdexOrder(idexAskOrder.params, observedTimestamp, 'ask', source, order))
+ ? aggregatedAsks.map(order =>
+ parseIdexOrder(idexAskOrder.params, observedTimestamp, OrderType.Ask, source, order),
+ )
: [];
return parsedBids.concat(parsedAsks);
}
@@ -62,7 +66,7 @@ export function parseIdexOrder(
tokenOrder.baseVolume = amount;
tokenOrder.quoteVolume = price.times(amount);
- if (orderType === 'bid') {
+ if (orderType === OrderType.Bid) {
tokenOrder.baseAssetSymbol = idexOrderParam.buySymbol;
tokenOrder.baseAssetAddress = idexOrderParam.tokenBuy;
tokenOrder.quoteAssetSymbol = idexOrderParam.sellSymbol;
diff --git a/packages/pipeline/src/parsers/oasis_orders/index.ts b/packages/pipeline/src/parsers/oasis_orders/index.ts
index 13997f31b..b71fb65b9 100644
--- a/packages/pipeline/src/parsers/oasis_orders/index.ts
+++ b/packages/pipeline/src/parsers/oasis_orders/index.ts
@@ -23,13 +23,13 @@ export function parseOasisOrders(
observedTimestamp: number,
source: string,
): TokenOrder[] {
- const aggregatedBids = aggregateOrders(R.filter(R.propEq('act', 'bid'), oasisOrderbook));
- const aggregatedAsks = aggregateOrders(R.filter(R.propEq('act', 'ask'), oasisOrderbook));
+ const aggregatedBids = aggregateOrders(R.filter(R.propEq('act', OrderType.Bid), oasisOrderbook));
+ const aggregatedAsks = aggregateOrders(R.filter(R.propEq('act', OrderType.Ask), oasisOrderbook));
const parsedBids = aggregatedBids.map(order =>
- parseOasisOrder(oasisMarket, observedTimestamp, 'bid', source, order),
+ parseOasisOrder(oasisMarket, observedTimestamp, OrderType.Bid, source, order),
);
const parsedAsks = aggregatedAsks.map(order =>
- parseOasisOrder(oasisMarket, observedTimestamp, 'ask', source, order),
+ parseOasisOrder(oasisMarket, observedTimestamp, OrderType.Ask, source, order),
);
return parsedBids.concat(parsedAsks);
}
diff --git a/packages/pipeline/src/parsers/paradex_orders/index.ts b/packages/pipeline/src/parsers/paradex_orders/index.ts
index 5ceeb64a4..85990dae4 100644
--- a/packages/pipeline/src/parsers/paradex_orders/index.ts
+++ b/packages/pipeline/src/parsers/paradex_orders/index.ts
@@ -21,10 +21,10 @@ export function parseParadexOrders(
source: string,
): TokenOrder[] {
const parsedBids = paradexOrderbookResponse.bids.map(order =>
- parseParadexOrder(paradexMarket, observedTimestamp, 'bid', source, order),
+ parseParadexOrder(paradexMarket, observedTimestamp, OrderType.Bid, source, order),
);
const parsedAsks = paradexOrderbookResponse.asks.map(order =>
- parseParadexOrder(paradexMarket, observedTimestamp, 'ask', source, order),
+ parseParadexOrder(paradexMarket, observedTimestamp, OrderType.Ask, source, order),
);
return parsedBids.concat(parsedAsks);
}
diff --git a/packages/pipeline/src/parsers/sra_orders/index.ts b/packages/pipeline/src/parsers/sra_orders/index.ts
index ef8901e40..13fe632a4 100644
--- a/packages/pipeline/src/parsers/sra_orders/index.ts
+++ b/packages/pipeline/src/parsers/sra_orders/index.ts
@@ -4,7 +4,7 @@ import { AssetProxyId, ERC721AssetData } from '@0x/types';
import * as R from 'ramda';
import { SraOrder } from '../../entities';
-import { bigNumbertoStringOrNull } from '../../utils';
+import { bigNumbertoStringOrNull, convertAssetProxyIdToType } from '../../utils';
/**
* Parses a raw order response from an SRA endpoint and returns an array of
@@ -22,9 +22,7 @@ export function parseSraOrders(rawOrdersResponse: OrdersResponse): SraOrder[] {
export function _convertToEntity(apiOrder: APIOrder): SraOrder {
// TODO(albrow): refactor out common asset data decoding code.
const makerAssetData = assetDataUtils.decodeAssetDataOrThrow(apiOrder.order.makerAssetData);
- const makerAssetType = makerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721';
const takerAssetData = assetDataUtils.decodeAssetDataOrThrow(apiOrder.order.takerAssetData);
- const takerAssetType = takerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721';
const sraOrder = new SraOrder();
sraOrder.exchangeAddress = apiOrder.order.exchangeAddress;
@@ -43,16 +41,24 @@ export function _convertToEntity(apiOrder: APIOrder): SraOrder {
sraOrder.signature = apiOrder.order.signature;
sraOrder.rawMakerAssetData = apiOrder.order.makerAssetData;
- sraOrder.makerAssetType = makerAssetType;
+ // tslint:disable-next-line:no-unnecessary-type-assertion
+ sraOrder.makerAssetType = convertAssetProxyIdToType(makerAssetData.assetProxyId as AssetProxyId);
sraOrder.makerAssetProxyId = makerAssetData.assetProxyId;
- sraOrder.makerTokenAddress = makerAssetData.tokenAddress;
+ // HACK(abandeali1): this event schema currently does not support multiple maker/taker assets, so we store the first token address from the MultiAssetProxy assetData
+ sraOrder.makerTokenAddress = assetDataUtils.isMultiAssetData(makerAssetData)
+ ? assetDataUtils.decodeMultiAssetDataRecursively(apiOrder.order.makerAssetData).nestedAssetData[0].tokenAddress
+ : makerAssetData.tokenAddress;
// tslint has a false positive here. Type assertion is required.
// tslint:disable-next-line:no-unnecessary-type-assertion
sraOrder.makerTokenId = bigNumbertoStringOrNull((makerAssetData as ERC721AssetData).tokenId);
sraOrder.rawTakerAssetData = apiOrder.order.takerAssetData;
- sraOrder.takerAssetType = takerAssetType;
+ // tslint:disable-next-line:no-unnecessary-type-assertion
+ sraOrder.takerAssetType = convertAssetProxyIdToType(takerAssetData.assetProxyId as AssetProxyId);
sraOrder.takerAssetProxyId = takerAssetData.assetProxyId;
- sraOrder.takerTokenAddress = takerAssetData.tokenAddress;
+ // HACK(abandeali1): this event schema currently does not support multiple maker/taker assets, so we store the first token address from the MultiAssetProxy assetData
+ sraOrder.takerTokenAddress = assetDataUtils.isMultiAssetData(takerAssetData)
+ ? assetDataUtils.decodeMultiAssetDataRecursively(apiOrder.order.takerAssetData).nestedAssetData[0].tokenAddress
+ : takerAssetData.tokenAddress;
// tslint:disable-next-line:no-unnecessary-type-assertion
sraOrder.takerTokenId = bigNumbertoStringOrNull((takerAssetData as ERC721AssetData).tokenId);
diff --git a/packages/pipeline/src/scripts/pull_missing_blocks.ts b/packages/pipeline/src/scripts/pull_missing_blocks.ts
index bb5385126..ced9d99eb 100644
--- a/packages/pipeline/src/scripts/pull_missing_blocks.ts
+++ b/packages/pipeline/src/scripts/pull_missing_blocks.ts
@@ -14,20 +14,29 @@ import { handleError, INFURA_ROOT_URL } from '../utils';
// Number of blocks to save at once.
const BATCH_SAVE_SIZE = 1000;
// Maximum number of requests to send at once.
-const MAX_CONCURRENCY = 10;
+const MAX_CONCURRENCY = 20;
// Maximum number of blocks to query for at once. This is also the maximum
// number of blocks we will hold in memory prior to being saved to the database.
const MAX_BLOCKS_PER_QUERY = 1000;
let connection: Connection;
+const tablesWithMissingBlocks = [
+ 'raw.exchange_fill_events',
+ 'raw.exchange_cancel_events',
+ 'raw.exchange_cancel_up_to_events',
+ 'raw.erc20_approval_events',
+];
+
(async () => {
connection = await createConnection(ormConfig as ConnectionOptions);
const provider = web3Factory.getRpcProvider({
rpcUrl: INFURA_ROOT_URL,
});
const web3Source = new Web3Source(provider);
- await getAllMissingBlocksAsync(web3Source);
+ for (const tableName of tablesWithMissingBlocks) {
+ await getAllMissingBlocksAsync(web3Source, tableName);
+ }
process.exit(0);
})().catch(handleError);
@@ -35,10 +44,11 @@ interface MissingBlocksResponse {
block_number: string;
}
-async function getAllMissingBlocksAsync(web3Source: Web3Source): Promise<void> {
+async function getAllMissingBlocksAsync(web3Source: Web3Source, tableName: string): Promise<void> {
const blocksRepository = connection.getRepository(Block);
while (true) {
- const blockNumbers = await getMissingBlockNumbersAsync();
+ console.log(`Checking for missing blocks in ${tableName}...`);
+ const blockNumbers = await getMissingBlockNumbersAsync(tableName);
if (blockNumbers.length === 0) {
// There are no more missing blocks. We're done.
break;
@@ -46,24 +56,14 @@ async function getAllMissingBlocksAsync(web3Source: Web3Source): Promise<void> {
await getAndSaveBlocksAsync(web3Source, blocksRepository, blockNumbers);
}
const totalBlocks = await blocksRepository.count();
- console.log(`Done saving blocks. There are now ${totalBlocks} total blocks.`);
+ console.log(`Done saving blocks for ${tableName}. There are now ${totalBlocks} total blocks.`);
}
-async function getMissingBlockNumbersAsync(): Promise<number[]> {
- // Note(albrow): The easiest way to get all the blocks we need is to
- // consider all the events tables together in a single query. If this query
- // gets too slow, we should consider re-architecting so that we can work on
- // getting the blocks for one type of event at a time.
+async function getMissingBlockNumbersAsync(tableName: string): Promise<number[]> {
+ // This query returns up to `MAX_BLOCKS_PER_QUERY` distinct block numbers
+ // which are present in `tableName` but not in `raw.blocks`.
const response = (await connection.query(
- `WITH all_events AS (
- SELECT block_number FROM raw.exchange_fill_events
- UNION SELECT block_number FROM raw.exchange_cancel_events
- UNION SELECT block_number FROM raw.exchange_cancel_up_to_events
- UNION SELECT block_number FROM raw.erc20_approval_events
- )
- SELECT DISTINCT(block_number) FROM all_events
- WHERE block_number NOT IN (SELECT number FROM raw.blocks)
- ORDER BY block_number ASC LIMIT $1`,
+ `SELECT DISTINCT(block_number) FROM ${tableName} LEFT JOIN raw.blocks ON ${tableName}.block_number = raw.blocks.number WHERE number IS NULL LIMIT $1;`,
[MAX_BLOCKS_PER_QUERY],
)) as MissingBlocksResponse[];
const blockNumberStrings = R.pluck('block_number', response);
@@ -86,4 +86,5 @@ async function getAndSaveBlocksAsync(
const blocks = R.map(parseBlock, rawBlocks);
console.log(`Saving ${blocks.length} blocks...`);
await blocksRepository.save(blocks, { chunk: Math.ceil(blocks.length / BATCH_SAVE_SIZE) });
+ console.log('Done saving this batch of blocks');
}
diff --git a/packages/pipeline/src/types.ts b/packages/pipeline/src/types.ts
index e02b42a40..5f2121807 100644
--- a/packages/pipeline/src/types.ts
+++ b/packages/pipeline/src/types.ts
@@ -1,2 +1,9 @@
-export type AssetType = 'erc20' | 'erc721';
-export type OrderType = 'bid' | 'ask';
+export enum AssetType {
+ ERC20 = 'erc20',
+ ERC721 = 'erc721',
+ MultiAsset = 'multiAsset',
+}
+export enum OrderType {
+ Bid = 'bid',
+ Ask = 'ask',
+}
diff --git a/packages/pipeline/src/utils/transformers/asset_proxy_id_types.ts b/packages/pipeline/src/utils/transformers/asset_proxy_id_types.ts
new file mode 100644
index 000000000..2cd05a616
--- /dev/null
+++ b/packages/pipeline/src/utils/transformers/asset_proxy_id_types.ts
@@ -0,0 +1,20 @@
+import { AssetProxyId } from '@0x/types';
+
+import { AssetType } from '../../types';
+
+/**
+ * Converts an assetProxyId to its string equivalent
+ * @param assetProxyId Id of AssetProxy
+ */
+export function convertAssetProxyIdToType(assetProxyId: AssetProxyId): AssetType {
+ switch (assetProxyId) {
+ case AssetProxyId.ERC20:
+ return AssetType.ERC20;
+ case AssetProxyId.ERC721:
+ return AssetType.ERC721;
+ case AssetProxyId.MultiAsset:
+ return AssetType.MultiAsset;
+ default:
+ throw new Error(`${assetProxyId} not a supported assetProxyId`);
+ }
+}
diff --git a/packages/pipeline/src/utils/transformers/index.ts b/packages/pipeline/src/utils/transformers/index.ts
index 232c1c5de..31a4c9223 100644
--- a/packages/pipeline/src/utils/transformers/index.ts
+++ b/packages/pipeline/src/utils/transformers/index.ts
@@ -1,2 +1,3 @@
export * from './big_number';
export * from './number_to_bigint';
+export * from './asset_proxy_id_types';