diff options
Diffstat (limited to 'packages/pipeline')
10 files changed, 122 insertions, 68 deletions
diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index f8adf9055..a40f3d21c 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -1,6 +1,6 @@ { "name": "@0x/pipeline", - "version": "1.0.1", + "version": "1.0.2", "private": true, "description": "Data pipeline for offline analysis", "scripts": { @@ -27,7 +27,7 @@ }, "license": "Apache-2.0", "devDependencies": { - "@0x/tslint-config": "^1.0.9", + "@0x/tslint-config": "^2.0.0", "@types/axios": "^0.14.0", "@types/ramda": "^0.25.38", "chai": "^4.1.2", @@ -39,23 +39,23 @@ "typescript": "3.0.1" }, "dependencies": { - "@0x/connect": "^3.0.9", + "@0x/connect": "^3.0.10", "@0x/contract-addresses": "^2.0.0", "@0x/contract-artifacts": "^1.0.1", "@0x/contract-wrappers": "^3.0.0", - "@0x/dev-utils": "^1.0.20", + "@0x/dev-utils": "^1.0.21", "@0x/order-utils": "^2.0.0", - "@0x/subproviders": "^2.1.7", - "@0x/types": "^1.4.0", - "@0x/utils": "^2.0.7", - "@0x/web3-wrapper": "^3.2.0", + "@0x/subproviders": "^2.1.8", + "@0x/types": "^1.4.1", + "@0x/utils": "^2.0.8", + "@0x/web3-wrapper": "^3.2.1", "@types/dockerode": "^2.5.9", "@types/p-limit": "^2.0.0", "async-parallel": "^1.2.3", "axios": "^0.18.0", "bottleneck": "^2.13.2", "dockerode": "^2.5.7", - "ethereum-types": "^1.1.3", + "ethereum-types": "^1.1.4", "pg": "^7.5.0", "prettier": "^1.15.3", "ramda": "^0.25.0", diff --git a/packages/pipeline/src/parsers/ddex_orders/index.ts b/packages/pipeline/src/parsers/ddex_orders/index.ts index 52a998f9f..d7b97efbe 100644 --- a/packages/pipeline/src/parsers/ddex_orders/index.ts +++ b/packages/pipeline/src/parsers/ddex_orders/index.ts @@ -54,12 +54,14 @@ export function parseDdexOrder( tokenOrder.orderType = orderType; tokenOrder.price = price; - tokenOrder.baseAssetSymbol = ddexMarket.baseToken; - tokenOrder.baseAssetAddress = ddexMarket.baseTokenAddress; - tokenOrder.baseVolume = price.times(amount); + // ddex currently confuses quote and base assets. + // We switch them here to maintain our internal consistency. + tokenOrder.baseAssetSymbol = ddexMarket.quoteToken; + tokenOrder.baseAssetAddress = ddexMarket.quoteTokenAddress; + tokenOrder.baseVolume = amount; - tokenOrder.quoteAssetSymbol = ddexMarket.quoteToken; - tokenOrder.quoteAssetAddress = ddexMarket.quoteTokenAddress; - tokenOrder.quoteVolume = amount; + tokenOrder.quoteAssetSymbol = ddexMarket.baseToken; + tokenOrder.quoteAssetAddress = ddexMarket.baseTokenAddress; + tokenOrder.quoteVolume = price.times(amount); return tokenOrder; } diff --git a/packages/pipeline/src/parsers/oasis_orders/index.ts b/packages/pipeline/src/parsers/oasis_orders/index.ts index 7aafbf460..13997f31b 100644 --- a/packages/pipeline/src/parsers/oasis_orders/index.ts +++ b/packages/pipeline/src/parsers/oasis_orders/index.ts @@ -62,10 +62,10 @@ export function parseOasisOrder( tokenOrder.baseAssetSymbol = oasisMarket.base; tokenOrder.baseAssetAddress = null; // Oasis doesn't provide address information - tokenOrder.baseVolume = price.times(amount); + tokenOrder.baseVolume = amount; tokenOrder.quoteAssetSymbol = oasisMarket.quote; tokenOrder.quoteAssetAddress = null; // Oasis doesn't provide address information - tokenOrder.quoteVolume = amount; + tokenOrder.quoteVolume = price.times(amount); return tokenOrder; } diff --git a/packages/pipeline/src/parsers/paradex_orders/index.ts b/packages/pipeline/src/parsers/paradex_orders/index.ts index 7966658a7..5ceeb64a4 100644 --- a/packages/pipeline/src/parsers/paradex_orders/index.ts +++ b/packages/pipeline/src/parsers/paradex_orders/index.ts @@ -57,10 +57,10 @@ export function parseParadexOrder( tokenOrder.baseAssetSymbol = paradexMarket.baseToken; tokenOrder.baseAssetAddress = paradexMarket.baseTokenAddress as string; - tokenOrder.baseVolume = price.times(amount); + tokenOrder.baseVolume = amount; tokenOrder.quoteAssetSymbol = paradexMarket.quoteToken; tokenOrder.quoteAssetAddress = paradexMarket.quoteTokenAddress as string; - tokenOrder.quoteVolume = amount; + tokenOrder.quoteVolume = price.times(amount); return tokenOrder; } diff --git a/packages/pipeline/src/scripts/pull_erc20_events.ts b/packages/pipeline/src/scripts/pull_erc20_events.ts index 0ad12c97a..bd520c610 100644 --- a/packages/pipeline/src/scripts/pull_erc20_events.ts +++ b/packages/pipeline/src/scripts/pull_erc20_events.ts @@ -1,10 +1,10 @@ -// tslint:disable:no-console import { getContractAddressesForNetworkOrThrow } from '@0x/contract-addresses'; import { web3Factory } from '@0x/dev-utils'; import { Web3ProviderEngine } from '@0x/subproviders'; +import { logUtils } from '@0x/utils'; import { Web3Wrapper } from '@0x/web3-wrapper'; import 'reflect-metadata'; -import { Connection, ConnectionOptions, createConnection, Repository } from 'typeorm'; +import { Connection, ConnectionOptions, createConnection } from 'typeorm'; import { ERC20EventsSource } from '../data_sources/contract-wrappers/erc20_events'; import { ERC20ApprovalEvent } from '../entities'; @@ -16,33 +16,63 @@ const NETWORK_ID = 1; const START_BLOCK_OFFSET = 100; // Number of blocks before the last known block to consider when updating fill events. const BATCH_SAVE_SIZE = 1000; // Number of events to save at once. const BLOCK_FINALITY_THRESHOLD = 10; // When to consider blocks as final. Used to compute default endBlock. -const WETH_START_BLOCK = 4719568; // Block number when the WETH contract was deployed. let connection: Connection; +interface Token { + // name is used for logging only. + name: string; + address: string; + defaultStartBlock: number; +} + +const tokensToGetApprovalEvents: Token[] = [ + { + name: 'WETH', + address: getContractAddressesForNetworkOrThrow(NETWORK_ID).etherToken, + defaultStartBlock: 4719568, // Block when the WETH contract was deployed. + }, + { + name: 'ZRX', + address: getContractAddressesForNetworkOrThrow(NETWORK_ID).zrxToken, + defaultStartBlock: 4145415, // Block when the ZRX contract was deployed. + }, + { + name: 'DAI', + address: '0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359', + defaultStartBlock: 4752008, // Block when the DAI contract was deployed. + }, +]; + (async () => { connection = await createConnection(ormConfig as ConnectionOptions); const provider = web3Factory.getRpcProvider({ rpcUrl: INFURA_ROOT_URL, }); const endBlock = await calculateEndBlockAsync(provider); - await getAndSaveWETHApprovalEventsAsync(provider, endBlock); + for (const token of tokensToGetApprovalEvents) { + await getAndSaveApprovalEventsAsync(provider, token, endBlock); + } process.exit(0); })().catch(handleError); -async function getAndSaveWETHApprovalEventsAsync(provider: Web3ProviderEngine, endBlock: number): Promise<void> { - console.log('Checking existing approval events...'); +async function getAndSaveApprovalEventsAsync( + provider: Web3ProviderEngine, + token: Token, + endBlock: number, +): Promise<void> { + logUtils.log(`Getting approval events for ${token.name}...`); + logUtils.log('Checking existing approval events...'); const repository = connection.getRepository(ERC20ApprovalEvent); - const startBlock = (await getStartBlockAsync(repository)) || WETH_START_BLOCK; + const startBlock = (await getStartBlockAsync(token)) || token.defaultStartBlock; - console.log(`Getting WETH approval events starting at ${startBlock}...`); - const wethTokenAddress = getContractAddressesForNetworkOrThrow(NETWORK_ID).etherToken; - const eventsSource = new ERC20EventsSource(provider, NETWORK_ID, wethTokenAddress); + logUtils.log(`Getting approval events starting at ${startBlock}...`); + const eventsSource = new ERC20EventsSource(provider, NETWORK_ID, token.address); const eventLogs = await eventsSource.getApprovalEventsAsync(startBlock, endBlock); - console.log(`Parsing ${eventLogs.length} WETH approval events...`); + logUtils.log(`Parsing ${eventLogs.length} approval events...`); const events = parseERC20ApprovalEvents(eventLogs); - console.log(`Retrieved and parsed ${events.length} total WETH approval events.`); + logUtils.log(`Retrieved and parsed ${events.length} total approval events.`); await repository.save(events, { chunk: Math.ceil(events.length / BATCH_SAVE_SIZE) }); } @@ -52,15 +82,15 @@ async function calculateEndBlockAsync(provider: Web3ProviderEngine): Promise<num return currentBlock - BLOCK_FINALITY_THRESHOLD; } -async function getStartBlockAsync(repository: Repository<ERC20ApprovalEvent>): Promise<number | null> { - const fillEventCount = await repository.count(); - if (fillEventCount === 0) { - console.log(`No existing approval events found.`); - return null; - } +async function getStartBlockAsync(token: Token): Promise<number | null> { const queryResult = await connection.query( - `SELECT block_number FROM raw.erc20_approval_events ORDER BY block_number DESC LIMIT 1`, + `SELECT block_number FROM raw.erc20_approval_events WHERE token_address = $1 ORDER BY block_number DESC LIMIT 1`, + [token.address], ); + if (queryResult.length === 0) { + logUtils.log(`No existing approval events found for ${token.name}.`); + return null; + } const lastKnownBlock = queryResult[0].block_number; return lastKnownBlock - START_BLOCK_OFFSET; } diff --git a/packages/pipeline/src/scripts/pull_missing_blocks.ts b/packages/pipeline/src/scripts/pull_missing_blocks.ts index a5203824c..bb5385126 100644 --- a/packages/pipeline/src/scripts/pull_missing_blocks.ts +++ b/packages/pipeline/src/scripts/pull_missing_blocks.ts @@ -9,7 +9,7 @@ import { Web3Source } from '../data_sources/web3'; import { Block } from '../entities'; import * as ormConfig from '../ormconfig'; import { parseBlock } from '../parsers/web3'; -import { EXCHANGE_START_BLOCK, handleError, INFURA_ROOT_URL } from '../utils'; +import { handleError, INFURA_ROOT_URL } from '../utils'; // Number of blocks to save at once. const BATCH_SAVE_SIZE = 1000; @@ -37,22 +37,19 @@ interface MissingBlocksResponse { async function getAllMissingBlocksAsync(web3Source: Web3Source): Promise<void> { const blocksRepository = connection.getRepository(Block); - let fromBlock = EXCHANGE_START_BLOCK; while (true) { - const blockNumbers = await getMissingBlockNumbersAsync(fromBlock); + const blockNumbers = await getMissingBlockNumbersAsync(); if (blockNumbers.length === 0) { // There are no more missing blocks. We're done. break; } await getAndSaveBlocksAsync(web3Source, blocksRepository, blockNumbers); - fromBlock = Math.max(...blockNumbers) + 1; } const totalBlocks = await blocksRepository.count(); console.log(`Done saving blocks. There are now ${totalBlocks} total blocks.`); } -async function getMissingBlockNumbersAsync(fromBlock: number): Promise<number[]> { - console.log(`Checking for missing blocks starting at ${fromBlock}...`); +async function getMissingBlockNumbersAsync(): Promise<number[]> { // Note(albrow): The easiest way to get all the blocks we need is to // consider all the events tables together in a single query. If this query // gets too slow, we should consider re-architecting so that we can work on @@ -66,13 +63,12 @@ async function getMissingBlockNumbersAsync(fromBlock: number): Promise<number[]> ) SELECT DISTINCT(block_number) FROM all_events WHERE block_number NOT IN (SELECT number FROM raw.blocks) - AND block_number >= $1 - ORDER BY block_number ASC LIMIT $2`, - [fromBlock, MAX_BLOCKS_PER_QUERY], + ORDER BY block_number ASC LIMIT $1`, + [MAX_BLOCKS_PER_QUERY], )) as MissingBlocksResponse[]; const blockNumberStrings = R.pluck('block_number', response); const blockNumbers = R.map(parseInt, blockNumberStrings); - console.log(`Found ${blockNumbers.length} missing blocks in the given range.`); + console.log(`Found ${blockNumbers.length} missing blocks.`); return blockNumbers; } diff --git a/packages/pipeline/src/utils/get_ohlcv_trading_pairs.ts b/packages/pipeline/src/utils/get_ohlcv_trading_pairs.ts index 9d3ef2fba..19f81344e 100644 --- a/packages/pipeline/src/utils/get_ohlcv_trading_pairs.ts +++ b/packages/pipeline/src/utils/get_ohlcv_trading_pairs.ts @@ -23,6 +23,18 @@ interface CryptoCompareCoin { const TO_CURRENCIES = ['USD', 'EUR', 'ETH', 'USDT']; const ETHEREUM_IDENTIFIER = '7605'; const HTTP_OK_STATUS = 200; + +interface StaticPair { + fromSymbol: string; + toSymbol: string; +} +const SPECIAL_CASES: StaticPair[] = [ + { + fromSymbol: 'ETH', + toSymbol: 'USD', + }, +]; + /** * Get trading pairs with latest scraped time for OHLCV records * @param conn a typeorm Connection to postgres @@ -44,6 +56,7 @@ export async function fetchOHLCVTradingPairsAsync( FROM raw.ohlcv_external GROUP BY from_symbol, to_symbol;`); + // build addressable index: { fromsym: { tosym: time }} const latestTradingPairsIndex: { [fromSym: string]: { [toSym: string]: number } } = {}; latestTradingPairs.forEach(pair => { const latestIndex: { [toSym: string]: number } = latestTradingPairsIndex[pair.from_symbol] || {}; @@ -51,6 +64,13 @@ export async function fetchOHLCVTradingPairsAsync( latestTradingPairsIndex[pair.from_symbol] = latestIndex; }); + // match time to special cases + const specialCases: TradingPair[] = SPECIAL_CASES.map(pair => { + const latestSavedTime = + R.path<number>([pair.fromSymbol, pair.toSymbol], latestTradingPairsIndex) || earliestBackfillTime; + return R.assoc('latestSavedTime', latestSavedTime, pair); + }); + // get token symbols used by Crypto Compare const allCoinsResp = await fetchAsync(COINLIST_API); if (allCoinsResp.status !== HTTP_OK_STATUS) { @@ -66,27 +86,31 @@ export async function fetchOHLCVTradingPairsAsync( }); // fetch all tokens that are traded on 0x - const rawTokenAddresses: Array<{ tokenaddress: string }> = await conn.query( + const rawEventTokenAddresses: Array<{ tokenaddress: string }> = await conn.query( `SELECT DISTINCT(maker_token_address) as tokenaddress FROM raw.exchange_fill_events UNION SELECT DISTINCT(taker_token_address) as tokenaddress FROM raw.exchange_fill_events`, ); - const tokenAddresses = R.pluck('tokenaddress', rawTokenAddresses); + + // tslint:disable-next-line:no-unbound-method + const eventTokenAddresses = R.pluck('tokenaddress', rawEventTokenAddresses).map(R.toLower); // join token addresses with CC symbols - const allTokenSymbols: string[] = tokenAddresses - .map(tokenAddress => erc20CoinsIndex.get(tokenAddress.toLowerCase()) || '') - .filter(x => x); + const eventTokenSymbols: string[] = eventTokenAddresses + .filter(tokenAddress => erc20CoinsIndex.has(tokenAddress)) + .map(tokenAddress => erc20CoinsIndex.get(tokenAddress) as string); - // generate list of all tokens with time of latest existing record OR default earliest time - const allTradingPairCombinations: TradingPair[] = R.chain(sym => { + // join traded tokens with fiat and latest backfill time + const eventTradingPairs: TradingPair[] = R.chain(sym => { return TO_CURRENCIES.map(fiat => { - return { + const pair = { fromSymbol: sym, toSymbol: fiat, latestSavedTime: R.path<number>([sym, fiat], latestTradingPairsIndex) || earliestBackfillTime, }; + return pair; }); - }, allTokenSymbols); + }, eventTokenSymbols); - return allTradingPairCombinations; + // join with special cases + return R.concat(eventTradingPairs, specialCases); } diff --git a/packages/pipeline/test/parsers/ddex_orders/index_test.ts b/packages/pipeline/test/parsers/ddex_orders/index_test.ts index 9f4bfe7e3..4a4a86bf8 100644 --- a/packages/pipeline/test/parsers/ddex_orders/index_test.ts +++ b/packages/pipeline/test/parsers/ddex_orders/index_test.ts @@ -39,12 +39,14 @@ describe('ddex_orders', () => { expected.observedTimestamp = observedTimestamp; expected.orderType = 'bid'; expected.price = new BigNumber(0.5); - expected.baseAssetSymbol = 'DEF'; - expected.baseAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81'; - expected.baseVolume = new BigNumber(5); - expected.quoteAssetSymbol = 'ABC'; - expected.quoteAssetAddress = '0x0000000000000000000000000000000000000000'; - expected.quoteVolume = new BigNumber(10); + // ddex currently confuses base and quote assets. + // Switch them to maintain our internal consistency. + expected.baseAssetSymbol = 'ABC'; + expected.baseAssetAddress = '0x0000000000000000000000000000000000000000'; + expected.baseVolume = new BigNumber(10); + expected.quoteAssetSymbol = 'DEF'; + expected.quoteAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81'; + expected.quoteVolume = new BigNumber(5); const actual = parseDdexOrder(ddexMarket, observedTimestamp, orderType, source, ddexOrder); expect(actual).deep.equal(expected); diff --git a/packages/pipeline/test/parsers/oasis_orders/index_test.ts b/packages/pipeline/test/parsers/oasis_orders/index_test.ts index 9e8ba9a40..433bfb665 100644 --- a/packages/pipeline/test/parsers/oasis_orders/index_test.ts +++ b/packages/pipeline/test/parsers/oasis_orders/index_test.ts @@ -37,10 +37,10 @@ describe('oasis_orders', () => { expected.price = new BigNumber(0.5); expected.baseAssetSymbol = 'DEF'; expected.baseAssetAddress = null; - expected.baseVolume = new BigNumber(5); + expected.baseVolume = new BigNumber(10); expected.quoteAssetSymbol = 'ABC'; expected.quoteAssetAddress = null; - expected.quoteVolume = new BigNumber(10); + expected.quoteVolume = new BigNumber(5); const actual = parseOasisOrder(oasisMarket, observedTimestamp, orderType, source, oasisOrder); expect(actual).deep.equal(expected); diff --git a/packages/pipeline/test/parsers/paradex_orders/index_test.ts b/packages/pipeline/test/parsers/paradex_orders/index_test.ts index 1522806bf..6b811b90d 100644 --- a/packages/pipeline/test/parsers/paradex_orders/index_test.ts +++ b/packages/pipeline/test/parsers/paradex_orders/index_test.ts @@ -42,10 +42,10 @@ describe('paradex_orders', () => { expected.price = new BigNumber(0.1245); expected.baseAssetSymbol = 'DEF'; expected.baseAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81'; - expected.baseVolume = new BigNumber(412 * 0.1245); + expected.baseVolume = new BigNumber(412); expected.quoteAssetSymbol = 'ABC'; expected.quoteAssetAddress = '0x0000000000000000000000000000000000000000'; - expected.quoteVolume = new BigNumber(412); + expected.quoteVolume = new BigNumber(412 * 0.1245); const actual = parseParadexOrder(paradexMarket, observedTimestamp, orderType, source, paradexOrder); expect(actual).deep.equal(expected); |