diff options
Diffstat (limited to 'packages/pipeline/src/utils')
7 files changed, 0 insertions, 242 deletions
diff --git a/packages/pipeline/src/utils/constants.ts b/packages/pipeline/src/utils/constants.ts deleted file mode 100644 index 56f3e82d8..000000000 --- a/packages/pipeline/src/utils/constants.ts +++ /dev/null @@ -1,3 +0,0 @@ -// Block number when the Exchange contract was deployed to mainnet. -export const EXCHANGE_START_BLOCK = 6271590; -export const INFURA_ROOT_URL = 'https://mainnet.infura.io'; diff --git a/packages/pipeline/src/utils/get_ohlcv_trading_pairs.ts b/packages/pipeline/src/utils/get_ohlcv_trading_pairs.ts deleted file mode 100644 index 19f81344e..000000000 --- a/packages/pipeline/src/utils/get_ohlcv_trading_pairs.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { fetchAsync } from '@0x/utils'; -import * as R from 'ramda'; -import { Connection } from 'typeorm'; - -export interface TradingPair { - fromSymbol: string; - toSymbol: string; - latestSavedTime: number; -} - -const COINLIST_API = 'https://min-api.cryptocompare.com/data/all/coinlist?BuiltOn=7605'; - -interface CryptoCompareCoinListResp { - Data: Map<string, CryptoCompareCoin>; -} - -interface CryptoCompareCoin { - Symbol: string; - BuiltOn: string; - SmartContractAddress: string; -} - -const TO_CURRENCIES = ['USD', 'EUR', 'ETH', 'USDT']; -const ETHEREUM_IDENTIFIER = '7605'; -const HTTP_OK_STATUS = 200; - -interface StaticPair { - fromSymbol: string; - toSymbol: string; -} -const SPECIAL_CASES: StaticPair[] = [ - { - fromSymbol: 'ETH', - toSymbol: 'USD', - }, -]; - -/** - * Get trading pairs with latest scraped time for OHLCV records - * @param conn a typeorm Connection to postgres - */ -export async function fetchOHLCVTradingPairsAsync( - conn: Connection, - source: string, - earliestBackfillTime: number, -): Promise<TradingPair[]> { - // fetch existing ohlcv records - const latestTradingPairs: Array<{ - from_symbol: string; - to_symbol: string; - latest: string; - }> = await conn.query(`SELECT - MAX(end_time) as latest, - from_symbol, - to_symbol - FROM raw.ohlcv_external - GROUP BY from_symbol, to_symbol;`); - - // build addressable index: { fromsym: { tosym: time }} - const latestTradingPairsIndex: { [fromSym: string]: { [toSym: string]: number } } = {}; - latestTradingPairs.forEach(pair => { - const latestIndex: { [toSym: string]: number } = latestTradingPairsIndex[pair.from_symbol] || {}; - latestIndex[pair.to_symbol] = parseInt(pair.latest, 10); // tslint:disable-line:custom-no-magic-numbers - latestTradingPairsIndex[pair.from_symbol] = latestIndex; - }); - - // match time to special cases - const specialCases: TradingPair[] = SPECIAL_CASES.map(pair => { - const latestSavedTime = - R.path<number>([pair.fromSymbol, pair.toSymbol], latestTradingPairsIndex) || earliestBackfillTime; - return R.assoc('latestSavedTime', latestSavedTime, pair); - }); - - // get token symbols used by Crypto Compare - const allCoinsResp = await fetchAsync(COINLIST_API); - if (allCoinsResp.status !== HTTP_OK_STATUS) { - return []; - } - const allCoins: CryptoCompareCoinListResp = await allCoinsResp.json(); - const erc20CoinsIndex: Map<string, string> = new Map(); - Object.entries(allCoins.Data).forEach(pair => { - const [symbol, coinData] = pair; - if (coinData.BuiltOn === ETHEREUM_IDENTIFIER && coinData.SmartContractAddress !== 'N/A') { - erc20CoinsIndex.set(coinData.SmartContractAddress.toLowerCase(), symbol); - } - }); - - // fetch all tokens that are traded on 0x - const rawEventTokenAddresses: Array<{ tokenaddress: string }> = await conn.query( - `SELECT DISTINCT(maker_token_address) as tokenaddress FROM raw.exchange_fill_events UNION - SELECT DISTINCT(taker_token_address) as tokenaddress FROM raw.exchange_fill_events`, - ); - - // tslint:disable-next-line:no-unbound-method - const eventTokenAddresses = R.pluck('tokenaddress', rawEventTokenAddresses).map(R.toLower); - - // join token addresses with CC symbols - const eventTokenSymbols: string[] = eventTokenAddresses - .filter(tokenAddress => erc20CoinsIndex.has(tokenAddress)) - .map(tokenAddress => erc20CoinsIndex.get(tokenAddress) as string); - - // join traded tokens with fiat and latest backfill time - const eventTradingPairs: TradingPair[] = R.chain(sym => { - return TO_CURRENCIES.map(fiat => { - const pair = { - fromSymbol: sym, - toSymbol: fiat, - latestSavedTime: R.path<number>([sym, fiat], latestTradingPairsIndex) || earliestBackfillTime, - }; - return pair; - }); - }, eventTokenSymbols); - - // join with special cases - return R.concat(eventTradingPairs, specialCases); -} diff --git a/packages/pipeline/src/utils/index.ts b/packages/pipeline/src/utils/index.ts deleted file mode 100644 index 094c0178e..000000000 --- a/packages/pipeline/src/utils/index.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { BigNumber } from '@0x/utils'; -export * from './transformers'; -export * from './constants'; - -/** - * If the given BigNumber is not null, returns the string representation of that - * number. Otherwise, returns null. - * @param n The number to convert. - */ -export function bigNumbertoStringOrNull(n: BigNumber): string | null { - if (n == null) { - return null; - } - return n.toString(); -} - -/** - * If value is null or undefined, returns null. Otherwise converts value to a - * BigNumber. - * @param value A string or number to be converted to a BigNumber - */ -export function toBigNumberOrNull(value: string | number | null): BigNumber | null { - switch (value) { - case null: - case undefined: - return null; - default: - return new BigNumber(value); - } -} - -/** - * Logs an error by intelligently checking for `message` and `stack` properties. - * Intended for use with top-level immediately invoked asynchronous functions. - * @param e the error to log. - */ -export function handleError(e: any): void { - if (e.message != null) { - // tslint:disable-next-line:no-console - console.error(e.message); - } else { - // tslint:disable-next-line:no-console - console.error('Unknown error'); - } - if (e.stack != null) { - // tslint:disable-next-line:no-console - console.error(e.stack); - } else { - // tslint:disable-next-line:no-console - console.error('(No stack trace)'); - } - process.exit(1); -} diff --git a/packages/pipeline/src/utils/transformers/asset_proxy_id_types.ts b/packages/pipeline/src/utils/transformers/asset_proxy_id_types.ts deleted file mode 100644 index 2cd05a616..000000000 --- a/packages/pipeline/src/utils/transformers/asset_proxy_id_types.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { AssetProxyId } from '@0x/types'; - -import { AssetType } from '../../types'; - -/** - * Converts an assetProxyId to its string equivalent - * @param assetProxyId Id of AssetProxy - */ -export function convertAssetProxyIdToType(assetProxyId: AssetProxyId): AssetType { - switch (assetProxyId) { - case AssetProxyId.ERC20: - return AssetType.ERC20; - case AssetProxyId.ERC721: - return AssetType.ERC721; - case AssetProxyId.MultiAsset: - return AssetType.MultiAsset; - default: - throw new Error(`${assetProxyId} not a supported assetProxyId`); - } -} diff --git a/packages/pipeline/src/utils/transformers/big_number.ts b/packages/pipeline/src/utils/transformers/big_number.ts deleted file mode 100644 index 5f2e4d565..000000000 --- a/packages/pipeline/src/utils/transformers/big_number.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { BigNumber } from '@0x/utils'; -import { ValueTransformer } from 'typeorm/decorator/options/ValueTransformer'; - -export class BigNumberTransformer implements ValueTransformer { - // tslint:disable-next-line:prefer-function-over-method - public to(value: BigNumber | null): string | null { - return value === null ? null : value.toString(); - } - - // tslint:disable-next-line:prefer-function-over-method - public from(value: string | null): BigNumber | null { - return value === null ? null : new BigNumber(value); - } -} - -export const bigNumberTransformer = new BigNumberTransformer(); diff --git a/packages/pipeline/src/utils/transformers/index.ts b/packages/pipeline/src/utils/transformers/index.ts deleted file mode 100644 index 31a4c9223..000000000 --- a/packages/pipeline/src/utils/transformers/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './big_number'; -export * from './number_to_bigint'; -export * from './asset_proxy_id_types'; diff --git a/packages/pipeline/src/utils/transformers/number_to_bigint.ts b/packages/pipeline/src/utils/transformers/number_to_bigint.ts deleted file mode 100644 index 8fbd52093..000000000 --- a/packages/pipeline/src/utils/transformers/number_to_bigint.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { BigNumber } from '@0x/utils'; -import { ValueTransformer } from 'typeorm/decorator/options/ValueTransformer'; - -const decimalRadix = 10; - -// Can be used to convert a JavaScript number type to a Postgres bigint type and -// vice versa. By default TypeORM will silently convert number types to string -// if the corresponding Postgres type is bigint. See -// https://github.com/typeorm/typeorm/issues/2400 for more information. -export class NumberToBigIntTransformer implements ValueTransformer { - // tslint:disable-next-line:prefer-function-over-method - public to(value: number): string | null { - if (value === null || value === undefined) { - return null; - } else { - return value.toString(); - } - } - - // tslint:disable-next-line:prefer-function-over-method - public from(value: string): number { - if (new BigNumber(value).isGreaterThan(Number.MAX_SAFE_INTEGER)) { - throw new Error( - `Attempted to convert PostgreSQL bigint value (${value}) to JavaScript number type but it is too big to safely convert`, - ); - } - return Number.parseInt(value, decimalRadix); - } -} - -export const numberToBigIntTransformer = new NumberToBigIntTransformer(); |