diff options
author | Alex Browne <stephenalexbrowne@gmail.com> | 2018-12-05 12:04:08 +0800 |
---|---|---|
committer | Alex Browne <stephenalexbrowne@gmail.com> | 2018-12-05 12:04:08 +0800 |
commit | 00f86ca0f7871639d2b0be496f6f8c5e0d8d7ffe (patch) | |
tree | e2e1c5394deac225bf493b3fe51da9f3074f1691 | |
parent | 549f5e4655f246062dd6451065ec01eb789dbd8f (diff) | |
download | dexon-sol-tools-00f86ca0f7871639d2b0be496f6f8c5e0d8d7ffe.tar dexon-sol-tools-00f86ca0f7871639d2b0be496f6f8c5e0d8d7ffe.tar.gz dexon-sol-tools-00f86ca0f7871639d2b0be496f6f8c5e0d8d7ffe.tar.bz2 dexon-sol-tools-00f86ca0f7871639d2b0be496f6f8c5e0d8d7ffe.tar.lz dexon-sol-tools-00f86ca0f7871639d2b0be496f6f8c5e0d8d7ffe.tar.xz dexon-sol-tools-00f86ca0f7871639d2b0be496f6f8c5e0d8d7ffe.tar.zst dexon-sol-tools-00f86ca0f7871639d2b0be496f6f8c5e0d8d7ffe.zip |
Address PR feedback
21 files changed, 96 insertions, 81 deletions
diff --git a/packages/pipeline/README.md b/packages/pipeline/README.md index fb563b14c..0f4abd935 100644 --- a/packages/pipeline/README.md +++ b/packages/pipeline/README.md @@ -42,7 +42,8 @@ Revert the most recent migration (CAUTION: may result in data loss!): `yarn migr There are several test scripts in **package.json**. You can run all the tests with `yarn test:all` or run certain tests seprately by following the -instructions below. Some tests may not work out of the box on certain platforms. +instructions below. Some tests may not work out of the box on certain platforms +or operating systems (see the "Database tests" section below). ### Unit tests @@ -71,8 +72,8 @@ Postgres is via Docker. Depending on your platform, you may need to prepend docker run --rm -d -p 5432:5432 --name pipeline_postgres postgres:11-alpine ``` -This will start a Postgres server with the default username and database name. -You should set the environment variable as follows: +This will start a Postgres server with the default username and database name +(`postgres` and `postgres`). You should set the environment variable as follows: ``` export ZEROEX_DATA_PIPELINE_DB_URL=postgresql://postgres@localhost/postgres @@ -149,17 +150,17 @@ set the`ZEROEX_DATA_PIPELINE_DB_URL` environment variable to a valid #### Additional guidelines and tips: -* Table names should be plural and separated by underscores (e.g., +- Table names should be plural and separated by underscores (e.g., `exchange_fill_events`). -* Any table which contains data which comes directly from a third-party source +- Any table which contains data which comes directly from a third-party source should be namespaced in the `raw` PostgreSQL schema. -* Column names in the database should be separated by underscores (e.g., +- Column names in the database should be separated by underscores (e.g., `maker_asset_type`). -* Field names in entity classes (like any other fields in TypeScript) should +- Field names in entity classes (like any other fields in TypeScript) should be camel-cased (e.g., `makerAssetType`). -* All timestamps should be stored as milliseconds since the Unix Epoch. -* Use the `BigNumber` type for TypeScript code which deals with 256-bit +- All timestamps should be stored as milliseconds since the Unix Epoch. +- Use the `BigNumber` type for TypeScript code which deals with 256-bit numbers from smart contracts or for any case where we are dealing with large floating point numbers. -* [TypeORM documentation](http://typeorm.io/#/) is pretty robust and can be a +- [TypeORM documentation](http://typeorm.io/#/) is pretty robust and can be a helpful resource. diff --git a/packages/pipeline/migrations/1543980079179-ConvertTokenMetadataDecimalsToBigNumber.ts b/packages/pipeline/migrations/1543980079179-ConvertTokenMetadataDecimalsToBigNumber.ts new file mode 100644 index 000000000..351bc7eb8 --- /dev/null +++ b/packages/pipeline/migrations/1543980079179-ConvertTokenMetadataDecimalsToBigNumber.ts @@ -0,0 +1,17 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class ConvertTokenMetadataDecimalsToBigNumber1543980079179 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise<any> { + await queryRunner.query( + `ALTER TABLE raw.token_metadata + ALTER COLUMN decimals TYPE numeric USING decimals::numeric;`, + ); + } + + public async down(queryRunner: QueryRunner): Promise<any> { + await queryRunner.query( + `ALTER TABLE raw.token_metadata + ALTER COLUMN decimals TYPE numeric USING decimals::integer;`, + ); + } +} diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index 6af81c048..0539618d4 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -1,6 +1,6 @@ { "name": "@0x/pipeline", - "version": "0.0.1", + "version": "1.0.0", "private": true, "description": "Data pipeline for offline analysis", "scripts": { @@ -21,11 +21,6 @@ "migrate:revert": "yarn typeorm migration:revert --config ./lib/src/ormconfig", "migrate:create": "yarn typeorm migration:create --config ./lib/src/ormconfig --dir migrations" }, - "config": { - "postpublish": { - "assets": [] - } - }, "repository": { "type": "git", "url": "https://github.com/0xProject/0x-monorepo" diff --git a/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts b/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts index 5721c0ea3..1717eb8b3 100644 --- a/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts +++ b/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts @@ -11,9 +11,10 @@ import { Web3ProviderEngine } from '@0x/subproviders'; import { Web3Wrapper } from '@0x/web3-wrapper'; import { LogWithDecodedArgs } from 'ethereum-types'; +import { EXCHANGE_START_BLOCK } from '../../utils'; + const BLOCK_FINALITY_THRESHOLD = 10; // When to consider blocks as final. Used to compute default toBlock. const NUM_BLOCKS_PER_QUERY = 20000; // Number of blocks to query for events at a time. -const EXCHANGE_START_BLOCK = 6271590; // Block number when the Exchange contract was deployed to mainnet. export class ExchangeEventsSource { private readonly _exchangeWrapper: ExchangeWrapper; diff --git a/packages/pipeline/src/entities/exchange_cancel_event.ts b/packages/pipeline/src/entities/exchange_cancel_event.ts index 2fcc17df6..38f99c903 100644 --- a/packages/pipeline/src/entities/exchange_cancel_event.ts +++ b/packages/pipeline/src/entities/exchange_cancel_event.ts @@ -48,6 +48,4 @@ export class ExchangeCancelEvent { public takerTokenAddress!: string; @Column({ nullable: true, type: String, name: 'taker_token_id' }) public takerTokenId!: string | null; - - // TODO(albrow): Include topics? } diff --git a/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts b/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts index 60ead324f..27580305e 100644 --- a/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts +++ b/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts @@ -12,7 +12,6 @@ export class ExchangeCancelUpToEvent { @PrimaryColumn({ name: 'block_number', transformer: numberToBigIntTransformer }) public blockNumber!: number; - // TODO(albrow): Include transaction hash @Column({ name: 'raw_data' }) public rawData!: string; @@ -24,5 +23,4 @@ export class ExchangeCancelUpToEvent { public senderAddress!: string; @Column({ name: 'order_epoch', type: 'numeric', transformer: bigNumberTransformer }) public orderEpoch!: BigNumber; - // TODO(albrow): Include topics? } diff --git a/packages/pipeline/src/entities/exchange_fill_event.ts b/packages/pipeline/src/entities/exchange_fill_event.ts index bbf0abf58..9b7727615 100644 --- a/packages/pipeline/src/entities/exchange_fill_event.ts +++ b/packages/pipeline/src/entities/exchange_fill_event.ts @@ -57,6 +57,4 @@ export class ExchangeFillEvent { public takerTokenAddress!: string; @Column({ nullable: true, type: String, name: 'taker_token_id' }) public takerTokenId!: string | null; - - // TODO(albrow): Include topics? } diff --git a/packages/pipeline/src/entities/relayer.ts b/packages/pipeline/src/entities/relayer.ts index b3a856fd8..5af8578b4 100644 --- a/packages/pipeline/src/entities/relayer.ts +++ b/packages/pipeline/src/entities/relayer.ts @@ -14,9 +14,6 @@ export class Relayer { @Column({ name: 'app_url', type: 'varchar', nullable: true }) public appUrl!: string | null; - // TODO(albrow): Add exchange contract or protocol version? - // TODO(albrow): Add network ids for addresses? - @Column({ name: 'fee_recipient_addresses', type: 'varchar', array: true }) public feeRecipientAddresses!: string[]; @Column({ name: 'taker_addresses', type: 'varchar', array: true }) diff --git a/packages/pipeline/src/entities/token_metadata.ts b/packages/pipeline/src/entities/token_metadata.ts index ca1e57937..911b53972 100644 --- a/packages/pipeline/src/entities/token_metadata.ts +++ b/packages/pipeline/src/entities/token_metadata.ts @@ -1,5 +1,8 @@ +import { BigNumber } from '@0x/utils'; import { Column, Entity, PrimaryColumn } from 'typeorm'; +import { bigNumberTransformer } from '../utils/transformers'; + @Entity({ name: 'token_metadata', schema: 'raw' }) export class TokenMetadata { @PrimaryColumn({ type: 'varchar', nullable: false }) @@ -8,10 +11,8 @@ export class TokenMetadata { @PrimaryColumn({ type: 'varchar', nullable: false }) public authority!: string; - // TODO(albrow): Convert decimals field to type BigNumber/numeric because it - // comes from a 256-bit integer in a smart contract. - @Column({ type: 'integer', nullable: true }) - public decimals!: number | null; + @Column({ type: 'numeric', transformer: bigNumberTransformer, nullable: true }) + public decimals!: BigNumber | null; @Column({ type: 'varchar', nullable: true }) public symbol!: string | null; diff --git a/packages/pipeline/src/parsers/events/index.ts b/packages/pipeline/src/parsers/events/index.ts index d42d1c57a..e18106c75 100644 --- a/packages/pipeline/src/parsers/events/index.ts +++ b/packages/pipeline/src/parsers/events/index.ts @@ -49,8 +49,8 @@ export function _convertToExchangeFillEvent(eventLog: LogWithDecodedArgs<Exchang exchangeFillEvent.logIndex = eventLog.logIndex as number; exchangeFillEvent.rawData = eventLog.data as string; exchangeFillEvent.transactionHash = eventLog.transactionHash; - exchangeFillEvent.makerAddress = eventLog.args.makerAddress.toString(); - exchangeFillEvent.takerAddress = eventLog.args.takerAddress.toString(); + exchangeFillEvent.makerAddress = eventLog.args.makerAddress; + exchangeFillEvent.takerAddress = eventLog.args.takerAddress; exchangeFillEvent.feeRecipientAddress = eventLog.args.feeRecipientAddress; exchangeFillEvent.senderAddress = eventLog.args.senderAddress; exchangeFillEvent.makerAssetFilledAmount = eventLog.args.makerAssetFilledAmount; @@ -92,9 +92,8 @@ export function _convertToExchangeCancelEvent( exchangeCancelEvent.logIndex = eventLog.logIndex as number; exchangeCancelEvent.rawData = eventLog.data as string; exchangeCancelEvent.transactionHash = eventLog.transactionHash; - exchangeCancelEvent.makerAddress = eventLog.args.makerAddress.toString(); - exchangeCancelEvent.takerAddress = - eventLog.args.takerAddress == null ? null : eventLog.args.takerAddress.toString(); + exchangeCancelEvent.makerAddress = eventLog.args.makerAddress; + exchangeCancelEvent.takerAddress = eventLog.args.takerAddress; exchangeCancelEvent.feeRecipientAddress = eventLog.args.feeRecipientAddress; exchangeCancelEvent.senderAddress = eventLog.args.senderAddress; exchangeCancelEvent.orderHash = eventLog.args.orderHash; @@ -127,8 +126,8 @@ export function _convertToExchangeCancelUpToEvent( exchangeCancelUpToEvent.logIndex = eventLog.logIndex as number; exchangeCancelUpToEvent.rawData = eventLog.data as string; exchangeCancelUpToEvent.transactionHash = eventLog.transactionHash; - exchangeCancelUpToEvent.makerAddress = eventLog.args.makerAddress.toString(); - exchangeCancelUpToEvent.senderAddress = eventLog.args.senderAddress.toString(); + exchangeCancelUpToEvent.makerAddress = eventLog.args.makerAddress; + exchangeCancelUpToEvent.senderAddress = eventLog.args.senderAddress; exchangeCancelUpToEvent.orderEpoch = eventLog.args.orderEpoch; return exchangeCancelUpToEvent; } diff --git a/packages/pipeline/src/parsers/relayer_registry/index.ts b/packages/pipeline/src/parsers/relayer_registry/index.ts index 7b1235005..9723880a4 100644 --- a/packages/pipeline/src/parsers/relayer_registry/index.ts +++ b/packages/pipeline/src/parsers/relayer_registry/index.ts @@ -18,12 +18,13 @@ function parseRelayer(relayerResp: RelayerResponse, uuid: string): Relayer { relayer.name = relayerResp.name; relayer.homepageUrl = relayerResp.homepage_url; relayer.appUrl = relayerResp.app_url; - const mainnet = getMainNetwork(relayerResp); - if (mainnet !== undefined) { - relayer.sraHttpEndpoint = mainnet.sra_http_endpoint || null; - relayer.sraWsEndpoint = mainnet.sra_ws_endpoint || null; - relayer.feeRecipientAddresses = R.path(['static_order_fields', 'fee_recipient_addresses'], mainnet) || []; - relayer.takerAddresses = R.path(['static_order_fields', 'taker_addresses'], mainnet) || []; + const mainNetworkRelayerInfo = getMainNetwork(relayerResp); + if (mainNetworkRelayerInfo !== undefined) { + relayer.sraHttpEndpoint = mainNetworkRelayerInfo.sra_http_endpoint || null; + relayer.sraWsEndpoint = mainNetworkRelayerInfo.sra_ws_endpoint || null; + relayer.feeRecipientAddresses = + R.path(['static_order_fields', 'fee_recipient_addresses'], mainNetworkRelayerInfo) || []; + relayer.takerAddresses = R.path(['static_order_fields', 'taker_addresses'], mainNetworkRelayerInfo) || []; } else { relayer.feeRecipientAddresses = []; relayer.takerAddresses = []; diff --git a/packages/pipeline/src/parsers/token_metadata/index.ts b/packages/pipeline/src/parsers/token_metadata/index.ts index 916448cb3..3b3e05d76 100644 --- a/packages/pipeline/src/parsers/token_metadata/index.ts +++ b/packages/pipeline/src/parsers/token_metadata/index.ts @@ -1,7 +1,9 @@ +import { BigNumber } from '@0x/utils'; import * as R from 'ramda'; import { MetamaskTrustedTokenMeta, ZeroExTrustedTokenMeta } from '../../data_sources/trusted_tokens'; import { TokenMetadata } from '../../entities'; +import {} from '../../utils'; /** * Parses Metamask's trusted tokens list. @@ -24,7 +26,7 @@ function parseMetamaskTrustedToken(resp: MetamaskTrustedTokenMeta, address: stri const trustedToken = new TokenMetadata(); trustedToken.address = address; - trustedToken.decimals = resp.decimals; + trustedToken.decimals = new BigNumber(resp.decimals); trustedToken.symbol = resp.symbol; trustedToken.name = resp.name; trustedToken.authority = 'metamask'; @@ -36,7 +38,7 @@ function parseZeroExTrustedToken(resp: ZeroExTrustedTokenMeta): TokenMetadata { const trustedToken = new TokenMetadata(); trustedToken.address = resp.address; - trustedToken.decimals = resp.decimals; + trustedToken.decimals = new BigNumber(resp.decimals); trustedToken.symbol = resp.symbol; trustedToken.name = resp.name; trustedToken.authority = '0x'; diff --git a/packages/pipeline/src/parsers/web3/index.ts b/packages/pipeline/src/parsers/web3/index.ts index 9b5b3b55d..86f924151 100644 --- a/packages/pipeline/src/parsers/web3/index.ts +++ b/packages/pipeline/src/parsers/web3/index.ts @@ -42,7 +42,6 @@ export function parseTransaction(rawTransaction: EthTransaction): Transaction { tx.blockNumber = rawTransaction.blockNumber; tx.gasUsed = rawTransaction.gas; - // TODO(albrow) figure out bignum solution. tx.gasPrice = rawTransaction.gasPrice.toNumber(); return tx; diff --git a/packages/pipeline/src/scripts/pull_missing_blocks.ts b/packages/pipeline/src/scripts/pull_missing_blocks.ts index 4a1483ab9..b7bd51f08 100644 --- a/packages/pipeline/src/scripts/pull_missing_blocks.ts +++ b/packages/pipeline/src/scripts/pull_missing_blocks.ts @@ -9,7 +9,7 @@ import { Web3Source } from '../data_sources/web3'; import { Block } from '../entities'; import * as ormConfig from '../ormconfig'; import { parseBlock } from '../parsers/web3'; -import { handleError } from '../utils'; +import { EXCHANGE_START_BLOCK, handleError, INFURA_ROOT_URL } from '../utils'; // Number of blocks to save at once. const BATCH_SAVE_SIZE = 1000; @@ -18,16 +18,13 @@ const MAX_CONCURRENCY = 10; // Maximum number of blocks to query for at once. This is also the maximum // number of blocks we will hold in memory prior to being saved to the database. const MAX_BLOCKS_PER_QUERY = 1000; -// Block number when the Exchange contract was deployed to mainnet. -// TODO(albrow): De-dupe this constant. -const EXCHANGE_START_BLOCK = 6271590; let connection: Connection; (async () => { connection = await createConnection(ormConfig as ConnectionOptions); const provider = web3Factory.getRpcProvider({ - rpcUrl: `https://mainnet.infura.io/${process.env.INFURA_API_KEY}`, + rpcUrl: `${INFURA_ROOT_URL}/${process.env.INFURA_API_KEY}`, }); const web3Source = new Web3Source(provider); await getAllMissingBlocks(web3Source); diff --git a/packages/pipeline/src/scripts/pull_missing_events.ts b/packages/pipeline/src/scripts/pull_missing_events.ts index c7e07c540..80abbb8b0 100644 --- a/packages/pipeline/src/scripts/pull_missing_events.ts +++ b/packages/pipeline/src/scripts/pull_missing_events.ts @@ -8,9 +8,8 @@ import { ExchangeEventsSource } from '../data_sources/contract-wrappers/exchange import { ExchangeCancelEvent, ExchangeCancelUpToEvent, ExchangeEvent, ExchangeFillEvent } from '../entities'; import * as ormConfig from '../ormconfig'; import { parseExchangeCancelEvents, parseExchangeCancelUpToEvents, parseExchangeFillEvents } from '../parsers/events'; -import { handleError } from '../utils'; +import { EXCHANGE_START_BLOCK, handleError, INFURA_ROOT_URL } from '../utils'; -const EXCHANGE_START_BLOCK = 6271590; // Block number when the Exchange contract was deployed to mainnet. const START_BLOCK_OFFSET = 100; // Number of blocks before the last known block to consider when updating fill events. const BATCH_SAVE_SIZE = 1000; // Number of events to save at once. @@ -19,7 +18,7 @@ let connection: Connection; (async () => { connection = await createConnection(ormConfig as ConnectionOptions); const provider = web3Factory.getRpcProvider({ - rpcUrl: 'https://mainnet.infura.io', + rpcUrl: INFURA_ROOT_URL, }); const eventsSource = new ExchangeEventsSource(provider, 1); await getFillEventsAsync(eventsSource); diff --git a/packages/pipeline/src/scripts/pull_radar_relay_orders.ts b/packages/pipeline/src/scripts/pull_radar_relay_orders.ts index b3a4d887e..bbbef9b47 100644 --- a/packages/pipeline/src/scripts/pull_radar_relay_orders.ts +++ b/packages/pipeline/src/scripts/pull_radar_relay_orders.ts @@ -16,11 +16,11 @@ let connection: Connection; (async () => { connection = await createConnection(ormConfig as ConnectionOptions); - await getOrderbook(); + await getOrderbookAsync(); process.exit(0); })().catch(handleError); -async function getOrderbook(): Promise<void> { +async function getOrderbookAsync(): Promise<void> { console.log('Getting all orders...'); const connectClient = new HttpClient(RADAR_RELAY_URL); const rawOrders = await connectClient.getOrdersAsync({ @@ -29,17 +29,22 @@ async function getOrderbook(): Promise<void> { console.log(`Got ${rawOrders.records.length} orders.`); console.log('Parsing orders...'); // Parse the sra orders, then add source url to each. - const orders = R.pipe(parseSraOrders, R.map(setSourceUrl(RADAR_RELAY_URL)))(rawOrders); + const orders = R.pipe( + parseSraOrders, + R.map(setSourceUrl(RADAR_RELAY_URL)), + )(rawOrders); // Save all the orders and update the observed time stamps in a single // transaction. console.log('Saving orders and updating timestamps...'); - await connection.transaction(async (manager: EntityManager): Promise<void> => { - for (const order of orders) { - await manager.save(SraOrder, order); - const observedTimestamp = createObservedTimestampForOrder(order); - await manager.save(observedTimestamp); - } - }); + await connection.transaction( + async (manager: EntityManager): Promise<void> => { + for (const order of orders) { + await manager.save(SraOrder, order); + const observedTimestamp = createObservedTimestampForOrder(order); + await manager.save(observedTimestamp); + } + }, + ); } const sourceUrlProp = R.lensProp('sourceUrl'); @@ -48,6 +53,8 @@ const sourceUrlProp = R.lensProp('sourceUrl'); * Sets the source url for a single order. Returns a new order instead of * mutating the given one. */ -const setSourceUrl = R.curry((sourceURL: string, order: SraOrder): SraOrder => { - return R.set(sourceUrlProp, sourceURL, order); -}); +const setSourceUrl = R.curry( + (sourceURL: string, order: SraOrder): SraOrder => { + return R.set(sourceUrlProp, sourceURL, order); + }, +); diff --git a/packages/pipeline/src/scripts/pull_trusted_tokens.ts b/packages/pipeline/src/scripts/pull_trusted_tokens.ts index b3a4466bb..1befc4437 100644 --- a/packages/pipeline/src/scripts/pull_trusted_tokens.ts +++ b/packages/pipeline/src/scripts/pull_trusted_tokens.ts @@ -8,7 +8,7 @@ import { parseMetamaskTrustedTokens, parseZeroExTrustedTokens } from '../parsers import { handleError } from '../utils'; const METAMASK_TRUSTED_TOKENS_URL = - 'https://raw.githubusercontent.com/MetaMask/eth-contract-metadata/master/contract-map.json'; + 'https://raw.githubusercontent.com/MetaMask/eth-contract-metadata/d45916c533116510cc8e9e048a8b5fc3732a6b6d/contract-map.json'; const ZEROEX_TRUSTED_TOKENS_URL = 'https://website-api.0xproject.com/tokens'; @@ -22,7 +22,7 @@ let connection: Connection; })().catch(handleError); async function getMetamaskTrustedTokens(): Promise<void> { - // tslint:disable-next-line + // tslint:disable-next-line:no-console console.log('Getting latest metamask trusted tokens list ...'); const trustedTokensRepository = connection.getRepository(TokenMetadata); const trustedTokensSource = new TrustedTokenSource<Map<string, MetamaskTrustedTokenMeta>>( @@ -30,23 +30,23 @@ async function getMetamaskTrustedTokens(): Promise<void> { ); const resp = await trustedTokensSource.getTrustedTokenMetaAsync(); const trustedTokens = parseMetamaskTrustedTokens(resp); - // tslint:disable-next-line + // tslint:disable-next-line:no-console console.log('Saving metamask trusted tokens list'); await trustedTokensRepository.save(trustedTokens); - // tslint:disable-next-line + // tslint:disable-next-line:no-console console.log('Done saving metamask trusted tokens.'); } async function getZeroExTrustedTokens(): Promise<void> { - // tslint:disable-next-line + // tslint:disable-next-line:no-console console.log('Getting latest 0x trusted tokens list ...'); const trustedTokensRepository = connection.getRepository(TokenMetadata); const trustedTokensSource = new TrustedTokenSource<ZeroExTrustedTokenMeta[]>(ZEROEX_TRUSTED_TOKENS_URL); const resp = await trustedTokensSource.getTrustedTokenMetaAsync(); const trustedTokens = parseZeroExTrustedTokens(resp); - // tslint:disable-next-line + // tslint:disable-next-line:no-console console.log('Saving metamask trusted tokens list'); await trustedTokensRepository.save(trustedTokens); - // tslint:disable-next-line + // tslint:disable-next-line:no-console console.log('Done saving metamask trusted tokens.'); } diff --git a/packages/pipeline/src/utils/constants.ts b/packages/pipeline/src/utils/constants.ts new file mode 100644 index 000000000..56f3e82d8 --- /dev/null +++ b/packages/pipeline/src/utils/constants.ts @@ -0,0 +1,3 @@ +// Block number when the Exchange contract was deployed to mainnet. +export const EXCHANGE_START_BLOCK = 6271590; +export const INFURA_ROOT_URL = 'https://mainnet.infura.io'; diff --git a/packages/pipeline/src/utils/index.ts b/packages/pipeline/src/utils/index.ts index 718ea6133..2096a0a39 100644 --- a/packages/pipeline/src/utils/index.ts +++ b/packages/pipeline/src/utils/index.ts @@ -1,5 +1,6 @@ import { BigNumber } from '@0x/utils'; export * from './transformers'; +export * from './constants'; /** * If the given BigNumber is not null, returns the string representation of that diff --git a/packages/pipeline/src/utils/transformers/big_number.ts b/packages/pipeline/src/utils/transformers/big_number.ts index a0471a5e3..5f2e4d565 100644 --- a/packages/pipeline/src/utils/transformers/big_number.ts +++ b/packages/pipeline/src/utils/transformers/big_number.ts @@ -3,13 +3,13 @@ import { ValueTransformer } from 'typeorm/decorator/options/ValueTransformer'; export class BigNumberTransformer implements ValueTransformer { // tslint:disable-next-line:prefer-function-over-method - public to(value: BigNumber): string { - return value.toString(); + public to(value: BigNumber | null): string | null { + return value === null ? null : value.toString(); } // tslint:disable-next-line:prefer-function-over-method - public from(value: string): BigNumber { - return new BigNumber(value); + public from(value: string | null): BigNumber | null { + return value === null ? null : new BigNumber(value); } } diff --git a/packages/pipeline/test/entities/token_metadata_test.ts b/packages/pipeline/test/entities/token_metadata_test.ts index 805b4b0f7..48e656644 100644 --- a/packages/pipeline/test/entities/token_metadata_test.ts +++ b/packages/pipeline/test/entities/token_metadata_test.ts @@ -1,3 +1,4 @@ +import { BigNumber } from '@0x/utils'; import 'mocha'; import 'reflect-metadata'; @@ -9,15 +10,15 @@ import { testSaveAndFindEntityAsync } from './util'; chaiSetup.configure(); -const metadataWithoutNullFields = { +const metadataWithoutNullFields: TokenMetadata = { address: '0xe41d2489571d322189246dafa5ebde1f4699f498', authority: 'https://website-api.0xproject.com/tokens', - decimals: 18, + decimals: new BigNumber(18), symbol: 'ZRX', name: '0x', }; -const metadataWithNullFields = { +const metadataWithNullFields: TokenMetadata = { address: '0xe41d2489571d322189246dafa5ebde1f4699f499', authority: 'https://website-api.0xproject.com/tokens', decimals: null, |