From 00f86ca0f7871639d2b0be496f6f8c5e0d8d7ffe Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Tue, 4 Dec 2018 20:04:08 -0800 Subject: Address PR feedback --- .../contract-wrappers/exchange_events.ts | 3 +- .../pipeline/src/entities/exchange_cancel_event.ts | 2 -- .../src/entities/exchange_cancel_up_to_event.ts | 2 -- .../pipeline/src/entities/exchange_fill_event.ts | 2 -- packages/pipeline/src/entities/relayer.ts | 3 -- packages/pipeline/src/entities/token_metadata.ts | 9 +++--- packages/pipeline/src/parsers/events/index.ts | 13 ++++----- .../pipeline/src/parsers/relayer_registry/index.ts | 13 +++++---- .../pipeline/src/parsers/token_metadata/index.ts | 6 ++-- packages/pipeline/src/parsers/web3/index.ts | 1 - .../pipeline/src/scripts/pull_missing_blocks.ts | 7 ++--- .../pipeline/src/scripts/pull_missing_events.ts | 5 ++-- .../src/scripts/pull_radar_relay_orders.ts | 33 +++++++++++++--------- .../pipeline/src/scripts/pull_trusted_tokens.ts | 14 ++++----- packages/pipeline/src/utils/constants.ts | 3 ++ packages/pipeline/src/utils/index.ts | 1 + .../pipeline/src/utils/transformers/big_number.ts | 8 +++--- 17 files changed, 63 insertions(+), 62 deletions(-) create mode 100644 packages/pipeline/src/utils/constants.ts (limited to 'packages/pipeline/src') diff --git a/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts b/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts index 5721c0ea3..1717eb8b3 100644 --- a/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts +++ b/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts @@ -11,9 +11,10 @@ import { Web3ProviderEngine } from '@0x/subproviders'; import { Web3Wrapper } from '@0x/web3-wrapper'; import { LogWithDecodedArgs } from 'ethereum-types'; +import { EXCHANGE_START_BLOCK } from '../../utils'; + const BLOCK_FINALITY_THRESHOLD = 10; // When to consider blocks as final. Used to compute default toBlock. const NUM_BLOCKS_PER_QUERY = 20000; // Number of blocks to query for events at a time. -const EXCHANGE_START_BLOCK = 6271590; // Block number when the Exchange contract was deployed to mainnet. export class ExchangeEventsSource { private readonly _exchangeWrapper: ExchangeWrapper; diff --git a/packages/pipeline/src/entities/exchange_cancel_event.ts b/packages/pipeline/src/entities/exchange_cancel_event.ts index 2fcc17df6..38f99c903 100644 --- a/packages/pipeline/src/entities/exchange_cancel_event.ts +++ b/packages/pipeline/src/entities/exchange_cancel_event.ts @@ -48,6 +48,4 @@ export class ExchangeCancelEvent { public takerTokenAddress!: string; @Column({ nullable: true, type: String, name: 'taker_token_id' }) public takerTokenId!: string | null; - - // TODO(albrow): Include topics? } diff --git a/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts b/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts index 60ead324f..27580305e 100644 --- a/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts +++ b/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts @@ -12,7 +12,6 @@ export class ExchangeCancelUpToEvent { @PrimaryColumn({ name: 'block_number', transformer: numberToBigIntTransformer }) public blockNumber!: number; - // TODO(albrow): Include transaction hash @Column({ name: 'raw_data' }) public rawData!: string; @@ -24,5 +23,4 @@ export class ExchangeCancelUpToEvent { public senderAddress!: string; @Column({ name: 'order_epoch', type: 'numeric', transformer: bigNumberTransformer }) public orderEpoch!: BigNumber; - // TODO(albrow): Include topics? } diff --git a/packages/pipeline/src/entities/exchange_fill_event.ts b/packages/pipeline/src/entities/exchange_fill_event.ts index bbf0abf58..9b7727615 100644 --- a/packages/pipeline/src/entities/exchange_fill_event.ts +++ b/packages/pipeline/src/entities/exchange_fill_event.ts @@ -57,6 +57,4 @@ export class ExchangeFillEvent { public takerTokenAddress!: string; @Column({ nullable: true, type: String, name: 'taker_token_id' }) public takerTokenId!: string | null; - - // TODO(albrow): Include topics? } diff --git a/packages/pipeline/src/entities/relayer.ts b/packages/pipeline/src/entities/relayer.ts index b3a856fd8..5af8578b4 100644 --- a/packages/pipeline/src/entities/relayer.ts +++ b/packages/pipeline/src/entities/relayer.ts @@ -14,9 +14,6 @@ export class Relayer { @Column({ name: 'app_url', type: 'varchar', nullable: true }) public appUrl!: string | null; - // TODO(albrow): Add exchange contract or protocol version? - // TODO(albrow): Add network ids for addresses? - @Column({ name: 'fee_recipient_addresses', type: 'varchar', array: true }) public feeRecipientAddresses!: string[]; @Column({ name: 'taker_addresses', type: 'varchar', array: true }) diff --git a/packages/pipeline/src/entities/token_metadata.ts b/packages/pipeline/src/entities/token_metadata.ts index ca1e57937..911b53972 100644 --- a/packages/pipeline/src/entities/token_metadata.ts +++ b/packages/pipeline/src/entities/token_metadata.ts @@ -1,5 +1,8 @@ +import { BigNumber } from '@0x/utils'; import { Column, Entity, PrimaryColumn } from 'typeorm'; +import { bigNumberTransformer } from '../utils/transformers'; + @Entity({ name: 'token_metadata', schema: 'raw' }) export class TokenMetadata { @PrimaryColumn({ type: 'varchar', nullable: false }) @@ -8,10 +11,8 @@ export class TokenMetadata { @PrimaryColumn({ type: 'varchar', nullable: false }) public authority!: string; - // TODO(albrow): Convert decimals field to type BigNumber/numeric because it - // comes from a 256-bit integer in a smart contract. - @Column({ type: 'integer', nullable: true }) - public decimals!: number | null; + @Column({ type: 'numeric', transformer: bigNumberTransformer, nullable: true }) + public decimals!: BigNumber | null; @Column({ type: 'varchar', nullable: true }) public symbol!: string | null; diff --git a/packages/pipeline/src/parsers/events/index.ts b/packages/pipeline/src/parsers/events/index.ts index d42d1c57a..e18106c75 100644 --- a/packages/pipeline/src/parsers/events/index.ts +++ b/packages/pipeline/src/parsers/events/index.ts @@ -49,8 +49,8 @@ export function _convertToExchangeFillEvent(eventLog: LogWithDecodedArgs { connection = await createConnection(ormConfig as ConnectionOptions); const provider = web3Factory.getRpcProvider({ - rpcUrl: `https://mainnet.infura.io/${process.env.INFURA_API_KEY}`, + rpcUrl: `${INFURA_ROOT_URL}/${process.env.INFURA_API_KEY}`, }); const web3Source = new Web3Source(provider); await getAllMissingBlocks(web3Source); diff --git a/packages/pipeline/src/scripts/pull_missing_events.ts b/packages/pipeline/src/scripts/pull_missing_events.ts index c7e07c540..80abbb8b0 100644 --- a/packages/pipeline/src/scripts/pull_missing_events.ts +++ b/packages/pipeline/src/scripts/pull_missing_events.ts @@ -8,9 +8,8 @@ import { ExchangeEventsSource } from '../data_sources/contract-wrappers/exchange import { ExchangeCancelEvent, ExchangeCancelUpToEvent, ExchangeEvent, ExchangeFillEvent } from '../entities'; import * as ormConfig from '../ormconfig'; import { parseExchangeCancelEvents, parseExchangeCancelUpToEvents, parseExchangeFillEvents } from '../parsers/events'; -import { handleError } from '../utils'; +import { EXCHANGE_START_BLOCK, handleError, INFURA_ROOT_URL } from '../utils'; -const EXCHANGE_START_BLOCK = 6271590; // Block number when the Exchange contract was deployed to mainnet. const START_BLOCK_OFFSET = 100; // Number of blocks before the last known block to consider when updating fill events. const BATCH_SAVE_SIZE = 1000; // Number of events to save at once. @@ -19,7 +18,7 @@ let connection: Connection; (async () => { connection = await createConnection(ormConfig as ConnectionOptions); const provider = web3Factory.getRpcProvider({ - rpcUrl: 'https://mainnet.infura.io', + rpcUrl: INFURA_ROOT_URL, }); const eventsSource = new ExchangeEventsSource(provider, 1); await getFillEventsAsync(eventsSource); diff --git a/packages/pipeline/src/scripts/pull_radar_relay_orders.ts b/packages/pipeline/src/scripts/pull_radar_relay_orders.ts index b3a4d887e..bbbef9b47 100644 --- a/packages/pipeline/src/scripts/pull_radar_relay_orders.ts +++ b/packages/pipeline/src/scripts/pull_radar_relay_orders.ts @@ -16,11 +16,11 @@ let connection: Connection; (async () => { connection = await createConnection(ormConfig as ConnectionOptions); - await getOrderbook(); + await getOrderbookAsync(); process.exit(0); })().catch(handleError); -async function getOrderbook(): Promise { +async function getOrderbookAsync(): Promise { console.log('Getting all orders...'); const connectClient = new HttpClient(RADAR_RELAY_URL); const rawOrders = await connectClient.getOrdersAsync({ @@ -29,17 +29,22 @@ async function getOrderbook(): Promise { console.log(`Got ${rawOrders.records.length} orders.`); console.log('Parsing orders...'); // Parse the sra orders, then add source url to each. - const orders = R.pipe(parseSraOrders, R.map(setSourceUrl(RADAR_RELAY_URL)))(rawOrders); + const orders = R.pipe( + parseSraOrders, + R.map(setSourceUrl(RADAR_RELAY_URL)), + )(rawOrders); // Save all the orders and update the observed time stamps in a single // transaction. console.log('Saving orders and updating timestamps...'); - await connection.transaction(async (manager: EntityManager): Promise => { - for (const order of orders) { - await manager.save(SraOrder, order); - const observedTimestamp = createObservedTimestampForOrder(order); - await manager.save(observedTimestamp); - } - }); + await connection.transaction( + async (manager: EntityManager): Promise => { + for (const order of orders) { + await manager.save(SraOrder, order); + const observedTimestamp = createObservedTimestampForOrder(order); + await manager.save(observedTimestamp); + } + }, + ); } const sourceUrlProp = R.lensProp('sourceUrl'); @@ -48,6 +53,8 @@ const sourceUrlProp = R.lensProp('sourceUrl'); * Sets the source url for a single order. Returns a new order instead of * mutating the given one. */ -const setSourceUrl = R.curry((sourceURL: string, order: SraOrder): SraOrder => { - return R.set(sourceUrlProp, sourceURL, order); -}); +const setSourceUrl = R.curry( + (sourceURL: string, order: SraOrder): SraOrder => { + return R.set(sourceUrlProp, sourceURL, order); + }, +); diff --git a/packages/pipeline/src/scripts/pull_trusted_tokens.ts b/packages/pipeline/src/scripts/pull_trusted_tokens.ts index b3a4466bb..1befc4437 100644 --- a/packages/pipeline/src/scripts/pull_trusted_tokens.ts +++ b/packages/pipeline/src/scripts/pull_trusted_tokens.ts @@ -8,7 +8,7 @@ import { parseMetamaskTrustedTokens, parseZeroExTrustedTokens } from '../parsers import { handleError } from '../utils'; const METAMASK_TRUSTED_TOKENS_URL = - 'https://raw.githubusercontent.com/MetaMask/eth-contract-metadata/master/contract-map.json'; + 'https://raw.githubusercontent.com/MetaMask/eth-contract-metadata/d45916c533116510cc8e9e048a8b5fc3732a6b6d/contract-map.json'; const ZEROEX_TRUSTED_TOKENS_URL = 'https://website-api.0xproject.com/tokens'; @@ -22,7 +22,7 @@ let connection: Connection; })().catch(handleError); async function getMetamaskTrustedTokens(): Promise { - // tslint:disable-next-line + // tslint:disable-next-line:no-console console.log('Getting latest metamask trusted tokens list ...'); const trustedTokensRepository = connection.getRepository(TokenMetadata); const trustedTokensSource = new TrustedTokenSource>( @@ -30,23 +30,23 @@ async function getMetamaskTrustedTokens(): Promise { ); const resp = await trustedTokensSource.getTrustedTokenMetaAsync(); const trustedTokens = parseMetamaskTrustedTokens(resp); - // tslint:disable-next-line + // tslint:disable-next-line:no-console console.log('Saving metamask trusted tokens list'); await trustedTokensRepository.save(trustedTokens); - // tslint:disable-next-line + // tslint:disable-next-line:no-console console.log('Done saving metamask trusted tokens.'); } async function getZeroExTrustedTokens(): Promise { - // tslint:disable-next-line + // tslint:disable-next-line:no-console console.log('Getting latest 0x trusted tokens list ...'); const trustedTokensRepository = connection.getRepository(TokenMetadata); const trustedTokensSource = new TrustedTokenSource(ZEROEX_TRUSTED_TOKENS_URL); const resp = await trustedTokensSource.getTrustedTokenMetaAsync(); const trustedTokens = parseZeroExTrustedTokens(resp); - // tslint:disable-next-line + // tslint:disable-next-line:no-console console.log('Saving metamask trusted tokens list'); await trustedTokensRepository.save(trustedTokens); - // tslint:disable-next-line + // tslint:disable-next-line:no-console console.log('Done saving metamask trusted tokens.'); } diff --git a/packages/pipeline/src/utils/constants.ts b/packages/pipeline/src/utils/constants.ts new file mode 100644 index 000000000..56f3e82d8 --- /dev/null +++ b/packages/pipeline/src/utils/constants.ts @@ -0,0 +1,3 @@ +// Block number when the Exchange contract was deployed to mainnet. +export const EXCHANGE_START_BLOCK = 6271590; +export const INFURA_ROOT_URL = 'https://mainnet.infura.io'; diff --git a/packages/pipeline/src/utils/index.ts b/packages/pipeline/src/utils/index.ts index 718ea6133..2096a0a39 100644 --- a/packages/pipeline/src/utils/index.ts +++ b/packages/pipeline/src/utils/index.ts @@ -1,5 +1,6 @@ import { BigNumber } from '@0x/utils'; export * from './transformers'; +export * from './constants'; /** * If the given BigNumber is not null, returns the string representation of that diff --git a/packages/pipeline/src/utils/transformers/big_number.ts b/packages/pipeline/src/utils/transformers/big_number.ts index a0471a5e3..5f2e4d565 100644 --- a/packages/pipeline/src/utils/transformers/big_number.ts +++ b/packages/pipeline/src/utils/transformers/big_number.ts @@ -3,13 +3,13 @@ import { ValueTransformer } from 'typeorm/decorator/options/ValueTransformer'; export class BigNumberTransformer implements ValueTransformer { // tslint:disable-next-line:prefer-function-over-method - public to(value: BigNumber): string { - return value.toString(); + public to(value: BigNumber | null): string | null { + return value === null ? null : value.toString(); } // tslint:disable-next-line:prefer-function-over-method - public from(value: string): BigNumber { - return new BigNumber(value); + public from(value: string | null): BigNumber | null { + return value === null ? null : new BigNumber(value); } } -- cgit v1.2.3