aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAlex Browne <stephenalexbrowne@gmail.com>2018-12-05 12:04:08 +0800
committerAlex Browne <stephenalexbrowne@gmail.com>2018-12-05 12:04:08 +0800
commit00f86ca0f7871639d2b0be496f6f8c5e0d8d7ffe (patch)
treee2e1c5394deac225bf493b3fe51da9f3074f1691
parent549f5e4655f246062dd6451065ec01eb789dbd8f (diff)
downloaddexon-sol-tools-00f86ca0f7871639d2b0be496f6f8c5e0d8d7ffe.tar
dexon-sol-tools-00f86ca0f7871639d2b0be496f6f8c5e0d8d7ffe.tar.gz
dexon-sol-tools-00f86ca0f7871639d2b0be496f6f8c5e0d8d7ffe.tar.bz2
dexon-sol-tools-00f86ca0f7871639d2b0be496f6f8c5e0d8d7ffe.tar.lz
dexon-sol-tools-00f86ca0f7871639d2b0be496f6f8c5e0d8d7ffe.tar.xz
dexon-sol-tools-00f86ca0f7871639d2b0be496f6f8c5e0d8d7ffe.tar.zst
dexon-sol-tools-00f86ca0f7871639d2b0be496f6f8c5e0d8d7ffe.zip
Address PR feedback
-rw-r--r--packages/pipeline/README.md21
-rw-r--r--packages/pipeline/migrations/1543980079179-ConvertTokenMetadataDecimalsToBigNumber.ts17
-rw-r--r--packages/pipeline/package.json7
-rw-r--r--packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts3
-rw-r--r--packages/pipeline/src/entities/exchange_cancel_event.ts2
-rw-r--r--packages/pipeline/src/entities/exchange_cancel_up_to_event.ts2
-rw-r--r--packages/pipeline/src/entities/exchange_fill_event.ts2
-rw-r--r--packages/pipeline/src/entities/relayer.ts3
-rw-r--r--packages/pipeline/src/entities/token_metadata.ts9
-rw-r--r--packages/pipeline/src/parsers/events/index.ts13
-rw-r--r--packages/pipeline/src/parsers/relayer_registry/index.ts13
-rw-r--r--packages/pipeline/src/parsers/token_metadata/index.ts6
-rw-r--r--packages/pipeline/src/parsers/web3/index.ts1
-rw-r--r--packages/pipeline/src/scripts/pull_missing_blocks.ts7
-rw-r--r--packages/pipeline/src/scripts/pull_missing_events.ts5
-rw-r--r--packages/pipeline/src/scripts/pull_radar_relay_orders.ts33
-rw-r--r--packages/pipeline/src/scripts/pull_trusted_tokens.ts14
-rw-r--r--packages/pipeline/src/utils/constants.ts3
-rw-r--r--packages/pipeline/src/utils/index.ts1
-rw-r--r--packages/pipeline/src/utils/transformers/big_number.ts8
-rw-r--r--packages/pipeline/test/entities/token_metadata_test.ts7
21 files changed, 96 insertions, 81 deletions
diff --git a/packages/pipeline/README.md b/packages/pipeline/README.md
index fb563b14c..0f4abd935 100644
--- a/packages/pipeline/README.md
+++ b/packages/pipeline/README.md
@@ -42,7 +42,8 @@ Revert the most recent migration (CAUTION: may result in data loss!): `yarn migr
There are several test scripts in **package.json**. You can run all the tests
with `yarn test:all` or run certain tests seprately by following the
-instructions below. Some tests may not work out of the box on certain platforms.
+instructions below. Some tests may not work out of the box on certain platforms
+or operating systems (see the "Database tests" section below).
### Unit tests
@@ -71,8 +72,8 @@ Postgres is via Docker. Depending on your platform, you may need to prepend
docker run --rm -d -p 5432:5432 --name pipeline_postgres postgres:11-alpine
```
-This will start a Postgres server with the default username and database name.
-You should set the environment variable as follows:
+This will start a Postgres server with the default username and database name
+(`postgres` and `postgres`). You should set the environment variable as follows:
```
export ZEROEX_DATA_PIPELINE_DB_URL=postgresql://postgres@localhost/postgres
@@ -149,17 +150,17 @@ set the`ZEROEX_DATA_PIPELINE_DB_URL` environment variable to a valid
#### Additional guidelines and tips:
-* Table names should be plural and separated by underscores (e.g.,
+- Table names should be plural and separated by underscores (e.g.,
`exchange_fill_events`).
-* Any table which contains data which comes directly from a third-party source
+- Any table which contains data which comes directly from a third-party source
should be namespaced in the `raw` PostgreSQL schema.
-* Column names in the database should be separated by underscores (e.g.,
+- Column names in the database should be separated by underscores (e.g.,
`maker_asset_type`).
-* Field names in entity classes (like any other fields in TypeScript) should
+- Field names in entity classes (like any other fields in TypeScript) should
be camel-cased (e.g., `makerAssetType`).
-* All timestamps should be stored as milliseconds since the Unix Epoch.
-* Use the `BigNumber` type for TypeScript code which deals with 256-bit
+- All timestamps should be stored as milliseconds since the Unix Epoch.
+- Use the `BigNumber` type for TypeScript code which deals with 256-bit
numbers from smart contracts or for any case where we are dealing with large
floating point numbers.
-* [TypeORM documentation](http://typeorm.io/#/) is pretty robust and can be a
+- [TypeORM documentation](http://typeorm.io/#/) is pretty robust and can be a
helpful resource.
diff --git a/packages/pipeline/migrations/1543980079179-ConvertTokenMetadataDecimalsToBigNumber.ts b/packages/pipeline/migrations/1543980079179-ConvertTokenMetadataDecimalsToBigNumber.ts
new file mode 100644
index 000000000..351bc7eb8
--- /dev/null
+++ b/packages/pipeline/migrations/1543980079179-ConvertTokenMetadataDecimalsToBigNumber.ts
@@ -0,0 +1,17 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class ConvertTokenMetadataDecimalsToBigNumber1543980079179 implements MigrationInterface {
+ public async up(queryRunner: QueryRunner): Promise<any> {
+ await queryRunner.query(
+ `ALTER TABLE raw.token_metadata
+ ALTER COLUMN decimals TYPE numeric USING decimals::numeric;`,
+ );
+ }
+
+ public async down(queryRunner: QueryRunner): Promise<any> {
+ await queryRunner.query(
+ `ALTER TABLE raw.token_metadata
+ ALTER COLUMN decimals TYPE numeric USING decimals::integer;`,
+ );
+ }
+}
diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json
index 6af81c048..0539618d4 100644
--- a/packages/pipeline/package.json
+++ b/packages/pipeline/package.json
@@ -1,6 +1,6 @@
{
"name": "@0x/pipeline",
- "version": "0.0.1",
+ "version": "1.0.0",
"private": true,
"description": "Data pipeline for offline analysis",
"scripts": {
@@ -21,11 +21,6 @@
"migrate:revert": "yarn typeorm migration:revert --config ./lib/src/ormconfig",
"migrate:create": "yarn typeorm migration:create --config ./lib/src/ormconfig --dir migrations"
},
- "config": {
- "postpublish": {
- "assets": []
- }
- },
"repository": {
"type": "git",
"url": "https://github.com/0xProject/0x-monorepo"
diff --git a/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts b/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts
index 5721c0ea3..1717eb8b3 100644
--- a/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts
+++ b/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts
@@ -11,9 +11,10 @@ import { Web3ProviderEngine } from '@0x/subproviders';
import { Web3Wrapper } from '@0x/web3-wrapper';
import { LogWithDecodedArgs } from 'ethereum-types';
+import { EXCHANGE_START_BLOCK } from '../../utils';
+
const BLOCK_FINALITY_THRESHOLD = 10; // When to consider blocks as final. Used to compute default toBlock.
const NUM_BLOCKS_PER_QUERY = 20000; // Number of blocks to query for events at a time.
-const EXCHANGE_START_BLOCK = 6271590; // Block number when the Exchange contract was deployed to mainnet.
export class ExchangeEventsSource {
private readonly _exchangeWrapper: ExchangeWrapper;
diff --git a/packages/pipeline/src/entities/exchange_cancel_event.ts b/packages/pipeline/src/entities/exchange_cancel_event.ts
index 2fcc17df6..38f99c903 100644
--- a/packages/pipeline/src/entities/exchange_cancel_event.ts
+++ b/packages/pipeline/src/entities/exchange_cancel_event.ts
@@ -48,6 +48,4 @@ export class ExchangeCancelEvent {
public takerTokenAddress!: string;
@Column({ nullable: true, type: String, name: 'taker_token_id' })
public takerTokenId!: string | null;
-
- // TODO(albrow): Include topics?
}
diff --git a/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts b/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts
index 60ead324f..27580305e 100644
--- a/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts
+++ b/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts
@@ -12,7 +12,6 @@ export class ExchangeCancelUpToEvent {
@PrimaryColumn({ name: 'block_number', transformer: numberToBigIntTransformer })
public blockNumber!: number;
- // TODO(albrow): Include transaction hash
@Column({ name: 'raw_data' })
public rawData!: string;
@@ -24,5 +23,4 @@ export class ExchangeCancelUpToEvent {
public senderAddress!: string;
@Column({ name: 'order_epoch', type: 'numeric', transformer: bigNumberTransformer })
public orderEpoch!: BigNumber;
- // TODO(albrow): Include topics?
}
diff --git a/packages/pipeline/src/entities/exchange_fill_event.ts b/packages/pipeline/src/entities/exchange_fill_event.ts
index bbf0abf58..9b7727615 100644
--- a/packages/pipeline/src/entities/exchange_fill_event.ts
+++ b/packages/pipeline/src/entities/exchange_fill_event.ts
@@ -57,6 +57,4 @@ export class ExchangeFillEvent {
public takerTokenAddress!: string;
@Column({ nullable: true, type: String, name: 'taker_token_id' })
public takerTokenId!: string | null;
-
- // TODO(albrow): Include topics?
}
diff --git a/packages/pipeline/src/entities/relayer.ts b/packages/pipeline/src/entities/relayer.ts
index b3a856fd8..5af8578b4 100644
--- a/packages/pipeline/src/entities/relayer.ts
+++ b/packages/pipeline/src/entities/relayer.ts
@@ -14,9 +14,6 @@ export class Relayer {
@Column({ name: 'app_url', type: 'varchar', nullable: true })
public appUrl!: string | null;
- // TODO(albrow): Add exchange contract or protocol version?
- // TODO(albrow): Add network ids for addresses?
-
@Column({ name: 'fee_recipient_addresses', type: 'varchar', array: true })
public feeRecipientAddresses!: string[];
@Column({ name: 'taker_addresses', type: 'varchar', array: true })
diff --git a/packages/pipeline/src/entities/token_metadata.ts b/packages/pipeline/src/entities/token_metadata.ts
index ca1e57937..911b53972 100644
--- a/packages/pipeline/src/entities/token_metadata.ts
+++ b/packages/pipeline/src/entities/token_metadata.ts
@@ -1,5 +1,8 @@
+import { BigNumber } from '@0x/utils';
import { Column, Entity, PrimaryColumn } from 'typeorm';
+import { bigNumberTransformer } from '../utils/transformers';
+
@Entity({ name: 'token_metadata', schema: 'raw' })
export class TokenMetadata {
@PrimaryColumn({ type: 'varchar', nullable: false })
@@ -8,10 +11,8 @@ export class TokenMetadata {
@PrimaryColumn({ type: 'varchar', nullable: false })
public authority!: string;
- // TODO(albrow): Convert decimals field to type BigNumber/numeric because it
- // comes from a 256-bit integer in a smart contract.
- @Column({ type: 'integer', nullable: true })
- public decimals!: number | null;
+ @Column({ type: 'numeric', transformer: bigNumberTransformer, nullable: true })
+ public decimals!: BigNumber | null;
@Column({ type: 'varchar', nullable: true })
public symbol!: string | null;
diff --git a/packages/pipeline/src/parsers/events/index.ts b/packages/pipeline/src/parsers/events/index.ts
index d42d1c57a..e18106c75 100644
--- a/packages/pipeline/src/parsers/events/index.ts
+++ b/packages/pipeline/src/parsers/events/index.ts
@@ -49,8 +49,8 @@ export function _convertToExchangeFillEvent(eventLog: LogWithDecodedArgs<Exchang
exchangeFillEvent.logIndex = eventLog.logIndex as number;
exchangeFillEvent.rawData = eventLog.data as string;
exchangeFillEvent.transactionHash = eventLog.transactionHash;
- exchangeFillEvent.makerAddress = eventLog.args.makerAddress.toString();
- exchangeFillEvent.takerAddress = eventLog.args.takerAddress.toString();
+ exchangeFillEvent.makerAddress = eventLog.args.makerAddress;
+ exchangeFillEvent.takerAddress = eventLog.args.takerAddress;
exchangeFillEvent.feeRecipientAddress = eventLog.args.feeRecipientAddress;
exchangeFillEvent.senderAddress = eventLog.args.senderAddress;
exchangeFillEvent.makerAssetFilledAmount = eventLog.args.makerAssetFilledAmount;
@@ -92,9 +92,8 @@ export function _convertToExchangeCancelEvent(
exchangeCancelEvent.logIndex = eventLog.logIndex as number;
exchangeCancelEvent.rawData = eventLog.data as string;
exchangeCancelEvent.transactionHash = eventLog.transactionHash;
- exchangeCancelEvent.makerAddress = eventLog.args.makerAddress.toString();
- exchangeCancelEvent.takerAddress =
- eventLog.args.takerAddress == null ? null : eventLog.args.takerAddress.toString();
+ exchangeCancelEvent.makerAddress = eventLog.args.makerAddress;
+ exchangeCancelEvent.takerAddress = eventLog.args.takerAddress;
exchangeCancelEvent.feeRecipientAddress = eventLog.args.feeRecipientAddress;
exchangeCancelEvent.senderAddress = eventLog.args.senderAddress;
exchangeCancelEvent.orderHash = eventLog.args.orderHash;
@@ -127,8 +126,8 @@ export function _convertToExchangeCancelUpToEvent(
exchangeCancelUpToEvent.logIndex = eventLog.logIndex as number;
exchangeCancelUpToEvent.rawData = eventLog.data as string;
exchangeCancelUpToEvent.transactionHash = eventLog.transactionHash;
- exchangeCancelUpToEvent.makerAddress = eventLog.args.makerAddress.toString();
- exchangeCancelUpToEvent.senderAddress = eventLog.args.senderAddress.toString();
+ exchangeCancelUpToEvent.makerAddress = eventLog.args.makerAddress;
+ exchangeCancelUpToEvent.senderAddress = eventLog.args.senderAddress;
exchangeCancelUpToEvent.orderEpoch = eventLog.args.orderEpoch;
return exchangeCancelUpToEvent;
}
diff --git a/packages/pipeline/src/parsers/relayer_registry/index.ts b/packages/pipeline/src/parsers/relayer_registry/index.ts
index 7b1235005..9723880a4 100644
--- a/packages/pipeline/src/parsers/relayer_registry/index.ts
+++ b/packages/pipeline/src/parsers/relayer_registry/index.ts
@@ -18,12 +18,13 @@ function parseRelayer(relayerResp: RelayerResponse, uuid: string): Relayer {
relayer.name = relayerResp.name;
relayer.homepageUrl = relayerResp.homepage_url;
relayer.appUrl = relayerResp.app_url;
- const mainnet = getMainNetwork(relayerResp);
- if (mainnet !== undefined) {
- relayer.sraHttpEndpoint = mainnet.sra_http_endpoint || null;
- relayer.sraWsEndpoint = mainnet.sra_ws_endpoint || null;
- relayer.feeRecipientAddresses = R.path(['static_order_fields', 'fee_recipient_addresses'], mainnet) || [];
- relayer.takerAddresses = R.path(['static_order_fields', 'taker_addresses'], mainnet) || [];
+ const mainNetworkRelayerInfo = getMainNetwork(relayerResp);
+ if (mainNetworkRelayerInfo !== undefined) {
+ relayer.sraHttpEndpoint = mainNetworkRelayerInfo.sra_http_endpoint || null;
+ relayer.sraWsEndpoint = mainNetworkRelayerInfo.sra_ws_endpoint || null;
+ relayer.feeRecipientAddresses =
+ R.path(['static_order_fields', 'fee_recipient_addresses'], mainNetworkRelayerInfo) || [];
+ relayer.takerAddresses = R.path(['static_order_fields', 'taker_addresses'], mainNetworkRelayerInfo) || [];
} else {
relayer.feeRecipientAddresses = [];
relayer.takerAddresses = [];
diff --git a/packages/pipeline/src/parsers/token_metadata/index.ts b/packages/pipeline/src/parsers/token_metadata/index.ts
index 916448cb3..3b3e05d76 100644
--- a/packages/pipeline/src/parsers/token_metadata/index.ts
+++ b/packages/pipeline/src/parsers/token_metadata/index.ts
@@ -1,7 +1,9 @@
+import { BigNumber } from '@0x/utils';
import * as R from 'ramda';
import { MetamaskTrustedTokenMeta, ZeroExTrustedTokenMeta } from '../../data_sources/trusted_tokens';
import { TokenMetadata } from '../../entities';
+import {} from '../../utils';
/**
* Parses Metamask's trusted tokens list.
@@ -24,7 +26,7 @@ function parseMetamaskTrustedToken(resp: MetamaskTrustedTokenMeta, address: stri
const trustedToken = new TokenMetadata();
trustedToken.address = address;
- trustedToken.decimals = resp.decimals;
+ trustedToken.decimals = new BigNumber(resp.decimals);
trustedToken.symbol = resp.symbol;
trustedToken.name = resp.name;
trustedToken.authority = 'metamask';
@@ -36,7 +38,7 @@ function parseZeroExTrustedToken(resp: ZeroExTrustedTokenMeta): TokenMetadata {
const trustedToken = new TokenMetadata();
trustedToken.address = resp.address;
- trustedToken.decimals = resp.decimals;
+ trustedToken.decimals = new BigNumber(resp.decimals);
trustedToken.symbol = resp.symbol;
trustedToken.name = resp.name;
trustedToken.authority = '0x';
diff --git a/packages/pipeline/src/parsers/web3/index.ts b/packages/pipeline/src/parsers/web3/index.ts
index 9b5b3b55d..86f924151 100644
--- a/packages/pipeline/src/parsers/web3/index.ts
+++ b/packages/pipeline/src/parsers/web3/index.ts
@@ -42,7 +42,6 @@ export function parseTransaction(rawTransaction: EthTransaction): Transaction {
tx.blockNumber = rawTransaction.blockNumber;
tx.gasUsed = rawTransaction.gas;
- // TODO(albrow) figure out bignum solution.
tx.gasPrice = rawTransaction.gasPrice.toNumber();
return tx;
diff --git a/packages/pipeline/src/scripts/pull_missing_blocks.ts b/packages/pipeline/src/scripts/pull_missing_blocks.ts
index 4a1483ab9..b7bd51f08 100644
--- a/packages/pipeline/src/scripts/pull_missing_blocks.ts
+++ b/packages/pipeline/src/scripts/pull_missing_blocks.ts
@@ -9,7 +9,7 @@ import { Web3Source } from '../data_sources/web3';
import { Block } from '../entities';
import * as ormConfig from '../ormconfig';
import { parseBlock } from '../parsers/web3';
-import { handleError } from '../utils';
+import { EXCHANGE_START_BLOCK, handleError, INFURA_ROOT_URL } from '../utils';
// Number of blocks to save at once.
const BATCH_SAVE_SIZE = 1000;
@@ -18,16 +18,13 @@ const MAX_CONCURRENCY = 10;
// Maximum number of blocks to query for at once. This is also the maximum
// number of blocks we will hold in memory prior to being saved to the database.
const MAX_BLOCKS_PER_QUERY = 1000;
-// Block number when the Exchange contract was deployed to mainnet.
-// TODO(albrow): De-dupe this constant.
-const EXCHANGE_START_BLOCK = 6271590;
let connection: Connection;
(async () => {
connection = await createConnection(ormConfig as ConnectionOptions);
const provider = web3Factory.getRpcProvider({
- rpcUrl: `https://mainnet.infura.io/${process.env.INFURA_API_KEY}`,
+ rpcUrl: `${INFURA_ROOT_URL}/${process.env.INFURA_API_KEY}`,
});
const web3Source = new Web3Source(provider);
await getAllMissingBlocks(web3Source);
diff --git a/packages/pipeline/src/scripts/pull_missing_events.ts b/packages/pipeline/src/scripts/pull_missing_events.ts
index c7e07c540..80abbb8b0 100644
--- a/packages/pipeline/src/scripts/pull_missing_events.ts
+++ b/packages/pipeline/src/scripts/pull_missing_events.ts
@@ -8,9 +8,8 @@ import { ExchangeEventsSource } from '../data_sources/contract-wrappers/exchange
import { ExchangeCancelEvent, ExchangeCancelUpToEvent, ExchangeEvent, ExchangeFillEvent } from '../entities';
import * as ormConfig from '../ormconfig';
import { parseExchangeCancelEvents, parseExchangeCancelUpToEvents, parseExchangeFillEvents } from '../parsers/events';
-import { handleError } from '../utils';
+import { EXCHANGE_START_BLOCK, handleError, INFURA_ROOT_URL } from '../utils';
-const EXCHANGE_START_BLOCK = 6271590; // Block number when the Exchange contract was deployed to mainnet.
const START_BLOCK_OFFSET = 100; // Number of blocks before the last known block to consider when updating fill events.
const BATCH_SAVE_SIZE = 1000; // Number of events to save at once.
@@ -19,7 +18,7 @@ let connection: Connection;
(async () => {
connection = await createConnection(ormConfig as ConnectionOptions);
const provider = web3Factory.getRpcProvider({
- rpcUrl: 'https://mainnet.infura.io',
+ rpcUrl: INFURA_ROOT_URL,
});
const eventsSource = new ExchangeEventsSource(provider, 1);
await getFillEventsAsync(eventsSource);
diff --git a/packages/pipeline/src/scripts/pull_radar_relay_orders.ts b/packages/pipeline/src/scripts/pull_radar_relay_orders.ts
index b3a4d887e..bbbef9b47 100644
--- a/packages/pipeline/src/scripts/pull_radar_relay_orders.ts
+++ b/packages/pipeline/src/scripts/pull_radar_relay_orders.ts
@@ -16,11 +16,11 @@ let connection: Connection;
(async () => {
connection = await createConnection(ormConfig as ConnectionOptions);
- await getOrderbook();
+ await getOrderbookAsync();
process.exit(0);
})().catch(handleError);
-async function getOrderbook(): Promise<void> {
+async function getOrderbookAsync(): Promise<void> {
console.log('Getting all orders...');
const connectClient = new HttpClient(RADAR_RELAY_URL);
const rawOrders = await connectClient.getOrdersAsync({
@@ -29,17 +29,22 @@ async function getOrderbook(): Promise<void> {
console.log(`Got ${rawOrders.records.length} orders.`);
console.log('Parsing orders...');
// Parse the sra orders, then add source url to each.
- const orders = R.pipe(parseSraOrders, R.map(setSourceUrl(RADAR_RELAY_URL)))(rawOrders);
+ const orders = R.pipe(
+ parseSraOrders,
+ R.map(setSourceUrl(RADAR_RELAY_URL)),
+ )(rawOrders);
// Save all the orders and update the observed time stamps in a single
// transaction.
console.log('Saving orders and updating timestamps...');
- await connection.transaction(async (manager: EntityManager): Promise<void> => {
- for (const order of orders) {
- await manager.save(SraOrder, order);
- const observedTimestamp = createObservedTimestampForOrder(order);
- await manager.save(observedTimestamp);
- }
- });
+ await connection.transaction(
+ async (manager: EntityManager): Promise<void> => {
+ for (const order of orders) {
+ await manager.save(SraOrder, order);
+ const observedTimestamp = createObservedTimestampForOrder(order);
+ await manager.save(observedTimestamp);
+ }
+ },
+ );
}
const sourceUrlProp = R.lensProp('sourceUrl');
@@ -48,6 +53,8 @@ const sourceUrlProp = R.lensProp('sourceUrl');
* Sets the source url for a single order. Returns a new order instead of
* mutating the given one.
*/
-const setSourceUrl = R.curry((sourceURL: string, order: SraOrder): SraOrder => {
- return R.set(sourceUrlProp, sourceURL, order);
-});
+const setSourceUrl = R.curry(
+ (sourceURL: string, order: SraOrder): SraOrder => {
+ return R.set(sourceUrlProp, sourceURL, order);
+ },
+);
diff --git a/packages/pipeline/src/scripts/pull_trusted_tokens.ts b/packages/pipeline/src/scripts/pull_trusted_tokens.ts
index b3a4466bb..1befc4437 100644
--- a/packages/pipeline/src/scripts/pull_trusted_tokens.ts
+++ b/packages/pipeline/src/scripts/pull_trusted_tokens.ts
@@ -8,7 +8,7 @@ import { parseMetamaskTrustedTokens, parseZeroExTrustedTokens } from '../parsers
import { handleError } from '../utils';
const METAMASK_TRUSTED_TOKENS_URL =
- 'https://raw.githubusercontent.com/MetaMask/eth-contract-metadata/master/contract-map.json';
+ 'https://raw.githubusercontent.com/MetaMask/eth-contract-metadata/d45916c533116510cc8e9e048a8b5fc3732a6b6d/contract-map.json';
const ZEROEX_TRUSTED_TOKENS_URL = 'https://website-api.0xproject.com/tokens';
@@ -22,7 +22,7 @@ let connection: Connection;
})().catch(handleError);
async function getMetamaskTrustedTokens(): Promise<void> {
- // tslint:disable-next-line
+ // tslint:disable-next-line:no-console
console.log('Getting latest metamask trusted tokens list ...');
const trustedTokensRepository = connection.getRepository(TokenMetadata);
const trustedTokensSource = new TrustedTokenSource<Map<string, MetamaskTrustedTokenMeta>>(
@@ -30,23 +30,23 @@ async function getMetamaskTrustedTokens(): Promise<void> {
);
const resp = await trustedTokensSource.getTrustedTokenMetaAsync();
const trustedTokens = parseMetamaskTrustedTokens(resp);
- // tslint:disable-next-line
+ // tslint:disable-next-line:no-console
console.log('Saving metamask trusted tokens list');
await trustedTokensRepository.save(trustedTokens);
- // tslint:disable-next-line
+ // tslint:disable-next-line:no-console
console.log('Done saving metamask trusted tokens.');
}
async function getZeroExTrustedTokens(): Promise<void> {
- // tslint:disable-next-line
+ // tslint:disable-next-line:no-console
console.log('Getting latest 0x trusted tokens list ...');
const trustedTokensRepository = connection.getRepository(TokenMetadata);
const trustedTokensSource = new TrustedTokenSource<ZeroExTrustedTokenMeta[]>(ZEROEX_TRUSTED_TOKENS_URL);
const resp = await trustedTokensSource.getTrustedTokenMetaAsync();
const trustedTokens = parseZeroExTrustedTokens(resp);
- // tslint:disable-next-line
+ // tslint:disable-next-line:no-console
console.log('Saving metamask trusted tokens list');
await trustedTokensRepository.save(trustedTokens);
- // tslint:disable-next-line
+ // tslint:disable-next-line:no-console
console.log('Done saving metamask trusted tokens.');
}
diff --git a/packages/pipeline/src/utils/constants.ts b/packages/pipeline/src/utils/constants.ts
new file mode 100644
index 000000000..56f3e82d8
--- /dev/null
+++ b/packages/pipeline/src/utils/constants.ts
@@ -0,0 +1,3 @@
+// Block number when the Exchange contract was deployed to mainnet.
+export const EXCHANGE_START_BLOCK = 6271590;
+export const INFURA_ROOT_URL = 'https://mainnet.infura.io';
diff --git a/packages/pipeline/src/utils/index.ts b/packages/pipeline/src/utils/index.ts
index 718ea6133..2096a0a39 100644
--- a/packages/pipeline/src/utils/index.ts
+++ b/packages/pipeline/src/utils/index.ts
@@ -1,5 +1,6 @@
import { BigNumber } from '@0x/utils';
export * from './transformers';
+export * from './constants';
/**
* If the given BigNumber is not null, returns the string representation of that
diff --git a/packages/pipeline/src/utils/transformers/big_number.ts b/packages/pipeline/src/utils/transformers/big_number.ts
index a0471a5e3..5f2e4d565 100644
--- a/packages/pipeline/src/utils/transformers/big_number.ts
+++ b/packages/pipeline/src/utils/transformers/big_number.ts
@@ -3,13 +3,13 @@ import { ValueTransformer } from 'typeorm/decorator/options/ValueTransformer';
export class BigNumberTransformer implements ValueTransformer {
// tslint:disable-next-line:prefer-function-over-method
- public to(value: BigNumber): string {
- return value.toString();
+ public to(value: BigNumber | null): string | null {
+ return value === null ? null : value.toString();
}
// tslint:disable-next-line:prefer-function-over-method
- public from(value: string): BigNumber {
- return new BigNumber(value);
+ public from(value: string | null): BigNumber | null {
+ return value === null ? null : new BigNumber(value);
}
}
diff --git a/packages/pipeline/test/entities/token_metadata_test.ts b/packages/pipeline/test/entities/token_metadata_test.ts
index 805b4b0f7..48e656644 100644
--- a/packages/pipeline/test/entities/token_metadata_test.ts
+++ b/packages/pipeline/test/entities/token_metadata_test.ts
@@ -1,3 +1,4 @@
+import { BigNumber } from '@0x/utils';
import 'mocha';
import 'reflect-metadata';
@@ -9,15 +10,15 @@ import { testSaveAndFindEntityAsync } from './util';
chaiSetup.configure();
-const metadataWithoutNullFields = {
+const metadataWithoutNullFields: TokenMetadata = {
address: '0xe41d2489571d322189246dafa5ebde1f4699f498',
authority: 'https://website-api.0xproject.com/tokens',
- decimals: 18,
+ decimals: new BigNumber(18),
symbol: 'ZRX',
name: '0x',
};
-const metadataWithNullFields = {
+const metadataWithNullFields: TokenMetadata = {
address: '0xe41d2489571d322189246dafa5ebde1f4699f499',
authority: 'https://website-api.0xproject.com/tokens',
decimals: null,