Change some column types from varchar to numeric

11 files changed, 121 insertions, 46 deletions

diff --git a/packages/pipeline/migrations/1542234704666-ConvertBigNumberToNumeric.ts b/packages/pipeline/migrations/1542234704666-ConvertBigNumberToNumeric.ts
new file mode 100644
index 000000000..5200ef7cc
--- /dev/null
+++ b/packages/pipeline/migrations/1542234704666-ConvertBigNumberToNumeric.ts
@@ -0,0 +1,53 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class ConvertBigNumberToNumeric1542234704666 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<any> {
+        await queryRunner.query(
+            `ALTER TABLE raw.exchange_fill_events
+                ALTER COLUMN maker_asset_filled_amount TYPE numeric USING maker_asset_filled_amount::numeric,
+                ALTER COLUMN taker_asset_filled_amount TYPE numeric USING taker_asset_filled_amount::numeric,
+                ALTER COLUMN maker_fee_paid TYPE numeric USING maker_fee_paid::numeric,
+                ALTER COLUMN taker_fee_paid TYPE numeric USING taker_fee_paid::numeric;`,
+        );
+
+        await queryRunner.query(
+            `ALTER TABLE raw.exchange_cancel_up_to_events
+                ALTER COLUMN order_epoch TYPE numeric USING order_epoch::numeric;`,
+        );
+
+        await queryRunner.query(
+            `ALTER TABLE raw.sra_orders
+                ALTER COLUMN maker_asset_amount TYPE numeric USING maker_asset_amount::numeric,
+                ALTER COLUMN taker_asset_amount TYPE numeric USING taker_asset_amount::numeric,
+                ALTER COLUMN maker_fee TYPE numeric USING maker_fee::numeric,
+                ALTER COLUMN taker_fee TYPE numeric USING taker_fee::numeric,
+                ALTER COLUMN expiration_time_seconds TYPE numeric USING expiration_time_seconds::numeric,
+                ALTER COLUMN salt TYPE numeric USING salt::numeric;`,
+        );
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<any> {
+        await queryRunner.query(
+            `ALTER TABLE raw.sra_orders
+                ALTER COLUMN maker_asset_amount TYPE varchar USING maker_asset_amount::varchar,
+                ALTER COLUMN taker_asset_amount TYPE varchar USING taker_asset_amount::varchar,
+                ALTER COLUMN maker_fee TYPE varchar USING maker_fee::varchar,
+                ALTER COLUMN taker_fee TYPE varchar USING taker_fee::varchar,
+                ALTER COLUMN expiration_time_seconds TYPE varchar USING expiration_time_seconds::varchar,
+                ALTER COLUMN salt TYPE varchar USING salt::varchar;`,
+        );
+
+        await queryRunner.query(
+            `ALTER TABLE raw.exchange_cancel_up_to_events
+                ALTER COLUMN order_epoch TYPE varchar USING order_epoch::varchar;`,
+        );
+
+        await queryRunner.query(
+            `ALTER TABLE raw.exchange_fill_events
+                ALTER COLUMN maker_asset_filled_amount TYPE varchar USING maker_asset_filled_amount::varchar,
+                ALTER COLUMN taker_asset_filled_amount TYPE varchar USING taker_asset_filled_amount::varchar,
+                ALTER COLUMN maker_fee_paid TYPE varchar USING maker_fee_paid::varchar,
+                ALTER COLUMN taker_fee_paid TYPE varchar USING taker_fee_paid::varchar;`,
+        );
+    }
+}
diff --git a/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts b/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts
index 7306a1a87..752631b85 100644
--- a/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts
+++ b/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts
@@ -1,5 +1,8 @@
+import { BigNumber } from '@0x/utils';
 import { Column, Entity, PrimaryColumn } from 'typeorm';
 
+import { bigNumberTransformer } from '../utils';
+
 @Entity({ name: 'exchange_cancel_up_to_events', schema: 'raw' })
 export class ExchangeCancelUpToEvent {
     @PrimaryColumn({ name: 'contract_address' })
@@ -17,7 +20,7 @@ export class ExchangeCancelUpToEvent {
     public makerAddress!: string;
     @Column({ name: 'sender_address' })
     public senderAddress!: string;
-    @Column({ name: 'order_epoch' })
-    public orderEpoch!: string;
+    @Column({ name: 'order_epoch', type: 'numeric', transformer: bigNumberTransformer })
+    public orderEpoch!: BigNumber;
     // TODO(albrow): Include topics?
 }
diff --git a/packages/pipeline/src/entities/exchange_fill_event.ts b/packages/pipeline/src/entities/exchange_fill_event.ts
index 6202e558b..aa082436b 100644
--- a/packages/pipeline/src/entities/exchange_fill_event.ts
+++ b/packages/pipeline/src/entities/exchange_fill_event.ts
@@ -1,6 +1,8 @@
+import { BigNumber } from '@0x/utils';
 import { Column, Entity, PrimaryColumn } from 'typeorm';
 
 import { AssetType } from '../types';
+import { bigNumberTransformer } from '../utils';
 
 @Entity({ name: 'exchange_fill_events', schema: 'raw' })
 export class ExchangeFillEvent {
@@ -24,14 +26,14 @@ export class ExchangeFillEvent {
     public feeRecipientAddress!: string;
     @Column({ name: 'sender_address' })
     public senderAddress!: string;
-    @Column({ name: 'maker_asset_filled_amount' })
-    public makerAssetFilledAmount!: string;
-    @Column({ name: 'taker_asset_filled_amount' })
-    public takerAssetFilledAmount!: string;
-    @Column({ name: 'maker_fee_paid' })
-    public makerFeePaid!: string;
-    @Column({ name: 'taker_fee_paid' })
-    public takerFeePaid!: string;
+    @Column({ name: 'maker_asset_filled_amount', type: 'numeric', transformer: bigNumberTransformer })
+    public makerAssetFilledAmount!: BigNumber;
+    @Column({ name: 'taker_asset_filled_amount', type: 'numeric', transformer: bigNumberTransformer })
+    public takerAssetFilledAmount!: BigNumber;
+    @Column({ name: 'maker_fee_paid', type: 'numeric', transformer: bigNumberTransformer })
+    public makerFeePaid!: BigNumber;
+    @Column({ name: 'taker_fee_paid', type: 'numeric', transformer: bigNumberTransformer })
+    public takerFeePaid!: BigNumber;
     @Column({ name: 'order_hash' })
     public orderHash!: string;
 
diff --git a/packages/pipeline/src/entities/sra_order.ts b/packages/pipeline/src/entities/sra_order.ts
index 4b7f652d3..9c730a0bb 100644
--- a/packages/pipeline/src/entities/sra_order.ts
+++ b/packages/pipeline/src/entities/sra_order.ts
@@ -1,6 +1,8 @@
+import { BigNumber } from '@0x/utils';
 import { Column, Entity, PrimaryColumn } from 'typeorm';
 
 import { AssetType } from '../types';
+import { bigNumberTransformer } from '../utils';
 
 @Entity({ name: 'sra_orders', schema: 'raw' })
 export class SraOrder {
@@ -19,18 +21,18 @@ export class SraOrder {
     public feeRecipientAddress!: string;
     @Column({ name: 'sender_address' })
     public senderAddress!: string;
-    @Column({ name: 'maker_asset_amount' })
-    public makerAssetAmount!: string;
-    @Column({ name: 'taker_asset_amount' })
-    public takerAssetAmount!: string;
-    @Column({ name: 'maker_fee' })
-    public makerFee!: string;
-    @Column({ name: 'taker_fee' })
-    public takerFee!: string;
-    @Column({ name: 'expiration_time_seconds' })
-    public expirationTimeSeconds!: string;
-    @Column({ name: 'salt' })
-    public salt!: string;
+    @Column({ name: 'maker_asset_amount', type: 'numeric', transformer: bigNumberTransformer })
+    public makerAssetAmount!: BigNumber;
+    @Column({ name: 'taker_asset_amount', type: 'numeric', transformer: bigNumberTransformer })
+    public takerAssetAmount!: BigNumber;
+    @Column({ name: 'maker_fee', type: 'numeric', transformer: bigNumberTransformer })
+    public makerFee!: BigNumber;
+    @Column({ name: 'taker_fee', type: 'numeric', transformer: bigNumberTransformer })
+    public takerFee!: BigNumber;
+    @Column({ name: 'expiration_time_seconds', type: 'numeric', transformer: bigNumberTransformer })
+    public expirationTimeSeconds!: BigNumber;
+    @Column({ name: 'salt', type: 'numeric', transformer: bigNumberTransformer })
+    public salt!: BigNumber;
     @Column({ name: 'signature' })
     public signature!: string;
 
diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts
index 2f5f7df33..0e489e560 100644
--- a/packages/pipeline/src/ormconfig.ts
+++ b/packages/pipeline/src/ormconfig.ts
@@ -26,7 +26,7 @@ const config: ConnectionOptions = {
     type: 'postgres',
     url: process.env.ZEROEX_DATA_PIPELINE_DB_URL,
     synchronize: false,
-    logging: false,
+    logging: ['error', 'warn'],
     entities,
     migrations: ['./lib/migrations/**/*.js'],
 };
diff --git a/packages/pipeline/src/parsers/events/index.ts b/packages/pipeline/src/parsers/events/index.ts
index d56dc7080..27e192d7b 100644
--- a/packages/pipeline/src/parsers/events/index.ts
+++ b/packages/pipeline/src/parsers/events/index.ts
@@ -65,10 +65,10 @@ export function _convertToExchangeFillEvent(eventLog: LogWithDecodedArgs<Exchang
     exchangeFillEvent.takerAddress = eventLog.args.takerAddress.toString();
     exchangeFillEvent.feeRecipientAddress = eventLog.args.feeRecipientAddress;
     exchangeFillEvent.senderAddress = eventLog.args.senderAddress;
-    exchangeFillEvent.makerAssetFilledAmount = eventLog.args.makerAssetFilledAmount.toString();
-    exchangeFillEvent.takerAssetFilledAmount = eventLog.args.takerAssetFilledAmount.toString();
-    exchangeFillEvent.makerFeePaid = eventLog.args.makerFeePaid.toString();
-    exchangeFillEvent.takerFeePaid = eventLog.args.takerFeePaid.toString();
+    exchangeFillEvent.makerAssetFilledAmount = eventLog.args.makerAssetFilledAmount;
+    exchangeFillEvent.takerAssetFilledAmount = eventLog.args.takerAssetFilledAmount;
+    exchangeFillEvent.makerFeePaid = eventLog.args.makerFeePaid;
+    exchangeFillEvent.takerFeePaid = eventLog.args.takerFeePaid;
     exchangeFillEvent.orderHash = eventLog.args.orderHash;
     exchangeFillEvent.rawMakerAssetData = eventLog.args.makerAssetData;
     exchangeFillEvent.makerAssetType = makerAssetType;
@@ -139,6 +139,6 @@ export function _convertToExchangeCancelUpToEvent(
     exchangeCancelUpToEvent.rawData = eventLog.data as string;
     exchangeCancelUpToEvent.makerAddress = eventLog.args.makerAddress.toString();
     exchangeCancelUpToEvent.senderAddress = eventLog.args.senderAddress.toString();
-    exchangeCancelUpToEvent.orderEpoch = eventLog.args.orderEpoch.toString();
+    exchangeCancelUpToEvent.orderEpoch = eventLog.args.orderEpoch;
     return exchangeCancelUpToEvent;
 }
diff --git a/packages/pipeline/src/parsers/sra_orders/index.ts b/packages/pipeline/src/parsers/sra_orders/index.ts
index 800521955..ef8901e40 100644
--- a/packages/pipeline/src/parsers/sra_orders/index.ts
+++ b/packages/pipeline/src/parsers/sra_orders/index.ts
@@ -34,12 +34,12 @@ export function _convertToEntity(apiOrder: APIOrder): SraOrder {
     sraOrder.takerAddress = apiOrder.order.takerAddress;
     sraOrder.feeRecipientAddress = apiOrder.order.feeRecipientAddress;
     sraOrder.senderAddress = apiOrder.order.senderAddress;
-    sraOrder.makerAssetAmount = apiOrder.order.makerAssetAmount.toString();
-    sraOrder.takerAssetAmount = apiOrder.order.takerAssetAmount.toString();
-    sraOrder.makerFee = apiOrder.order.makerFee.toString();
-    sraOrder.takerFee = apiOrder.order.takerFee.toString();
-    sraOrder.expirationTimeSeconds = apiOrder.order.expirationTimeSeconds.toString();
-    sraOrder.salt = apiOrder.order.salt.toString();
+    sraOrder.makerAssetAmount = apiOrder.order.makerAssetAmount;
+    sraOrder.takerAssetAmount = apiOrder.order.takerAssetAmount;
+    sraOrder.makerFee = apiOrder.order.makerFee;
+    sraOrder.takerFee = apiOrder.order.takerFee;
+    sraOrder.expirationTimeSeconds = apiOrder.order.expirationTimeSeconds;
+    sraOrder.salt = apiOrder.order.salt;
     sraOrder.signature = apiOrder.order.signature;
 
     sraOrder.rawMakerAssetData = apiOrder.order.makerAssetData;
diff --git a/packages/pipeline/src/scripts/pull_missing_events.ts b/packages/pipeline/src/scripts/pull_missing_events.ts
index 0af999a77..1693bb59a 100644
--- a/packages/pipeline/src/scripts/pull_missing_events.ts
+++ b/packages/pipeline/src/scripts/pull_missing_events.ts
@@ -13,7 +13,7 @@ import { handleError } from '../utils';
 
 const EXCHANGE_START_BLOCK = 6271590; // Block number when the Exchange contract was deployed to mainnet.
 const START_BLOCK_OFFSET = 1000; // Number of blocks before the last known block to consider when updating fill events.
-const BATCH_SAVE_SIZE = 1000; // Number of events to save at once.
+const BATCH_SAVE_SIZE = 10000; // Number of events to save at once.
 
 let connection: Connection;
 
diff --git a/packages/pipeline/src/utils/index.ts b/packages/pipeline/src/utils/index.ts
index a083cd8c9..78aa89374 100644
--- a/packages/pipeline/src/utils/index.ts
+++ b/packages/pipeline/src/utils/index.ts
@@ -1,4 +1,5 @@
 import { BigNumber } from '@0x/utils';
+import { ValueTransformer } from 'typeorm/decorator/options/ValueTransformer';
 
 /**
  * If the given BigNumber is not null, returns the string representation of that
@@ -34,3 +35,17 @@ export function handleError(e: any): void {
     }
     process.exit(1);
 }
+
+class BigNumberTransformer implements ValueTransformer {
+    // tslint:disable-next-line:prefer-function-over-method
+    public to(value: BigNumber): string {
+        return value.toString();
+    }
+
+    // tslint:disable-next-line:prefer-function-over-method
+    public from(value: string): BigNumber {
+        return new BigNumber(value);
+    }
+}
+
+export const bigNumberTransformer = new BigNumberTransformer();
diff --git a/packages/pipeline/test/parsers/events/index_test.ts b/packages/pipeline/test/parsers/events/index_test.ts
index fdd07f246..63e080edc 100644
--- a/packages/pipeline/test/parsers/events/index_test.ts
+++ b/packages/pipeline/test/parsers/events/index_test.ts
@@ -39,7 +39,7 @@ describe('exchange_events', () => {
                     makerAssetFilledAmount: new BigNumber('10000000000000000'),
                     takerAssetFilledAmount: new BigNumber('100000000000000000'),
                     makerFeePaid: new BigNumber('0'),
-                    takerFeePaid: new BigNumber('0'),
+                    takerFeePaid: new BigNumber('12345'),
                     orderHash: '0xab12ed2cbaa5615ab690b9da75a46e53ddfcf3f1a68655b5fe0d94c75a1aac4a',
                     makerAssetData: '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2',
                     takerAssetData: '0xf47261b0000000000000000000000000e41d2489571d322189246dafa5ebde1f4699f498',
@@ -56,10 +56,10 @@ describe('exchange_events', () => {
             expected.takerAddress = '0xf6da68519f78b0d0bc93c701e86affcb75c92428';
             expected.feeRecipientAddress = '0xc370d2a5920344aa6b7d8d11250e3e861434cbdd';
             expected.senderAddress = '0xf6da68519f78b0d0bc93c701e86affcb75c92428';
-            expected.makerAssetFilledAmount = '10000000000000000';
-            expected.takerAssetFilledAmount = '100000000000000000';
-            expected.makerFeePaid = '0';
-            expected.takerFeePaid = '0';
+            expected.makerAssetFilledAmount = new BigNumber('10000000000000000');
+            expected.takerAssetFilledAmount = new BigNumber('100000000000000000');
+            expected.makerFeePaid = new BigNumber('0');
+            expected.takerFeePaid = new BigNumber('12345');
             expected.orderHash = '0xab12ed2cbaa5615ab690b9da75a46e53ddfcf3f1a68655b5fe0d94c75a1aac4a';
             expected.rawMakerAssetData = '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2';
             expected.makerAssetType = 'erc20';
diff --git a/packages/pipeline/test/parsers/sra_orders/index_test.ts b/packages/pipeline/test/parsers/sra_orders/index_test.ts
index 534d84ab3..ee2842ef3 100644
--- a/packages/pipeline/test/parsers/sra_orders/index_test.ts
+++ b/packages/pipeline/test/parsers/sra_orders/index_test.ts
@@ -41,12 +41,12 @@ describe('sra_orders', () => {
             expected.takerAddress = '0x0000000000000000000000000000000000000000';
             expected.feeRecipientAddress = '0xa258b39954cef5cb142fd567a46cddb31a670124';
             expected.senderAddress = '0x0000000000000000000000000000000000000000';
-            expected.makerAssetAmount = '1619310371000000000';
-            expected.takerAssetAmount = '8178335207070707070707';
-            expected.makerFee = '0';
-            expected.takerFee = '0';
-            expected.expirationTimeSeconds = '1538529488';
-            expected.salt = '1537924688891';
+            expected.makerAssetAmount = new BigNumber('1619310371000000000');
+            expected.takerAssetAmount = new BigNumber('8178335207070707070707');
+            expected.makerFee = new BigNumber('0');
+            expected.takerFee = new BigNumber('0');
+            expected.expirationTimeSeconds = new BigNumber('1538529488');
+            expected.salt = new BigNumber('1537924688891');
             expected.signature =
                 '0x1b5a5d672b0d647b5797387ccbb89d822d5d2e873346b014f4ff816ff0783f2a7a0d2824d2d7042ec8ea375bc7f870963e1cb8248f1db03ddf125e27b5963aa11f03';
             expected.rawMakerAssetData = '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2';