Introduce framework for running basic tests for entities (#1344)

* Introduce framework for running basic tests for entities

* Add pipeline tests to CircleCI config

* Make pipeline tests more configurable and fix CircleCI config

* Add coverage dir to pipeline package

* Add basic tests for all exchange event entities

* Add tests for remaining entities

* Create separate test scripts in package.json and add new info to README

* Update db_setup.ts to revert migrations even if you are using docker

* Automatically pull the postgres image if needed

* Add comment about why NumberToBigIntTransformer is needed

11 files changed, 70 insertions, 29 deletions

diff --git a/packages/pipeline/src/entities/block.ts b/packages/pipeline/src/entities/block.ts
index f2efc6390..398946622 100644
--- a/packages/pipeline/src/entities/block.ts
+++ b/packages/pipeline/src/entities/block.ts
@@ -1,10 +1,13 @@
 import { Column, Entity, PrimaryColumn } from 'typeorm';
 
+import { numberToBigIntTransformer } from '../utils';
+
 @Entity({ name: 'blocks', schema: 'raw' })
 export class Block {
     @PrimaryColumn() public hash!: string;
-    @PrimaryColumn() public number!: number;
+    @PrimaryColumn({ transformer: numberToBigIntTransformer })
+    public number!: number;
 
-    @Column({ name: 'timestamp' })
+    @Column({ name: 'timestamp', transformer: numberToBigIntTransformer })
     public timestamp!: number;
 }
diff --git a/packages/pipeline/src/entities/exchange_cancel_event.ts b/packages/pipeline/src/entities/exchange_cancel_event.ts
index 5a40ba799..2fcc17df6 100644
--- a/packages/pipeline/src/entities/exchange_cancel_event.ts
+++ b/packages/pipeline/src/entities/exchange_cancel_event.ts
@@ -1,6 +1,7 @@
 import { Column, Entity, PrimaryColumn } from 'typeorm';
 
 import { AssetType } from '../types';
+import { numberToBigIntTransformer } from '../utils';
 
 @Entity({ name: 'exchange_cancel_events', schema: 'raw' })
 export class ExchangeCancelEvent {
@@ -8,7 +9,7 @@ export class ExchangeCancelEvent {
     public contractAddress!: string;
     @PrimaryColumn({ name: 'log_index' })
     public logIndex!: number;
-    @PrimaryColumn({ name: 'block_number' })
+    @PrimaryColumn({ name: 'block_number', transformer: numberToBigIntTransformer })
     public blockNumber!: number;
 
     @Column({ name: 'raw_data' })
diff --git a/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts b/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts
index 9b1c6174a..60ead324f 100644
--- a/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts
+++ b/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts
@@ -1,7 +1,7 @@
 import { BigNumber } from '@0x/utils';
 import { Column, Entity, PrimaryColumn } from 'typeorm';
 
-import { bigNumberTransformer } from '../utils';
+import { bigNumberTransformer, numberToBigIntTransformer } from '../utils';
 
 @Entity({ name: 'exchange_cancel_up_to_events', schema: 'raw' })
 export class ExchangeCancelUpToEvent {
@@ -9,7 +9,7 @@ export class ExchangeCancelUpToEvent {
     public contractAddress!: string;
     @PrimaryColumn({ name: 'log_index' })
     public logIndex!: number;
-    @PrimaryColumn({ name: 'block_number' })
+    @PrimaryColumn({ name: 'block_number', transformer: numberToBigIntTransformer })
     public blockNumber!: number;
 
     // TODO(albrow): Include transaction hash
diff --git a/packages/pipeline/src/entities/exchange_fill_event.ts b/packages/pipeline/src/entities/exchange_fill_event.ts
index aa082436b..bbf0abf58 100644
--- a/packages/pipeline/src/entities/exchange_fill_event.ts
+++ b/packages/pipeline/src/entities/exchange_fill_event.ts
@@ -2,7 +2,7 @@ import { BigNumber } from '@0x/utils';
 import { Column, Entity, PrimaryColumn } from 'typeorm';
 
 import { AssetType } from '../types';
-import { bigNumberTransformer } from '../utils';
+import { bigNumberTransformer, numberToBigIntTransformer } from '../utils';
 
 @Entity({ name: 'exchange_fill_events', schema: 'raw' })
 export class ExchangeFillEvent {
@@ -10,7 +10,7 @@ export class ExchangeFillEvent {
     public contractAddress!: string;
     @PrimaryColumn({ name: 'log_index' })
     public logIndex!: number;
-    @PrimaryColumn({ name: 'block_number' })
+    @PrimaryColumn({ name: 'block_number', transformer: numberToBigIntTransformer })
     public blockNumber!: number;
 
     @Column({ name: 'raw_data' })
diff --git a/packages/pipeline/src/entities/sra_order_observed_timestamp.ts b/packages/pipeline/src/entities/sra_order_observed_timestamp.ts
index bdb6cd36b..cd2d41397 100644
--- a/packages/pipeline/src/entities/sra_order_observed_timestamp.ts
+++ b/packages/pipeline/src/entities/sra_order_observed_timestamp.ts
@@ -1,5 +1,7 @@
 import { Entity, PrimaryColumn } from 'typeorm';
 
+import { numberToBigIntTransformer } from '../utils';
+
 import { SraOrder } from './sra_order';
 
 @Entity({ name: 'sra_orders_observed_timestamps', schema: 'raw' })
@@ -11,7 +13,7 @@ export class SraOrdersObservedTimeStamp {
     @PrimaryColumn({ name: 'source_url' })
     public sourceUrl!: string;
 
-    @PrimaryColumn({ name: 'observed_timestamp' })
+    @PrimaryColumn({ name: 'observed_timestamp', transformer: numberToBigIntTransformer })
     public observedTimestamp!: number;
 }
 
diff --git a/packages/pipeline/src/entities/token_metadata.ts b/packages/pipeline/src/entities/token_metadata.ts
index 4a0bceefe..ca1e57937 100644
--- a/packages/pipeline/src/entities/token_metadata.ts
+++ b/packages/pipeline/src/entities/token_metadata.ts
@@ -8,12 +8,14 @@ export class TokenMetadata {
     @PrimaryColumn({ type: 'varchar', nullable: false })
     public authority!: string;
 
+    // TODO(albrow): Convert decimals field to type BigNumber/numeric because it
+    // comes from a 256-bit integer in a smart contract.
     @Column({ type: 'integer', nullable: true })
-    public decimals!: number;
+    public decimals!: number | null;
 
     @Column({ type: 'varchar', nullable: true })
-    public symbol!: string;
+    public symbol!: string | null;
 
     @Column({ type: 'varchar', nullable: true })
-    public name!: string;
+    public name!: string | null;
 }
diff --git a/packages/pipeline/src/entities/transaction.ts b/packages/pipeline/src/entities/transaction.ts
index dd2143df5..91e4ecb5d 100644
--- a/packages/pipeline/src/entities/transaction.ts
+++ b/packages/pipeline/src/entities/transaction.ts
@@ -1,16 +1,18 @@
 import { Column, Entity, PrimaryColumn } from 'typeorm';
 
+import { numberToBigIntTransformer } from '../utils';
+
 @Entity({ name: 'transactions', schema: 'raw' })
 export class Transaction {
     @PrimaryColumn({ name: 'transaction_hash' })
     public transactionHash!: string;
     @PrimaryColumn({ name: 'block_hash' })
     public blockHash!: string;
-    @PrimaryColumn({ name: 'block_number' })
+    @PrimaryColumn({ name: 'block_number', transformer: numberToBigIntTransformer })
     public blockNumber!: number;
 
-    @Column({ type: 'bigint', name: 'gas_used' })
+    @Column({ type: 'bigint', name: 'gas_used', transformer: numberToBigIntTransformer })
     public gasUsed!: number;
-    @Column({ type: 'bigint', name: 'gas_price' })
+    @Column({ type: 'bigint', name: 'gas_price', transformer: numberToBigIntTransformer })
     public gasPrice!: number;
 }
diff --git a/packages/pipeline/src/utils/index.ts b/packages/pipeline/src/utils/index.ts
index b6d84a565..918cfc695 100644
--- a/packages/pipeline/src/utils/index.ts
+++ b/packages/pipeline/src/utils/index.ts
@@ -1,5 +1,5 @@
 import { BigNumber } from '@0x/utils';
-import { ValueTransformer } from 'typeorm/decorator/options/ValueTransformer';
+export * from './transformers';
 
 /**
  * If the given BigNumber is not null, returns the string representation of that
@@ -36,20 +36,6 @@ export function handleError(e: any): void {
     process.exit(1);
 }
 
-class BigNumberTransformer implements ValueTransformer {
-    // tslint:disable-next-line:prefer-function-over-method
-    public to(value: BigNumber): string {
-        return value.toString();
-    }
-
-    // tslint:disable-next-line:prefer-function-over-method
-    public from(value: string): BigNumber {
-        return new BigNumber(value);
-    }
-}
-
-export const bigNumberTransformer = new BigNumberTransformer();
-
 /**
  * Returns the unix timestamp of the current hour
  */
diff --git a/packages/pipeline/src/utils/transformers/big_number.ts b/packages/pipeline/src/utils/transformers/big_number.ts
new file mode 100644
index 000000000..a0471a5e3
--- /dev/null
+++ b/packages/pipeline/src/utils/transformers/big_number.ts
@@ -0,0 +1,16 @@
+import { BigNumber } from '@0x/utils';
+import { ValueTransformer } from 'typeorm/decorator/options/ValueTransformer';
+
+export class BigNumberTransformer implements ValueTransformer {
+    // tslint:disable-next-line:prefer-function-over-method
+    public to(value: BigNumber): string {
+        return value.toString();
+    }
+
+    // tslint:disable-next-line:prefer-function-over-method
+    public from(value: string): BigNumber {
+        return new BigNumber(value);
+    }
+}
+
+export const bigNumberTransformer = new BigNumberTransformer();
diff --git a/packages/pipeline/src/utils/transformers/index.ts b/packages/pipeline/src/utils/transformers/index.ts
new file mode 100644
index 000000000..232c1c5de
--- /dev/null
+++ b/packages/pipeline/src/utils/transformers/index.ts
@@ -0,0 +1,2 @@
+export * from './big_number';
+export * from './number_to_bigint';
diff --git a/packages/pipeline/src/utils/transformers/number_to_bigint.ts b/packages/pipeline/src/utils/transformers/number_to_bigint.ts
new file mode 100644
index 000000000..85560c1f0
--- /dev/null
+++ b/packages/pipeline/src/utils/transformers/number_to_bigint.ts
@@ -0,0 +1,27 @@
+import { BigNumber } from '@0x/utils';
+import { ValueTransformer } from 'typeorm/decorator/options/ValueTransformer';
+
+const decimalRadix = 10;
+
+// Can be used to convert a JavaScript number type to a Postgres bigint type and
+// vice versa. By default TypeORM will silently convert number types to string
+// if the corresponding Postgres type is bigint. See
+// https://github.com/typeorm/typeorm/issues/2400 for more information.
+export class NumberToBigIntTransformer implements ValueTransformer {
+    // tslint:disable-next-line:prefer-function-over-method
+    public to(value: number): string {
+        return value.toString();
+    }
+
+    // tslint:disable-next-line:prefer-function-over-method
+    public from(value: string): number {
+        if (new BigNumber(value).greaterThan(Number.MAX_SAFE_INTEGER)) {
+            throw new Error(
+                `Attempted to convert PostgreSQL bigint value (${value}) to JavaScript number type but it is too big to safely convert`,
+            );
+        }
+        return Number.parseInt(value, decimalRadix);
+    }
+}
+
+export const numberToBigIntTransformer = new NumberToBigIntTransformer();