aboutsummaryrefslogtreecommitdiffstats
path: root/packages/pipeline/src
diff options
context:
space:
mode:
authorAlex Browne <stephenalexbrowne@gmail.com>2018-11-29 05:21:04 +0800
committerAlex Browne <stephenalexbrowne@gmail.com>2018-12-05 06:25:38 +0800
commit3d211c415b58a67f84332ff512bf9372cac5a3ac (patch)
tree087909fde4c6f2c8f99d8487a1f0ce45bf91591f /packages/pipeline/src
parent4061731245a8513e8d990f3af87e182fb674838b (diff)
downloaddexon-sol-tools-3d211c415b58a67f84332ff512bf9372cac5a3ac.tar
dexon-sol-tools-3d211c415b58a67f84332ff512bf9372cac5a3ac.tar.gz
dexon-sol-tools-3d211c415b58a67f84332ff512bf9372cac5a3ac.tar.bz2
dexon-sol-tools-3d211c415b58a67f84332ff512bf9372cac5a3ac.tar.lz
dexon-sol-tools-3d211c415b58a67f84332ff512bf9372cac5a3ac.tar.xz
dexon-sol-tools-3d211c415b58a67f84332ff512bf9372cac5a3ac.tar.zst
dexon-sol-tools-3d211c415b58a67f84332ff512bf9372cac5a3ac.zip
Introduce framework for running basic tests for entities (#1344)
* Introduce framework for running basic tests for entities * Add pipeline tests to CircleCI config * Make pipeline tests more configurable and fix CircleCI config * Add coverage dir to pipeline package * Add basic tests for all exchange event entities * Add tests for remaining entities * Create separate test scripts in package.json and add new info to README * Update db_setup.ts to revert migrations even if you are using docker * Automatically pull the postgres image if needed * Add comment about why NumberToBigIntTransformer is needed
Diffstat (limited to 'packages/pipeline/src')
-rw-r--r--packages/pipeline/src/entities/block.ts7
-rw-r--r--packages/pipeline/src/entities/exchange_cancel_event.ts3
-rw-r--r--packages/pipeline/src/entities/exchange_cancel_up_to_event.ts4
-rw-r--r--packages/pipeline/src/entities/exchange_fill_event.ts4
-rw-r--r--packages/pipeline/src/entities/sra_order_observed_timestamp.ts4
-rw-r--r--packages/pipeline/src/entities/token_metadata.ts8
-rw-r--r--packages/pipeline/src/entities/transaction.ts8
-rw-r--r--packages/pipeline/src/utils/index.ts16
-rw-r--r--packages/pipeline/src/utils/transformers/big_number.ts16
-rw-r--r--packages/pipeline/src/utils/transformers/index.ts2
-rw-r--r--packages/pipeline/src/utils/transformers/number_to_bigint.ts27
11 files changed, 70 insertions, 29 deletions
diff --git a/packages/pipeline/src/entities/block.ts b/packages/pipeline/src/entities/block.ts
index f2efc6390..398946622 100644
--- a/packages/pipeline/src/entities/block.ts
+++ b/packages/pipeline/src/entities/block.ts
@@ -1,10 +1,13 @@
import { Column, Entity, PrimaryColumn } from 'typeorm';
+import { numberToBigIntTransformer } from '../utils';
+
@Entity({ name: 'blocks', schema: 'raw' })
export class Block {
@PrimaryColumn() public hash!: string;
- @PrimaryColumn() public number!: number;
+ @PrimaryColumn({ transformer: numberToBigIntTransformer })
+ public number!: number;
- @Column({ name: 'timestamp' })
+ @Column({ name: 'timestamp', transformer: numberToBigIntTransformer })
public timestamp!: number;
}
diff --git a/packages/pipeline/src/entities/exchange_cancel_event.ts b/packages/pipeline/src/entities/exchange_cancel_event.ts
index 5a40ba799..2fcc17df6 100644
--- a/packages/pipeline/src/entities/exchange_cancel_event.ts
+++ b/packages/pipeline/src/entities/exchange_cancel_event.ts
@@ -1,6 +1,7 @@
import { Column, Entity, PrimaryColumn } from 'typeorm';
import { AssetType } from '../types';
+import { numberToBigIntTransformer } from '../utils';
@Entity({ name: 'exchange_cancel_events', schema: 'raw' })
export class ExchangeCancelEvent {
@@ -8,7 +9,7 @@ export class ExchangeCancelEvent {
public contractAddress!: string;
@PrimaryColumn({ name: 'log_index' })
public logIndex!: number;
- @PrimaryColumn({ name: 'block_number' })
+ @PrimaryColumn({ name: 'block_number', transformer: numberToBigIntTransformer })
public blockNumber!: number;
@Column({ name: 'raw_data' })
diff --git a/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts b/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts
index 9b1c6174a..60ead324f 100644
--- a/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts
+++ b/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts
@@ -1,7 +1,7 @@
import { BigNumber } from '@0x/utils';
import { Column, Entity, PrimaryColumn } from 'typeorm';
-import { bigNumberTransformer } from '../utils';
+import { bigNumberTransformer, numberToBigIntTransformer } from '../utils';
@Entity({ name: 'exchange_cancel_up_to_events', schema: 'raw' })
export class ExchangeCancelUpToEvent {
@@ -9,7 +9,7 @@ export class ExchangeCancelUpToEvent {
public contractAddress!: string;
@PrimaryColumn({ name: 'log_index' })
public logIndex!: number;
- @PrimaryColumn({ name: 'block_number' })
+ @PrimaryColumn({ name: 'block_number', transformer: numberToBigIntTransformer })
public blockNumber!: number;
// TODO(albrow): Include transaction hash
diff --git a/packages/pipeline/src/entities/exchange_fill_event.ts b/packages/pipeline/src/entities/exchange_fill_event.ts
index aa082436b..bbf0abf58 100644
--- a/packages/pipeline/src/entities/exchange_fill_event.ts
+++ b/packages/pipeline/src/entities/exchange_fill_event.ts
@@ -2,7 +2,7 @@ import { BigNumber } from '@0x/utils';
import { Column, Entity, PrimaryColumn } from 'typeorm';
import { AssetType } from '../types';
-import { bigNumberTransformer } from '../utils';
+import { bigNumberTransformer, numberToBigIntTransformer } from '../utils';
@Entity({ name: 'exchange_fill_events', schema: 'raw' })
export class ExchangeFillEvent {
@@ -10,7 +10,7 @@ export class ExchangeFillEvent {
public contractAddress!: string;
@PrimaryColumn({ name: 'log_index' })
public logIndex!: number;
- @PrimaryColumn({ name: 'block_number' })
+ @PrimaryColumn({ name: 'block_number', transformer: numberToBigIntTransformer })
public blockNumber!: number;
@Column({ name: 'raw_data' })
diff --git a/packages/pipeline/src/entities/sra_order_observed_timestamp.ts b/packages/pipeline/src/entities/sra_order_observed_timestamp.ts
index bdb6cd36b..cd2d41397 100644
--- a/packages/pipeline/src/entities/sra_order_observed_timestamp.ts
+++ b/packages/pipeline/src/entities/sra_order_observed_timestamp.ts
@@ -1,5 +1,7 @@
import { Entity, PrimaryColumn } from 'typeorm';
+import { numberToBigIntTransformer } from '../utils';
+
import { SraOrder } from './sra_order';
@Entity({ name: 'sra_orders_observed_timestamps', schema: 'raw' })
@@ -11,7 +13,7 @@ export class SraOrdersObservedTimeStamp {
@PrimaryColumn({ name: 'source_url' })
public sourceUrl!: string;
- @PrimaryColumn({ name: 'observed_timestamp' })
+ @PrimaryColumn({ name: 'observed_timestamp', transformer: numberToBigIntTransformer })
public observedTimestamp!: number;
}
diff --git a/packages/pipeline/src/entities/token_metadata.ts b/packages/pipeline/src/entities/token_metadata.ts
index 4a0bceefe..ca1e57937 100644
--- a/packages/pipeline/src/entities/token_metadata.ts
+++ b/packages/pipeline/src/entities/token_metadata.ts
@@ -8,12 +8,14 @@ export class TokenMetadata {
@PrimaryColumn({ type: 'varchar', nullable: false })
public authority!: string;
+ // TODO(albrow): Convert decimals field to type BigNumber/numeric because it
+ // comes from a 256-bit integer in a smart contract.
@Column({ type: 'integer', nullable: true })
- public decimals!: number;
+ public decimals!: number | null;
@Column({ type: 'varchar', nullable: true })
- public symbol!: string;
+ public symbol!: string | null;
@Column({ type: 'varchar', nullable: true })
- public name!: string;
+ public name!: string | null;
}
diff --git a/packages/pipeline/src/entities/transaction.ts b/packages/pipeline/src/entities/transaction.ts
index dd2143df5..91e4ecb5d 100644
--- a/packages/pipeline/src/entities/transaction.ts
+++ b/packages/pipeline/src/entities/transaction.ts
@@ -1,16 +1,18 @@
import { Column, Entity, PrimaryColumn } from 'typeorm';
+import { numberToBigIntTransformer } from '../utils';
+
@Entity({ name: 'transactions', schema: 'raw' })
export class Transaction {
@PrimaryColumn({ name: 'transaction_hash' })
public transactionHash!: string;
@PrimaryColumn({ name: 'block_hash' })
public blockHash!: string;
- @PrimaryColumn({ name: 'block_number' })
+ @PrimaryColumn({ name: 'block_number', transformer: numberToBigIntTransformer })
public blockNumber!: number;
- @Column({ type: 'bigint', name: 'gas_used' })
+ @Column({ type: 'bigint', name: 'gas_used', transformer: numberToBigIntTransformer })
public gasUsed!: number;
- @Column({ type: 'bigint', name: 'gas_price' })
+ @Column({ type: 'bigint', name: 'gas_price', transformer: numberToBigIntTransformer })
public gasPrice!: number;
}
diff --git a/packages/pipeline/src/utils/index.ts b/packages/pipeline/src/utils/index.ts
index b6d84a565..918cfc695 100644
--- a/packages/pipeline/src/utils/index.ts
+++ b/packages/pipeline/src/utils/index.ts
@@ -1,5 +1,5 @@
import { BigNumber } from '@0x/utils';
-import { ValueTransformer } from 'typeorm/decorator/options/ValueTransformer';
+export * from './transformers';
/**
* If the given BigNumber is not null, returns the string representation of that
@@ -36,20 +36,6 @@ export function handleError(e: any): void {
process.exit(1);
}
-class BigNumberTransformer implements ValueTransformer {
- // tslint:disable-next-line:prefer-function-over-method
- public to(value: BigNumber): string {
- return value.toString();
- }
-
- // tslint:disable-next-line:prefer-function-over-method
- public from(value: string): BigNumber {
- return new BigNumber(value);
- }
-}
-
-export const bigNumberTransformer = new BigNumberTransformer();
-
/**
* Returns the unix timestamp of the current hour
*/
diff --git a/packages/pipeline/src/utils/transformers/big_number.ts b/packages/pipeline/src/utils/transformers/big_number.ts
new file mode 100644
index 000000000..a0471a5e3
--- /dev/null
+++ b/packages/pipeline/src/utils/transformers/big_number.ts
@@ -0,0 +1,16 @@
+import { BigNumber } from '@0x/utils';
+import { ValueTransformer } from 'typeorm/decorator/options/ValueTransformer';
+
+export class BigNumberTransformer implements ValueTransformer {
+ // tslint:disable-next-line:prefer-function-over-method
+ public to(value: BigNumber): string {
+ return value.toString();
+ }
+
+ // tslint:disable-next-line:prefer-function-over-method
+ public from(value: string): BigNumber {
+ return new BigNumber(value);
+ }
+}
+
+export const bigNumberTransformer = new BigNumberTransformer();
diff --git a/packages/pipeline/src/utils/transformers/index.ts b/packages/pipeline/src/utils/transformers/index.ts
new file mode 100644
index 000000000..232c1c5de
--- /dev/null
+++ b/packages/pipeline/src/utils/transformers/index.ts
@@ -0,0 +1,2 @@
+export * from './big_number';
+export * from './number_to_bigint';
diff --git a/packages/pipeline/src/utils/transformers/number_to_bigint.ts b/packages/pipeline/src/utils/transformers/number_to_bigint.ts
new file mode 100644
index 000000000..85560c1f0
--- /dev/null
+++ b/packages/pipeline/src/utils/transformers/number_to_bigint.ts
@@ -0,0 +1,27 @@
+import { BigNumber } from '@0x/utils';
+import { ValueTransformer } from 'typeorm/decorator/options/ValueTransformer';
+
+const decimalRadix = 10;
+
+// Can be used to convert a JavaScript number type to a Postgres bigint type and
+// vice versa. By default TypeORM will silently convert number types to string
+// if the corresponding Postgres type is bigint. See
+// https://github.com/typeorm/typeorm/issues/2400 for more information.
+export class NumberToBigIntTransformer implements ValueTransformer {
+ // tslint:disable-next-line:prefer-function-over-method
+ public to(value: number): string {
+ return value.toString();
+ }
+
+ // tslint:disable-next-line:prefer-function-over-method
+ public from(value: string): number {
+ if (new BigNumber(value).greaterThan(Number.MAX_SAFE_INTEGER)) {
+ throw new Error(
+ `Attempted to convert PostgreSQL bigint value (${value}) to JavaScript number type but it is too big to safely convert`,
+ );
+ }
+ return Number.parseInt(value, decimalRadix);
+ }
+}
+
+export const numberToBigIntTransformer = new NumberToBigIntTransformer();