aboutsummaryrefslogtreecommitdiffstats
path: root/packages/pipeline/src
diff options
context:
space:
mode:
authorAlex Browne <stephenalexbrowne@gmail.com>2018-11-15 07:58:36 +0800
committerAlex Browne <stephenalexbrowne@gmail.com>2018-12-05 06:24:48 +0800
commit303bbc42f4322448998f3fde202574335d1190e6 (patch)
treeb0a45e28d54fef51d237ff32352dd82c891419cf /packages/pipeline/src
parent10e93bb01ffcf029821430781ef582a24901a461 (diff)
downloaddexon-sol-tools-303bbc42f4322448998f3fde202574335d1190e6.tar
dexon-sol-tools-303bbc42f4322448998f3fde202574335d1190e6.tar.gz
dexon-sol-tools-303bbc42f4322448998f3fde202574335d1190e6.tar.bz2
dexon-sol-tools-303bbc42f4322448998f3fde202574335d1190e6.tar.lz
dexon-sol-tools-303bbc42f4322448998f3fde202574335d1190e6.tar.xz
dexon-sol-tools-303bbc42f4322448998f3fde202574335d1190e6.tar.zst
dexon-sol-tools-303bbc42f4322448998f3fde202574335d1190e6.zip
Change some column types from varchar to numeric
Diffstat (limited to 'packages/pipeline/src')
-rw-r--r--packages/pipeline/src/entities/exchange_cancel_up_to_event.ts7
-rw-r--r--packages/pipeline/src/entities/exchange_fill_event.ts18
-rw-r--r--packages/pipeline/src/entities/sra_order.ts26
-rw-r--r--packages/pipeline/src/ormconfig.ts2
-rw-r--r--packages/pipeline/src/parsers/events/index.ts10
-rw-r--r--packages/pipeline/src/parsers/sra_orders/index.ts12
-rw-r--r--packages/pipeline/src/scripts/pull_missing_events.ts2
-rw-r--r--packages/pipeline/src/utils/index.ts15
8 files changed, 57 insertions, 35 deletions
diff --git a/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts b/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts
index 7306a1a87..752631b85 100644
--- a/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts
+++ b/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts
@@ -1,5 +1,8 @@
+import { BigNumber } from '@0x/utils';
import { Column, Entity, PrimaryColumn } from 'typeorm';
+import { bigNumberTransformer } from '../utils';
+
@Entity({ name: 'exchange_cancel_up_to_events', schema: 'raw' })
export class ExchangeCancelUpToEvent {
@PrimaryColumn({ name: 'contract_address' })
@@ -17,7 +20,7 @@ export class ExchangeCancelUpToEvent {
public makerAddress!: string;
@Column({ name: 'sender_address' })
public senderAddress!: string;
- @Column({ name: 'order_epoch' })
- public orderEpoch!: string;
+ @Column({ name: 'order_epoch', type: 'numeric', transformer: bigNumberTransformer })
+ public orderEpoch!: BigNumber;
// TODO(albrow): Include topics?
}
diff --git a/packages/pipeline/src/entities/exchange_fill_event.ts b/packages/pipeline/src/entities/exchange_fill_event.ts
index 6202e558b..aa082436b 100644
--- a/packages/pipeline/src/entities/exchange_fill_event.ts
+++ b/packages/pipeline/src/entities/exchange_fill_event.ts
@@ -1,6 +1,8 @@
+import { BigNumber } from '@0x/utils';
import { Column, Entity, PrimaryColumn } from 'typeorm';
import { AssetType } from '../types';
+import { bigNumberTransformer } from '../utils';
@Entity({ name: 'exchange_fill_events', schema: 'raw' })
export class ExchangeFillEvent {
@@ -24,14 +26,14 @@ export class ExchangeFillEvent {
public feeRecipientAddress!: string;
@Column({ name: 'sender_address' })
public senderAddress!: string;
- @Column({ name: 'maker_asset_filled_amount' })
- public makerAssetFilledAmount!: string;
- @Column({ name: 'taker_asset_filled_amount' })
- public takerAssetFilledAmount!: string;
- @Column({ name: 'maker_fee_paid' })
- public makerFeePaid!: string;
- @Column({ name: 'taker_fee_paid' })
- public takerFeePaid!: string;
+ @Column({ name: 'maker_asset_filled_amount', type: 'numeric', transformer: bigNumberTransformer })
+ public makerAssetFilledAmount!: BigNumber;
+ @Column({ name: 'taker_asset_filled_amount', type: 'numeric', transformer: bigNumberTransformer })
+ public takerAssetFilledAmount!: BigNumber;
+ @Column({ name: 'maker_fee_paid', type: 'numeric', transformer: bigNumberTransformer })
+ public makerFeePaid!: BigNumber;
+ @Column({ name: 'taker_fee_paid', type: 'numeric', transformer: bigNumberTransformer })
+ public takerFeePaid!: BigNumber;
@Column({ name: 'order_hash' })
public orderHash!: string;
diff --git a/packages/pipeline/src/entities/sra_order.ts b/packages/pipeline/src/entities/sra_order.ts
index 4b7f652d3..9c730a0bb 100644
--- a/packages/pipeline/src/entities/sra_order.ts
+++ b/packages/pipeline/src/entities/sra_order.ts
@@ -1,6 +1,8 @@
+import { BigNumber } from '@0x/utils';
import { Column, Entity, PrimaryColumn } from 'typeorm';
import { AssetType } from '../types';
+import { bigNumberTransformer } from '../utils';
@Entity({ name: 'sra_orders', schema: 'raw' })
export class SraOrder {
@@ -19,18 +21,18 @@ export class SraOrder {
public feeRecipientAddress!: string;
@Column({ name: 'sender_address' })
public senderAddress!: string;
- @Column({ name: 'maker_asset_amount' })
- public makerAssetAmount!: string;
- @Column({ name: 'taker_asset_amount' })
- public takerAssetAmount!: string;
- @Column({ name: 'maker_fee' })
- public makerFee!: string;
- @Column({ name: 'taker_fee' })
- public takerFee!: string;
- @Column({ name: 'expiration_time_seconds' })
- public expirationTimeSeconds!: string;
- @Column({ name: 'salt' })
- public salt!: string;
+ @Column({ name: 'maker_asset_amount', type: 'numeric', transformer: bigNumberTransformer })
+ public makerAssetAmount!: BigNumber;
+ @Column({ name: 'taker_asset_amount', type: 'numeric', transformer: bigNumberTransformer })
+ public takerAssetAmount!: BigNumber;
+ @Column({ name: 'maker_fee', type: 'numeric', transformer: bigNumberTransformer })
+ public makerFee!: BigNumber;
+ @Column({ name: 'taker_fee', type: 'numeric', transformer: bigNumberTransformer })
+ public takerFee!: BigNumber;
+ @Column({ name: 'expiration_time_seconds', type: 'numeric', transformer: bigNumberTransformer })
+ public expirationTimeSeconds!: BigNumber;
+ @Column({ name: 'salt', type: 'numeric', transformer: bigNumberTransformer })
+ public salt!: BigNumber;
@Column({ name: 'signature' })
public signature!: string;
diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts
index 2f5f7df33..0e489e560 100644
--- a/packages/pipeline/src/ormconfig.ts
+++ b/packages/pipeline/src/ormconfig.ts
@@ -26,7 +26,7 @@ const config: ConnectionOptions = {
type: 'postgres',
url: process.env.ZEROEX_DATA_PIPELINE_DB_URL,
synchronize: false,
- logging: false,
+ logging: ['error', 'warn'],
entities,
migrations: ['./lib/migrations/**/*.js'],
};
diff --git a/packages/pipeline/src/parsers/events/index.ts b/packages/pipeline/src/parsers/events/index.ts
index d56dc7080..27e192d7b 100644
--- a/packages/pipeline/src/parsers/events/index.ts
+++ b/packages/pipeline/src/parsers/events/index.ts
@@ -65,10 +65,10 @@ export function _convertToExchangeFillEvent(eventLog: LogWithDecodedArgs<Exchang
exchangeFillEvent.takerAddress = eventLog.args.takerAddress.toString();
exchangeFillEvent.feeRecipientAddress = eventLog.args.feeRecipientAddress;
exchangeFillEvent.senderAddress = eventLog.args.senderAddress;
- exchangeFillEvent.makerAssetFilledAmount = eventLog.args.makerAssetFilledAmount.toString();
- exchangeFillEvent.takerAssetFilledAmount = eventLog.args.takerAssetFilledAmount.toString();
- exchangeFillEvent.makerFeePaid = eventLog.args.makerFeePaid.toString();
- exchangeFillEvent.takerFeePaid = eventLog.args.takerFeePaid.toString();
+ exchangeFillEvent.makerAssetFilledAmount = eventLog.args.makerAssetFilledAmount;
+ exchangeFillEvent.takerAssetFilledAmount = eventLog.args.takerAssetFilledAmount;
+ exchangeFillEvent.makerFeePaid = eventLog.args.makerFeePaid;
+ exchangeFillEvent.takerFeePaid = eventLog.args.takerFeePaid;
exchangeFillEvent.orderHash = eventLog.args.orderHash;
exchangeFillEvent.rawMakerAssetData = eventLog.args.makerAssetData;
exchangeFillEvent.makerAssetType = makerAssetType;
@@ -139,6 +139,6 @@ export function _convertToExchangeCancelUpToEvent(
exchangeCancelUpToEvent.rawData = eventLog.data as string;
exchangeCancelUpToEvent.makerAddress = eventLog.args.makerAddress.toString();
exchangeCancelUpToEvent.senderAddress = eventLog.args.senderAddress.toString();
- exchangeCancelUpToEvent.orderEpoch = eventLog.args.orderEpoch.toString();
+ exchangeCancelUpToEvent.orderEpoch = eventLog.args.orderEpoch;
return exchangeCancelUpToEvent;
}
diff --git a/packages/pipeline/src/parsers/sra_orders/index.ts b/packages/pipeline/src/parsers/sra_orders/index.ts
index 800521955..ef8901e40 100644
--- a/packages/pipeline/src/parsers/sra_orders/index.ts
+++ b/packages/pipeline/src/parsers/sra_orders/index.ts
@@ -34,12 +34,12 @@ export function _convertToEntity(apiOrder: APIOrder): SraOrder {
sraOrder.takerAddress = apiOrder.order.takerAddress;
sraOrder.feeRecipientAddress = apiOrder.order.feeRecipientAddress;
sraOrder.senderAddress = apiOrder.order.senderAddress;
- sraOrder.makerAssetAmount = apiOrder.order.makerAssetAmount.toString();
- sraOrder.takerAssetAmount = apiOrder.order.takerAssetAmount.toString();
- sraOrder.makerFee = apiOrder.order.makerFee.toString();
- sraOrder.takerFee = apiOrder.order.takerFee.toString();
- sraOrder.expirationTimeSeconds = apiOrder.order.expirationTimeSeconds.toString();
- sraOrder.salt = apiOrder.order.salt.toString();
+ sraOrder.makerAssetAmount = apiOrder.order.makerAssetAmount;
+ sraOrder.takerAssetAmount = apiOrder.order.takerAssetAmount;
+ sraOrder.makerFee = apiOrder.order.makerFee;
+ sraOrder.takerFee = apiOrder.order.takerFee;
+ sraOrder.expirationTimeSeconds = apiOrder.order.expirationTimeSeconds;
+ sraOrder.salt = apiOrder.order.salt;
sraOrder.signature = apiOrder.order.signature;
sraOrder.rawMakerAssetData = apiOrder.order.makerAssetData;
diff --git a/packages/pipeline/src/scripts/pull_missing_events.ts b/packages/pipeline/src/scripts/pull_missing_events.ts
index 0af999a77..1693bb59a 100644
--- a/packages/pipeline/src/scripts/pull_missing_events.ts
+++ b/packages/pipeline/src/scripts/pull_missing_events.ts
@@ -13,7 +13,7 @@ import { handleError } from '../utils';
const EXCHANGE_START_BLOCK = 6271590; // Block number when the Exchange contract was deployed to mainnet.
const START_BLOCK_OFFSET = 1000; // Number of blocks before the last known block to consider when updating fill events.
-const BATCH_SAVE_SIZE = 1000; // Number of events to save at once.
+const BATCH_SAVE_SIZE = 10000; // Number of events to save at once.
let connection: Connection;
diff --git a/packages/pipeline/src/utils/index.ts b/packages/pipeline/src/utils/index.ts
index a083cd8c9..78aa89374 100644
--- a/packages/pipeline/src/utils/index.ts
+++ b/packages/pipeline/src/utils/index.ts
@@ -1,4 +1,5 @@
import { BigNumber } from '@0x/utils';
+import { ValueTransformer } from 'typeorm/decorator/options/ValueTransformer';
/**
* If the given BigNumber is not null, returns the string representation of that
@@ -34,3 +35,17 @@ export function handleError(e: any): void {
}
process.exit(1);
}
+
+class BigNumberTransformer implements ValueTransformer {
+ // tslint:disable-next-line:prefer-function-over-method
+ public to(value: BigNumber): string {
+ return value.toString();
+ }
+
+ // tslint:disable-next-line:prefer-function-over-method
+ public from(value: string): BigNumber {
+ return new BigNumber(value);
+ }
+}
+
+export const bigNumberTransformer = new BigNumberTransformer();