aboutsummaryrefslogtreecommitdiffstats
path: root/packages/pipeline
diff options
context:
space:
mode:
Diffstat (limited to 'packages/pipeline')
-rw-r--r--packages/pipeline/migrations/1542234704666-ConvertBigNumberToNumeric.ts53
-rw-r--r--packages/pipeline/src/entities/exchange_cancel_up_to_event.ts7
-rw-r--r--packages/pipeline/src/entities/exchange_fill_event.ts18
-rw-r--r--packages/pipeline/src/entities/sra_order.ts26
-rw-r--r--packages/pipeline/src/ormconfig.ts2
-rw-r--r--packages/pipeline/src/parsers/events/index.ts10
-rw-r--r--packages/pipeline/src/parsers/sra_orders/index.ts12
-rw-r--r--packages/pipeline/src/scripts/pull_missing_events.ts2
-rw-r--r--packages/pipeline/src/utils/index.ts15
-rw-r--r--packages/pipeline/test/parsers/events/index_test.ts10
-rw-r--r--packages/pipeline/test/parsers/sra_orders/index_test.ts12
11 files changed, 121 insertions, 46 deletions
diff --git a/packages/pipeline/migrations/1542234704666-ConvertBigNumberToNumeric.ts b/packages/pipeline/migrations/1542234704666-ConvertBigNumberToNumeric.ts
new file mode 100644
index 000000000..5200ef7cc
--- /dev/null
+++ b/packages/pipeline/migrations/1542234704666-ConvertBigNumberToNumeric.ts
@@ -0,0 +1,53 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class ConvertBigNumberToNumeric1542234704666 implements MigrationInterface {
+ public async up(queryRunner: QueryRunner): Promise<any> {
+ await queryRunner.query(
+ `ALTER TABLE raw.exchange_fill_events
+ ALTER COLUMN maker_asset_filled_amount TYPE numeric USING maker_asset_filled_amount::numeric,
+ ALTER COLUMN taker_asset_filled_amount TYPE numeric USING taker_asset_filled_amount::numeric,
+ ALTER COLUMN maker_fee_paid TYPE numeric USING maker_fee_paid::numeric,
+ ALTER COLUMN taker_fee_paid TYPE numeric USING taker_fee_paid::numeric;`,
+ );
+
+ await queryRunner.query(
+ `ALTER TABLE raw.exchange_cancel_up_to_events
+ ALTER COLUMN order_epoch TYPE numeric USING order_epoch::numeric;`,
+ );
+
+ await queryRunner.query(
+ `ALTER TABLE raw.sra_orders
+ ALTER COLUMN maker_asset_amount TYPE numeric USING maker_asset_amount::numeric,
+ ALTER COLUMN taker_asset_amount TYPE numeric USING taker_asset_amount::numeric,
+ ALTER COLUMN maker_fee TYPE numeric USING maker_fee::numeric,
+ ALTER COLUMN taker_fee TYPE numeric USING taker_fee::numeric,
+ ALTER COLUMN expiration_time_seconds TYPE numeric USING expiration_time_seconds::numeric,
+ ALTER COLUMN salt TYPE numeric USING salt::numeric;`,
+ );
+ }
+
+ public async down(queryRunner: QueryRunner): Promise<any> {
+ await queryRunner.query(
+ `ALTER TABLE raw.sra_orders
+ ALTER COLUMN maker_asset_amount TYPE varchar USING maker_asset_amount::varchar,
+ ALTER COLUMN taker_asset_amount TYPE varchar USING taker_asset_amount::varchar,
+ ALTER COLUMN maker_fee TYPE varchar USING maker_fee::varchar,
+ ALTER COLUMN taker_fee TYPE varchar USING taker_fee::varchar,
+ ALTER COLUMN expiration_time_seconds TYPE varchar USING expiration_time_seconds::varchar,
+ ALTER COLUMN salt TYPE varchar USING salt::varchar;`,
+ );
+
+ await queryRunner.query(
+ `ALTER TABLE raw.exchange_cancel_up_to_events
+ ALTER COLUMN order_epoch TYPE varchar USING order_epoch::varchar;`,
+ );
+
+ await queryRunner.query(
+ `ALTER TABLE raw.exchange_fill_events
+ ALTER COLUMN maker_asset_filled_amount TYPE varchar USING maker_asset_filled_amount::varchar,
+ ALTER COLUMN taker_asset_filled_amount TYPE varchar USING taker_asset_filled_amount::varchar,
+ ALTER COLUMN maker_fee_paid TYPE varchar USING maker_fee_paid::varchar,
+ ALTER COLUMN taker_fee_paid TYPE varchar USING taker_fee_paid::varchar;`,
+ );
+ }
+}
diff --git a/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts b/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts
index 7306a1a87..752631b85 100644
--- a/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts
+++ b/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts
@@ -1,5 +1,8 @@
+import { BigNumber } from '@0x/utils';
import { Column, Entity, PrimaryColumn } from 'typeorm';
+import { bigNumberTransformer } from '../utils';
+
@Entity({ name: 'exchange_cancel_up_to_events', schema: 'raw' })
export class ExchangeCancelUpToEvent {
@PrimaryColumn({ name: 'contract_address' })
@@ -17,7 +20,7 @@ export class ExchangeCancelUpToEvent {
public makerAddress!: string;
@Column({ name: 'sender_address' })
public senderAddress!: string;
- @Column({ name: 'order_epoch' })
- public orderEpoch!: string;
+ @Column({ name: 'order_epoch', type: 'numeric', transformer: bigNumberTransformer })
+ public orderEpoch!: BigNumber;
// TODO(albrow): Include topics?
}
diff --git a/packages/pipeline/src/entities/exchange_fill_event.ts b/packages/pipeline/src/entities/exchange_fill_event.ts
index 6202e558b..aa082436b 100644
--- a/packages/pipeline/src/entities/exchange_fill_event.ts
+++ b/packages/pipeline/src/entities/exchange_fill_event.ts
@@ -1,6 +1,8 @@
+import { BigNumber } from '@0x/utils';
import { Column, Entity, PrimaryColumn } from 'typeorm';
import { AssetType } from '../types';
+import { bigNumberTransformer } from '../utils';
@Entity({ name: 'exchange_fill_events', schema: 'raw' })
export class ExchangeFillEvent {
@@ -24,14 +26,14 @@ export class ExchangeFillEvent {
public feeRecipientAddress!: string;
@Column({ name: 'sender_address' })
public senderAddress!: string;
- @Column({ name: 'maker_asset_filled_amount' })
- public makerAssetFilledAmount!: string;
- @Column({ name: 'taker_asset_filled_amount' })
- public takerAssetFilledAmount!: string;
- @Column({ name: 'maker_fee_paid' })
- public makerFeePaid!: string;
- @Column({ name: 'taker_fee_paid' })
- public takerFeePaid!: string;
+ @Column({ name: 'maker_asset_filled_amount', type: 'numeric', transformer: bigNumberTransformer })
+ public makerAssetFilledAmount!: BigNumber;
+ @Column({ name: 'taker_asset_filled_amount', type: 'numeric', transformer: bigNumberTransformer })
+ public takerAssetFilledAmount!: BigNumber;
+ @Column({ name: 'maker_fee_paid', type: 'numeric', transformer: bigNumberTransformer })
+ public makerFeePaid!: BigNumber;
+ @Column({ name: 'taker_fee_paid', type: 'numeric', transformer: bigNumberTransformer })
+ public takerFeePaid!: BigNumber;
@Column({ name: 'order_hash' })
public orderHash!: string;
diff --git a/packages/pipeline/src/entities/sra_order.ts b/packages/pipeline/src/entities/sra_order.ts
index 4b7f652d3..9c730a0bb 100644
--- a/packages/pipeline/src/entities/sra_order.ts
+++ b/packages/pipeline/src/entities/sra_order.ts
@@ -1,6 +1,8 @@
+import { BigNumber } from '@0x/utils';
import { Column, Entity, PrimaryColumn } from 'typeorm';
import { AssetType } from '../types';
+import { bigNumberTransformer } from '../utils';
@Entity({ name: 'sra_orders', schema: 'raw' })
export class SraOrder {
@@ -19,18 +21,18 @@ export class SraOrder {
public feeRecipientAddress!: string;
@Column({ name: 'sender_address' })
public senderAddress!: string;
- @Column({ name: 'maker_asset_amount' })
- public makerAssetAmount!: string;
- @Column({ name: 'taker_asset_amount' })
- public takerAssetAmount!: string;
- @Column({ name: 'maker_fee' })
- public makerFee!: string;
- @Column({ name: 'taker_fee' })
- public takerFee!: string;
- @Column({ name: 'expiration_time_seconds' })
- public expirationTimeSeconds!: string;
- @Column({ name: 'salt' })
- public salt!: string;
+ @Column({ name: 'maker_asset_amount', type: 'numeric', transformer: bigNumberTransformer })
+ public makerAssetAmount!: BigNumber;
+ @Column({ name: 'taker_asset_amount', type: 'numeric', transformer: bigNumberTransformer })
+ public takerAssetAmount!: BigNumber;
+ @Column({ name: 'maker_fee', type: 'numeric', transformer: bigNumberTransformer })
+ public makerFee!: BigNumber;
+ @Column({ name: 'taker_fee', type: 'numeric', transformer: bigNumberTransformer })
+ public takerFee!: BigNumber;
+ @Column({ name: 'expiration_time_seconds', type: 'numeric', transformer: bigNumberTransformer })
+ public expirationTimeSeconds!: BigNumber;
+ @Column({ name: 'salt', type: 'numeric', transformer: bigNumberTransformer })
+ public salt!: BigNumber;
@Column({ name: 'signature' })
public signature!: string;
diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts
index 2f5f7df33..0e489e560 100644
--- a/packages/pipeline/src/ormconfig.ts
+++ b/packages/pipeline/src/ormconfig.ts
@@ -26,7 +26,7 @@ const config: ConnectionOptions = {
type: 'postgres',
url: process.env.ZEROEX_DATA_PIPELINE_DB_URL,
synchronize: false,
- logging: false,
+ logging: ['error', 'warn'],
entities,
migrations: ['./lib/migrations/**/*.js'],
};
diff --git a/packages/pipeline/src/parsers/events/index.ts b/packages/pipeline/src/parsers/events/index.ts
index d56dc7080..27e192d7b 100644
--- a/packages/pipeline/src/parsers/events/index.ts
+++ b/packages/pipeline/src/parsers/events/index.ts
@@ -65,10 +65,10 @@ export function _convertToExchangeFillEvent(eventLog: LogWithDecodedArgs<Exchang
exchangeFillEvent.takerAddress = eventLog.args.takerAddress.toString();
exchangeFillEvent.feeRecipientAddress = eventLog.args.feeRecipientAddress;
exchangeFillEvent.senderAddress = eventLog.args.senderAddress;
- exchangeFillEvent.makerAssetFilledAmount = eventLog.args.makerAssetFilledAmount.toString();
- exchangeFillEvent.takerAssetFilledAmount = eventLog.args.takerAssetFilledAmount.toString();
- exchangeFillEvent.makerFeePaid = eventLog.args.makerFeePaid.toString();
- exchangeFillEvent.takerFeePaid = eventLog.args.takerFeePaid.toString();
+ exchangeFillEvent.makerAssetFilledAmount = eventLog.args.makerAssetFilledAmount;
+ exchangeFillEvent.takerAssetFilledAmount = eventLog.args.takerAssetFilledAmount;
+ exchangeFillEvent.makerFeePaid = eventLog.args.makerFeePaid;
+ exchangeFillEvent.takerFeePaid = eventLog.args.takerFeePaid;
exchangeFillEvent.orderHash = eventLog.args.orderHash;
exchangeFillEvent.rawMakerAssetData = eventLog.args.makerAssetData;
exchangeFillEvent.makerAssetType = makerAssetType;
@@ -139,6 +139,6 @@ export function _convertToExchangeCancelUpToEvent(
exchangeCancelUpToEvent.rawData = eventLog.data as string;
exchangeCancelUpToEvent.makerAddress = eventLog.args.makerAddress.toString();
exchangeCancelUpToEvent.senderAddress = eventLog.args.senderAddress.toString();
- exchangeCancelUpToEvent.orderEpoch = eventLog.args.orderEpoch.toString();
+ exchangeCancelUpToEvent.orderEpoch = eventLog.args.orderEpoch;
return exchangeCancelUpToEvent;
}
diff --git a/packages/pipeline/src/parsers/sra_orders/index.ts b/packages/pipeline/src/parsers/sra_orders/index.ts
index 800521955..ef8901e40 100644
--- a/packages/pipeline/src/parsers/sra_orders/index.ts
+++ b/packages/pipeline/src/parsers/sra_orders/index.ts
@@ -34,12 +34,12 @@ export function _convertToEntity(apiOrder: APIOrder): SraOrder {
sraOrder.takerAddress = apiOrder.order.takerAddress;
sraOrder.feeRecipientAddress = apiOrder.order.feeRecipientAddress;
sraOrder.senderAddress = apiOrder.order.senderAddress;
- sraOrder.makerAssetAmount = apiOrder.order.makerAssetAmount.toString();
- sraOrder.takerAssetAmount = apiOrder.order.takerAssetAmount.toString();
- sraOrder.makerFee = apiOrder.order.makerFee.toString();
- sraOrder.takerFee = apiOrder.order.takerFee.toString();
- sraOrder.expirationTimeSeconds = apiOrder.order.expirationTimeSeconds.toString();
- sraOrder.salt = apiOrder.order.salt.toString();
+ sraOrder.makerAssetAmount = apiOrder.order.makerAssetAmount;
+ sraOrder.takerAssetAmount = apiOrder.order.takerAssetAmount;
+ sraOrder.makerFee = apiOrder.order.makerFee;
+ sraOrder.takerFee = apiOrder.order.takerFee;
+ sraOrder.expirationTimeSeconds = apiOrder.order.expirationTimeSeconds;
+ sraOrder.salt = apiOrder.order.salt;
sraOrder.signature = apiOrder.order.signature;
sraOrder.rawMakerAssetData = apiOrder.order.makerAssetData;
diff --git a/packages/pipeline/src/scripts/pull_missing_events.ts b/packages/pipeline/src/scripts/pull_missing_events.ts
index 0af999a77..1693bb59a 100644
--- a/packages/pipeline/src/scripts/pull_missing_events.ts
+++ b/packages/pipeline/src/scripts/pull_missing_events.ts
@@ -13,7 +13,7 @@ import { handleError } from '../utils';
const EXCHANGE_START_BLOCK = 6271590; // Block number when the Exchange contract was deployed to mainnet.
const START_BLOCK_OFFSET = 1000; // Number of blocks before the last known block to consider when updating fill events.
-const BATCH_SAVE_SIZE = 1000; // Number of events to save at once.
+const BATCH_SAVE_SIZE = 10000; // Number of events to save at once.
let connection: Connection;
diff --git a/packages/pipeline/src/utils/index.ts b/packages/pipeline/src/utils/index.ts
index a083cd8c9..78aa89374 100644
--- a/packages/pipeline/src/utils/index.ts
+++ b/packages/pipeline/src/utils/index.ts
@@ -1,4 +1,5 @@
import { BigNumber } from '@0x/utils';
+import { ValueTransformer } from 'typeorm/decorator/options/ValueTransformer';
/**
* If the given BigNumber is not null, returns the string representation of that
@@ -34,3 +35,17 @@ export function handleError(e: any): void {
}
process.exit(1);
}
+
+class BigNumberTransformer implements ValueTransformer {
+ // tslint:disable-next-line:prefer-function-over-method
+ public to(value: BigNumber): string {
+ return value.toString();
+ }
+
+ // tslint:disable-next-line:prefer-function-over-method
+ public from(value: string): BigNumber {
+ return new BigNumber(value);
+ }
+}
+
+export const bigNumberTransformer = new BigNumberTransformer();
diff --git a/packages/pipeline/test/parsers/events/index_test.ts b/packages/pipeline/test/parsers/events/index_test.ts
index fdd07f246..63e080edc 100644
--- a/packages/pipeline/test/parsers/events/index_test.ts
+++ b/packages/pipeline/test/parsers/events/index_test.ts
@@ -39,7 +39,7 @@ describe('exchange_events', () => {
makerAssetFilledAmount: new BigNumber('10000000000000000'),
takerAssetFilledAmount: new BigNumber('100000000000000000'),
makerFeePaid: new BigNumber('0'),
- takerFeePaid: new BigNumber('0'),
+ takerFeePaid: new BigNumber('12345'),
orderHash: '0xab12ed2cbaa5615ab690b9da75a46e53ddfcf3f1a68655b5fe0d94c75a1aac4a',
makerAssetData: '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2',
takerAssetData: '0xf47261b0000000000000000000000000e41d2489571d322189246dafa5ebde1f4699f498',
@@ -56,10 +56,10 @@ describe('exchange_events', () => {
expected.takerAddress = '0xf6da68519f78b0d0bc93c701e86affcb75c92428';
expected.feeRecipientAddress = '0xc370d2a5920344aa6b7d8d11250e3e861434cbdd';
expected.senderAddress = '0xf6da68519f78b0d0bc93c701e86affcb75c92428';
- expected.makerAssetFilledAmount = '10000000000000000';
- expected.takerAssetFilledAmount = '100000000000000000';
- expected.makerFeePaid = '0';
- expected.takerFeePaid = '0';
+ expected.makerAssetFilledAmount = new BigNumber('10000000000000000');
+ expected.takerAssetFilledAmount = new BigNumber('100000000000000000');
+ expected.makerFeePaid = new BigNumber('0');
+ expected.takerFeePaid = new BigNumber('12345');
expected.orderHash = '0xab12ed2cbaa5615ab690b9da75a46e53ddfcf3f1a68655b5fe0d94c75a1aac4a';
expected.rawMakerAssetData = '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2';
expected.makerAssetType = 'erc20';
diff --git a/packages/pipeline/test/parsers/sra_orders/index_test.ts b/packages/pipeline/test/parsers/sra_orders/index_test.ts
index 534d84ab3..ee2842ef3 100644
--- a/packages/pipeline/test/parsers/sra_orders/index_test.ts
+++ b/packages/pipeline/test/parsers/sra_orders/index_test.ts
@@ -41,12 +41,12 @@ describe('sra_orders', () => {
expected.takerAddress = '0x0000000000000000000000000000000000000000';
expected.feeRecipientAddress = '0xa258b39954cef5cb142fd567a46cddb31a670124';
expected.senderAddress = '0x0000000000000000000000000000000000000000';
- expected.makerAssetAmount = '1619310371000000000';
- expected.takerAssetAmount = '8178335207070707070707';
- expected.makerFee = '0';
- expected.takerFee = '0';
- expected.expirationTimeSeconds = '1538529488';
- expected.salt = '1537924688891';
+ expected.makerAssetAmount = new BigNumber('1619310371000000000');
+ expected.takerAssetAmount = new BigNumber('8178335207070707070707');
+ expected.makerFee = new BigNumber('0');
+ expected.takerFee = new BigNumber('0');
+ expected.expirationTimeSeconds = new BigNumber('1538529488');
+ expected.salt = new BigNumber('1537924688891');
expected.signature =
'0x1b5a5d672b0d647b5797387ccbb89d822d5d2e873346b014f4ff816ff0783f2a7a0d2824d2d7042ec8ea375bc7f870963e1cb8248f1db03ddf125e27b5963aa11f03';
expected.rawMakerAssetData = '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2';
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-06 18:27:56 +0800