From 3a19faa5ff8a66395c5eb8d1400338cefaa21187 Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Thu, 18 Oct 2018 17:44:48 -0700 Subject: Update script to work with existing v1 pipeline data --- packages/pipeline/package.json | 7 +- .../contract-wrappers/exchange_events.ts | 5 +- .../pipeline/src/entities/ExchangeCancelEvent.ts | 5 +- .../src/entities/ExchangeCancelUpToEvent.ts | 5 +- .../pipeline/src/entities/ExchangeFillEvent.ts | 7 +- packages/pipeline/src/entities/SraOrder.ts | 4 +- packages/pipeline/src/index.ts | 76 +++++++++++++++------- packages/pipeline/src/ormconfig.ts | 15 ++++- packages/pipeline/src/parsers/events/index.ts | 3 +- .../pipeline/test/parsers/events/index_test.ts | 3 +- 10 files changed, 90 insertions(+), 40 deletions(-) (limited to 'packages') diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index 0071fab2c..47033bf33 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -25,8 +25,8 @@ }, "license": "Apache-2.0", "devDependencies": { - "@types/ramda": "^0.25.38", "@0x/tslint-config": "^1.0.9", + "@types/ramda": "^0.25.38", "chai": "^4.1.2", "chai-as-promised": "^7.1.1", "chai-bignumber": "^2.0.2", @@ -36,10 +36,10 @@ "typescript": "3.0.1" }, "dependencies": { - "@0x/dev-utils": "^1.0.13", - "@0x/contract-artifacts": "^1.0.1", "@0x/connect": "^3.0.2", + "@0x/contract-artifacts": "^1.0.1", "@0x/contract-wrappers": "^3.0.0", + "@0x/dev-utils": "^1.0.13", "@0x/order-utils": "^2.0.0", "@0x/subproviders": "^2.1.0", "@0x/types": "^1.2.0", @@ -47,6 +47,7 @@ "@0x/web3-wrapper": "^3.1.0", "axios": "^0.18.0", "ethereum-types": "^1.0.6", + "pg": "^7.5.0", "ramda": "^0.25.0", "reflect-metadata": "^0.1.12", "sqlite3": "^4.0.2", diff --git a/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts b/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts index 46f115705..5ce8381cd 100644 --- a/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts +++ b/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts @@ -29,7 +29,10 @@ export class ExchangeEventsSource { let events: Array> = []; for (let currFromBlock = fromBlock; currFromBlock <= calculatedToBlock; currFromBlock += NUM_BLOCKS_PER_QUERY) { events = events.concat( - await this._getFillEventsForRangeAsync(currFromBlock, currFromBlock + NUM_BLOCKS_PER_QUERY - 1), + await this._getFillEventsForRangeAsync( + currFromBlock, + Math.min(currFromBlock + NUM_BLOCKS_PER_QUERY - 1, calculatedToBlock), + ), ); } return events; diff --git a/packages/pipeline/src/entities/ExchangeCancelEvent.ts b/packages/pipeline/src/entities/ExchangeCancelEvent.ts index 7010ab9f2..698b9e2ec 100644 --- a/packages/pipeline/src/entities/ExchangeCancelEvent.ts +++ b/packages/pipeline/src/entities/ExchangeCancelEvent.ts @@ -1,15 +1,16 @@ -import { BaseEntity, Column, Entity, PrimaryColumn } from 'typeorm'; +import { Column, Entity, PrimaryColumn } from 'typeorm'; import { AssetType } from '../types'; @Entity() -export class ExchangeCancelEvent extends BaseEntity { +export class ExchangeCancelEvent { @PrimaryColumn() public contractAddress!: string; @PrimaryColumn() public logIndex!: number; @PrimaryColumn() public blockNumber!: number; @Column() public rawData!: string; + // TODO(albrow): Include transaction hash @Column() public makerAddress!: string; @Column({ nullable: true, type: String }) public takerAddress!: string; diff --git a/packages/pipeline/src/entities/ExchangeCancelUpToEvent.ts b/packages/pipeline/src/entities/ExchangeCancelUpToEvent.ts index 03a02b069..3ca75ccf7 100644 --- a/packages/pipeline/src/entities/ExchangeCancelUpToEvent.ts +++ b/packages/pipeline/src/entities/ExchangeCancelUpToEvent.ts @@ -1,11 +1,12 @@ -import { BaseEntity, Column, Entity, PrimaryColumn } from 'typeorm'; +import { Column, Entity, PrimaryColumn } from 'typeorm'; @Entity() -export class ExchangeCancelUpToEvent extends BaseEntity { +export class ExchangeCancelUpToEvent { @PrimaryColumn() public contractAddress!: string; @PrimaryColumn() public logIndex!: number; @PrimaryColumn() public blockNumber!: number; + // TODO(albrow): Include transaction hash @Column() public rawData!: string; @Column() public makerAddress!: string; diff --git a/packages/pipeline/src/entities/ExchangeFillEvent.ts b/packages/pipeline/src/entities/ExchangeFillEvent.ts index 5eafa7449..6e549af93 100644 --- a/packages/pipeline/src/entities/ExchangeFillEvent.ts +++ b/packages/pipeline/src/entities/ExchangeFillEvent.ts @@ -1,18 +1,19 @@ -import { BaseEntity, Column, Entity, PrimaryColumn } from 'typeorm'; +import { Column, Entity, PrimaryColumn } from 'typeorm'; import { AssetType } from '../types'; @Entity() -export class ExchangeFillEvent extends BaseEntity { +export class ExchangeFillEvent { @PrimaryColumn() public contractAddress!: string; @PrimaryColumn() public logIndex!: number; @PrimaryColumn() public blockNumber!: number; @Column() public rawData!: string; + @Column() public transactionHash!: string; @Column() public makerAddress!: string; @Column() public takerAddress!: string; - @Column() public feeRecepientAddress!: string; + @Column() public feeRecipientAddress!: string; @Column() public senderAddress!: string; @Column() public makerAssetFilledAmount!: string; @Column() public takerAssetFilledAmount!: string; diff --git a/packages/pipeline/src/entities/SraOrder.ts b/packages/pipeline/src/entities/SraOrder.ts index e4987df57..a22f7c4e5 100644 --- a/packages/pipeline/src/entities/SraOrder.ts +++ b/packages/pipeline/src/entities/SraOrder.ts @@ -1,9 +1,9 @@ -import { BaseEntity, Column, Entity, PrimaryColumn } from 'typeorm'; +import { Column, Entity, PrimaryColumn } from 'typeorm'; import { AssetType } from '../types'; @Entity() -export class SraOrder extends BaseEntity { +export class SraOrder { @PrimaryColumn() public exchangeAddress!: string; @PrimaryColumn() public orderHashHex!: string; diff --git a/packages/pipeline/src/index.ts b/packages/pipeline/src/index.ts index d4bca65c5..58646fc56 100644 --- a/packages/pipeline/src/index.ts +++ b/packages/pipeline/src/index.ts @@ -1,50 +1,78 @@ -import { HttpClient } from '@0x/connect'; import { web3Factory } from '@0x/dev-utils'; import 'reflect-metadata'; import { Connection, createConnection } from 'typeorm'; import { ExchangeEventsSource } from './data_sources/contract-wrappers/exchange_events'; -import { SraOrder } from './entities/SraOrder'; -import { config } from './ormconfig'; +import { deployConfig } from './ormconfig'; import { parseExchangeEvents } from './parsers/events'; -import { parseSraOrders } from './parsers/sra_orders'; +import { ExchangeFillEvent } from './entities/ExchangeFillEvent'; let connection: Connection; (async () => { - connection = await createConnection(config); + connection = await createConnection(deployConfig); await getExchangeEventsAsync(); - // await getSraOrdersAsync(); + await mergeExchangeEventsAsync(); + console.log('Exiting process'); + process.exit(0); })(); // TODO(albrow): Separately: Errors do not appear to be handled correctly. If you use the // wrong rpcUrl it just returns early with no error. async function getExchangeEventsAsync(): Promise { + console.log('Getting event logs...'); const provider = web3Factory.getRpcProvider({ rpcUrl: 'https://mainnet.infura.io', }); + const eventsRepository = connection.getRepository(ExchangeFillEvent); const exchangeEvents = new ExchangeEventsSource(provider, 1); const eventLogs = await exchangeEvents.getFillEventsAsync(); + console.log('Parsing events...'); const events = parseExchangeEvents(eventLogs); - console.log('Got events: ' + events.length); - for (const event of events) { - await event.save(); - } + console.log(`Retrieved and parsed ${events.length} total events.`); + console.log('Saving events...'); + eventsRepository.save(events); console.log('Saved events.'); - console.log('Exiting process'); - process.exit(0); } -async function getSraOrdersAsync(): Promise { - const orderRepository = connection.getRepository(SraOrder); - console.log(`found ${await orderRepository.count()} existing orders`); - const sraUrl = 'https://api.radarrelay.com/0x/v2'; - const connect = new HttpClient(sraUrl); - const rawOrders = await connect.getOrdersAsync(); - const orders = parseSraOrders(rawOrders); - for (const order of orders) { - order.sourceUrl = sraUrl; - await order.save(); - } - console.log(`now there are ${await orderRepository.count()} total orders`); +const insertEventsRawQuery = `INSERT INTO events_raw ( + event_type, + error_id, + order_hash, + maker, + maker_amount, + maker_fee, + maker_token, + taker, + taker_amount, + taker_fee, + taker_token, + txn_hash, + fee_recipient, + block_number, + log_index +) +( + SELECT + 'LogFill', + null, + "orderHash", + "makerAddress", + "makerAssetFilledAmount"::numeric(78), + "makerFeePaid"::numeric(78), + "makerTokenAddress", + "takerAddress", + "takerAssetFilledAmount"::numeric(78), + "takerFeePaid"::numeric(78), + "takerTokenAddress", + "transactionHash", + "feeRecipientAddress", + "blockNumber", + "logIndex" + FROM exchange_fill_event +) ON CONFLICT (order_hash, txn_hash, log_index) DO NOTHING`; + +async function mergeExchangeEventsAsync(): Promise { + console.log('Merging results into events_raw...'); + await connection.query(insertEventsRawQuery); } diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts index 48e316078..eaafeb7d5 100644 --- a/packages/pipeline/src/ormconfig.ts +++ b/packages/pipeline/src/ormconfig.ts @@ -1,6 +1,6 @@ import { ConnectionOptions } from 'typeorm'; -export const config: ConnectionOptions = { +export const testConfig: ConnectionOptions = { type: 'sqlite', database: 'database.sqlite', synchronize: true, @@ -12,3 +12,16 @@ export const config: ConnectionOptions = { migrationsDir: 'lib/src/migrations', }, }; + +export const deployConfig: ConnectionOptions = { + type: 'postgres', + url: process.env.ZEROEX_DATA_PIPELINE_DB_URL, + synchronize: true, + logging: false, + entities: ['./lib/src/entities/**/*.js'], + migrations: ['./lib/src/migrations/**/*.js'], + cli: { + entitiesDir: 'lib/src/entities', + migrationsDir: 'lib/src/migrations', + }, +}; diff --git a/packages/pipeline/src/parsers/events/index.ts b/packages/pipeline/src/parsers/events/index.ts index b9b4d02cb..abfb7ddb6 100644 --- a/packages/pipeline/src/parsers/events/index.ts +++ b/packages/pipeline/src/parsers/events/index.ts @@ -43,9 +43,10 @@ export function _convertToExchangeFillEvent(eventLog: LogWithDecodedArgs { expected.logIndex = 102; expected.rawData = '0x000000000000000000000000f6da68519f78b0d0bc93c701e86affcb75c92428000000000000000000000000f6da68519f78b0d0bc93c701e86affcb75c92428000000000000000000000000000000000000000000000000002386f26fc10000000000000000000000000000000000000000000000000000016345785d8a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000001600000000000000000000000000000000000000000000000000000000000000024f47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000024f47261b0000000000000000000000000e41d2489571d322189246dafa5ebde1f4699f49800000000000000000000000000000000000000000000000000000000'; + expected.transactionHash = '0x6dd106d002873746072fc5e496dd0fb2541b68c77bcf9184ae19a42fd33657fe'; expected.makerAddress = '0xf6da68519f78b0d0bc93c701e86affcb75c92428'; expected.takerAddress = '0xf6da68519f78b0d0bc93c701e86affcb75c92428'; - expected.feeRecepientAddress = '0xc370d2a5920344aa6b7d8d11250e3e861434cbdd'; + expected.feeRecipientAddress = '0xc370d2a5920344aa6b7d8d11250e3e861434cbdd'; expected.senderAddress = '0xf6da68519f78b0d0bc93c701e86affcb75c92428'; expected.makerAssetFilledAmount = '10000000000000000'; expected.takerAssetFilledAmount = '100000000000000000'; -- cgit v1.2.3