Update script to work with existing v1 pipeline data

10 files changed, 90 insertions, 40 deletions

diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json
index 0071fab2c..47033bf33 100644
--- a/packages/pipeline/package.json
+++ b/packages/pipeline/package.json
@@ -25,8 +25,8 @@
     },
     "license": "Apache-2.0",
     "devDependencies": {
-        "@types/ramda": "^0.25.38",
         "@0x/tslint-config": "^1.0.9",
+        "@types/ramda": "^0.25.38",
         "chai": "^4.1.2",
         "chai-as-promised": "^7.1.1",
         "chai-bignumber": "^2.0.2",
@@ -36,10 +36,10 @@
         "typescript": "3.0.1"
     },
     "dependencies": {
-        "@0x/dev-utils": "^1.0.13",
-        "@0x/contract-artifacts": "^1.0.1",
         "@0x/connect": "^3.0.2",
+        "@0x/contract-artifacts": "^1.0.1",
         "@0x/contract-wrappers": "^3.0.0",
+        "@0x/dev-utils": "^1.0.13",
         "@0x/order-utils": "^2.0.0",
         "@0x/subproviders": "^2.1.0",
         "@0x/types": "^1.2.0",
@@ -47,6 +47,7 @@
         "@0x/web3-wrapper": "^3.1.0",
         "axios": "^0.18.0",
         "ethereum-types": "^1.0.6",
+        "pg": "^7.5.0",
         "ramda": "^0.25.0",
         "reflect-metadata": "^0.1.12",
         "sqlite3": "^4.0.2",
diff --git a/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts b/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts
index 46f115705..5ce8381cd 100644
--- a/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts
+++ b/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts
@@ -29,7 +29,10 @@ export class ExchangeEventsSource {
         let events: Array<LogWithDecodedArgs<ExchangeFillEventArgs>> = [];
         for (let currFromBlock = fromBlock; currFromBlock <= calculatedToBlock; currFromBlock += NUM_BLOCKS_PER_QUERY) {
             events = events.concat(
-                await this._getFillEventsForRangeAsync(currFromBlock, currFromBlock + NUM_BLOCKS_PER_QUERY - 1),
+                await this._getFillEventsForRangeAsync(
+                    currFromBlock,
+                    Math.min(currFromBlock + NUM_BLOCKS_PER_QUERY - 1, calculatedToBlock),
+                ),
             );
         }
         return events;
diff --git a/packages/pipeline/src/entities/ExchangeCancelEvent.ts b/packages/pipeline/src/entities/ExchangeCancelEvent.ts
index 7010ab9f2..698b9e2ec 100644
--- a/packages/pipeline/src/entities/ExchangeCancelEvent.ts
+++ b/packages/pipeline/src/entities/ExchangeCancelEvent.ts
@@ -1,15 +1,16 @@
-import { BaseEntity, Column, Entity, PrimaryColumn } from 'typeorm';
+import { Column, Entity, PrimaryColumn } from 'typeorm';
 
 import { AssetType } from '../types';
 
 @Entity()
-export class ExchangeCancelEvent extends BaseEntity {
+export class ExchangeCancelEvent {
     @PrimaryColumn() public contractAddress!: string;
     @PrimaryColumn() public logIndex!: number;
     @PrimaryColumn() public blockNumber!: number;
 
     @Column() public rawData!: string;
 
+    // TODO(albrow): Include transaction hash
     @Column() public makerAddress!: string;
     @Column({ nullable: true, type: String })
     public takerAddress!: string;
diff --git a/packages/pipeline/src/entities/ExchangeCancelUpToEvent.ts b/packages/pipeline/src/entities/ExchangeCancelUpToEvent.ts
index 03a02b069..3ca75ccf7 100644
--- a/packages/pipeline/src/entities/ExchangeCancelUpToEvent.ts
+++ b/packages/pipeline/src/entities/ExchangeCancelUpToEvent.ts
@@ -1,11 +1,12 @@
-import { BaseEntity, Column, Entity, PrimaryColumn } from 'typeorm';
+import { Column, Entity, PrimaryColumn } from 'typeorm';
 
 @Entity()
-export class ExchangeCancelUpToEvent extends BaseEntity {
+export class ExchangeCancelUpToEvent {
     @PrimaryColumn() public contractAddress!: string;
     @PrimaryColumn() public logIndex!: number;
     @PrimaryColumn() public blockNumber!: number;
 
+    // TODO(albrow): Include transaction hash
     @Column() public rawData!: string;
 
     @Column() public makerAddress!: string;
diff --git a/packages/pipeline/src/entities/ExchangeFillEvent.ts b/packages/pipeline/src/entities/ExchangeFillEvent.ts
index 5eafa7449..6e549af93 100644
--- a/packages/pipeline/src/entities/ExchangeFillEvent.ts
+++ b/packages/pipeline/src/entities/ExchangeFillEvent.ts
@@ -1,18 +1,19 @@
-import { BaseEntity, Column, Entity, PrimaryColumn } from 'typeorm';
+import { Column, Entity, PrimaryColumn } from 'typeorm';
 
 import { AssetType } from '../types';
 
 @Entity()
-export class ExchangeFillEvent extends BaseEntity {
+export class ExchangeFillEvent {
     @PrimaryColumn() public contractAddress!: string;
     @PrimaryColumn() public logIndex!: number;
     @PrimaryColumn() public blockNumber!: number;
 
     @Column() public rawData!: string;
 
+    @Column() public transactionHash!: string;
     @Column() public makerAddress!: string;
     @Column() public takerAddress!: string;
-    @Column() public feeRecepientAddress!: string;
+    @Column() public feeRecipientAddress!: string;
     @Column() public senderAddress!: string;
     @Column() public makerAssetFilledAmount!: string;
     @Column() public takerAssetFilledAmount!: string;
diff --git a/packages/pipeline/src/entities/SraOrder.ts b/packages/pipeline/src/entities/SraOrder.ts
index e4987df57..a22f7c4e5 100644
--- a/packages/pipeline/src/entities/SraOrder.ts
+++ b/packages/pipeline/src/entities/SraOrder.ts
@@ -1,9 +1,9 @@
-import { BaseEntity, Column, Entity, PrimaryColumn } from 'typeorm';
+import { Column, Entity, PrimaryColumn } from 'typeorm';
 
 import { AssetType } from '../types';
 
 @Entity()
-export class SraOrder extends BaseEntity {
+export class SraOrder {
     @PrimaryColumn() public exchangeAddress!: string;
     @PrimaryColumn() public orderHashHex!: string;
 
diff --git a/packages/pipeline/src/index.ts b/packages/pipeline/src/index.ts
index d4bca65c5..58646fc56 100644
--- a/packages/pipeline/src/index.ts
+++ b/packages/pipeline/src/index.ts
@@ -1,50 +1,78 @@
-import { HttpClient } from '@0x/connect';
 import { web3Factory } from '@0x/dev-utils';
 import 'reflect-metadata';
 import { Connection, createConnection } from 'typeorm';
 
 import { ExchangeEventsSource } from './data_sources/contract-wrappers/exchange_events';
-import { SraOrder } from './entities/SraOrder';
-import { config } from './ormconfig';
+import { deployConfig } from './ormconfig';
 import { parseExchangeEvents } from './parsers/events';
-import { parseSraOrders } from './parsers/sra_orders';
+import { ExchangeFillEvent } from './entities/ExchangeFillEvent';
 
 let connection: Connection;
 
 (async () => {
-    connection = await createConnection(config);
+    connection = await createConnection(deployConfig);
     await getExchangeEventsAsync();
-    // await getSraOrdersAsync();
+    await mergeExchangeEventsAsync();
+    console.log('Exiting process');
+    process.exit(0);
 })();
 
 // TODO(albrow): Separately: Errors do not appear to be handled correctly. If you use the
 // wrong rpcUrl it just returns early with no error.
 async function getExchangeEventsAsync(): Promise<void> {
+    console.log('Getting event logs...');
     const provider = web3Factory.getRpcProvider({
         rpcUrl: 'https://mainnet.infura.io',
     });
+    const eventsRepository = connection.getRepository(ExchangeFillEvent);
     const exchangeEvents = new ExchangeEventsSource(provider, 1);
     const eventLogs = await exchangeEvents.getFillEventsAsync();
+    console.log('Parsing events...');
     const events = parseExchangeEvents(eventLogs);
-    console.log('Got events: ' + events.length);
-    for (const event of events) {
-        await event.save();
-    }
+    console.log(`Retrieved and parsed ${events.length} total events.`);
+    console.log('Saving events...');
+    eventsRepository.save(events);
     console.log('Saved events.');
-    console.log('Exiting process');
-    process.exit(0);
 }
 
-async function getSraOrdersAsync(): Promise<void> {
-    const orderRepository = connection.getRepository(SraOrder);
-    console.log(`found ${await orderRepository.count()} existing orders`);
-    const sraUrl = 'https://api.radarrelay.com/0x/v2';
-    const connect = new HttpClient(sraUrl);
-    const rawOrders = await connect.getOrdersAsync();
-    const orders = parseSraOrders(rawOrders);
-    for (const order of orders) {
-        order.sourceUrl = sraUrl;
-        await order.save();
-    }
-    console.log(`now there are ${await orderRepository.count()} total orders`);
+const insertEventsRawQuery = `INSERT INTO events_raw (
+    event_type,
+    error_id,
+    order_hash,
+    maker,
+    maker_amount,
+    maker_fee,
+    maker_token,
+    taker,
+    taker_amount,
+    taker_fee,
+    taker_token,
+    txn_hash,
+    fee_recipient,
+    block_number,
+    log_index
+)
+(
+    SELECT
+        'LogFill',
+        null,
+        "orderHash",
+        "makerAddress",
+        "makerAssetFilledAmount"::numeric(78),
+        "makerFeePaid"::numeric(78),
+        "makerTokenAddress",
+        "takerAddress",
+        "takerAssetFilledAmount"::numeric(78),
+        "takerFeePaid"::numeric(78),
+        "takerTokenAddress",
+        "transactionHash",
+        "feeRecipientAddress",
+        "blockNumber",
+        "logIndex"
+    FROM exchange_fill_event
+) ON CONFLICT (order_hash, txn_hash, log_index) DO NOTHING`;
+
+async function mergeExchangeEventsAsync(): Promise<void> {
+    console.log('Merging results into events_raw...');
+    await connection.query(insertEventsRawQuery);
 }
diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts
index 48e316078..eaafeb7d5 100644
--- a/packages/pipeline/src/ormconfig.ts
+++ b/packages/pipeline/src/ormconfig.ts
@@ -1,6 +1,6 @@
 import { ConnectionOptions } from 'typeorm';
 
-export const config: ConnectionOptions = {
+export const testConfig: ConnectionOptions = {
     type: 'sqlite',
     database: 'database.sqlite',
     synchronize: true,
@@ -12,3 +12,16 @@ export const config: ConnectionOptions = {
         migrationsDir: 'lib/src/migrations',
     },
 };
+
+export const deployConfig: ConnectionOptions = {
+    type: 'postgres',
+    url: process.env.ZEROEX_DATA_PIPELINE_DB_URL,
+    synchronize: true,
+    logging: false,
+    entities: ['./lib/src/entities/**/*.js'],
+    migrations: ['./lib/src/migrations/**/*.js'],
+    cli: {
+        entitiesDir: 'lib/src/entities',
+        migrationsDir: 'lib/src/migrations',
+    },
+};
diff --git a/packages/pipeline/src/parsers/events/index.ts b/packages/pipeline/src/parsers/events/index.ts
index b9b4d02cb..abfb7ddb6 100644
--- a/packages/pipeline/src/parsers/events/index.ts
+++ b/packages/pipeline/src/parsers/events/index.ts
@@ -43,9 +43,10 @@ export function _convertToExchangeFillEvent(eventLog: LogWithDecodedArgs<Exchang
     exchangeFillEvent.blockNumber = eventLog.blockNumber as number;
     exchangeFillEvent.logIndex = eventLog.logIndex as number;
     exchangeFillEvent.rawData = eventLog.data as string;
+    exchangeFillEvent.transactionHash = eventLog.transactionHash;
     exchangeFillEvent.makerAddress = eventLog.args.makerAddress.toString();
     exchangeFillEvent.takerAddress = eventLog.args.takerAddress.toString();
-    exchangeFillEvent.feeRecepientAddress = eventLog.args.feeRecipientAddress;
+    exchangeFillEvent.feeRecipientAddress = eventLog.args.feeRecipientAddress;
     exchangeFillEvent.senderAddress = eventLog.args.senderAddress;
     exchangeFillEvent.makerAssetFilledAmount = eventLog.args.makerAssetFilledAmount.toString();
     exchangeFillEvent.takerAssetFilledAmount = eventLog.args.takerAssetFilledAmount.toString();
diff --git a/packages/pipeline/test/parsers/events/index_test.ts b/packages/pipeline/test/parsers/events/index_test.ts
index 0730e4013..451988f8e 100644
--- a/packages/pipeline/test/parsers/events/index_test.ts
+++ b/packages/pipeline/test/parsers/events/index_test.ts
@@ -51,9 +51,10 @@ describe('exchange_events', () => {
             expected.logIndex = 102;
             expected.rawData =
                 '0x000000000000000000000000f6da68519f78b0d0bc93c701e86affcb75c92428000000000000000000000000f6da68519f78b0d0bc93c701e86affcb75c92428000000000000000000000000000000000000000000000000002386f26fc10000000000000000000000000000000000000000000000000000016345785d8a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000001600000000000000000000000000000000000000000000000000000000000000024f47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000024f47261b0000000000000000000000000e41d2489571d322189246dafa5ebde1f4699f49800000000000000000000000000000000000000000000000000000000';
+            expected.transactionHash = '0x6dd106d002873746072fc5e496dd0fb2541b68c77bcf9184ae19a42fd33657fe';
             expected.makerAddress = '0xf6da68519f78b0d0bc93c701e86affcb75c92428';
             expected.takerAddress = '0xf6da68519f78b0d0bc93c701e86affcb75c92428';
-            expected.feeRecepientAddress = '0xc370d2a5920344aa6b7d8d11250e3e861434cbdd';
+            expected.feeRecipientAddress = '0xc370d2a5920344aa6b7d8d11250e3e861434cbdd';
             expected.senderAddress = '0xf6da68519f78b0d0bc93c701e86affcb75c92428';
             expected.makerAssetFilledAmount = '10000000000000000';
             expected.takerAssetFilledAmount = '100000000000000000';