Add typeorm and begin inserting database records

8 files changed, 166 insertions, 22 deletions

diff --git a/.gitignore b/.gitignore
index db53d8639..390d5d166 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,6 +14,10 @@ pids
 # OSX folder metadata
 .DS_Store
 
+# SQLite database files
+*.db
+*.sqlite
+
 # Directory for instrumented libs generated by jscoverage/JSCover
 lib-cov
 
diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json
index cdcd5ed18..d81ea2668 100644
--- a/packages/pipeline/package.json
+++ b/packages/pipeline/package.json
@@ -30,13 +30,13 @@
     "license": "Apache-2.0",
     "devDependencies": {
         "@0xproject/tslint-config": "^1.0.7",
-        "tslint": "5.11.0",
-        "typescript": "3.0.1",
-        "mocha": "^5.2.0",
         "chai": "^4.1.2",
         "chai-as-promised": "^7.1.1",
         "chai-bignumber": "^2.0.2",
-        "dirty-chai": "^2.0.1"
+        "dirty-chai": "^2.0.1",
+        "mocha": "^5.2.0",
+        "tslint": "5.11.0",
+        "typescript": "3.0.1"
     },
     "dependencies": {
         "@0xproject/contract-wrappers": "^1.0.1",
@@ -45,6 +45,9 @@
         "@types/ramda": "^0.25.38",
         "axios": "^0.18.0",
         "ethereum-types": "^1.0.6",
-        "ramda": "^0.25.0"
+        "ramda": "^0.25.0",
+        "reflect-metadata": "^0.1.12",
+        "sqlite3": "^4.0.2",
+        "typeorm": "^0.2.7"
     }
 }
diff --git a/packages/pipeline/src/data-sources/etherscan/events.ts b/packages/pipeline/src/data-sources/etherscan/events.ts
index 89b3ffac1..edc8cde7b 100644
--- a/packages/pipeline/src/data-sources/etherscan/events.ts
+++ b/packages/pipeline/src/data-sources/etherscan/events.ts
@@ -1,5 +1,6 @@
+import { ExchangeEventArgs } from '@0xproject/contract-wrappers';
 import { AbiDecoder } from '@0xproject/utils';
-import { AbiDefinition, DecodedLogArgs, LogEntry, LogWithDecodedArgs } from 'ethereum-types';
+import { AbiDefinition, LogEntry, LogWithDecodedArgs } from 'ethereum-types';
 import * as R from 'ramda';
 
 // Raw events response from etherescan.io
@@ -47,12 +48,12 @@ export function _convertResponseToLogEntry(result: EventsResponseResult): LogEnt
 // Decodes a LogEntry into a LogWithDecodedArgs
 // tslint:disable-next-line:completed-docs
 export const _decodeLogEntry = R.curry((contractAbi: AbiDefinition[], log: LogEntry): LogWithDecodedArgs<
-    DecodedLogArgs
+    ExchangeEventArgs
 > => {
     const abiDecoder = new AbiDecoder([contractAbi]);
     const logWithDecodedArgs = abiDecoder.tryToDecodeLogOrNoop(log);
     // tslint:disable-next-line:no-unnecessary-type-assertion
-    return logWithDecodedArgs as LogWithDecodedArgs<DecodedLogArgs>;
+    return logWithDecodedArgs as LogWithDecodedArgs<ExchangeEventArgs>;
 });
 
 /**
@@ -64,7 +65,7 @@ export const _decodeLogEntry = R.curry((contractAbi: AbiDefinition[], log: LogEn
 export function parseRawEventsResponse(
     contractAbi: AbiDefinition[],
     rawEventsResponse: EventsResponse,
-): Array<LogWithDecodedArgs<DecodedLogArgs>> {
+): Array<LogWithDecodedArgs<ExchangeEventArgs>> {
     return R.pipe(R.map(_convertResponseToLogEntry), R.map(_decodeLogEntry(contractAbi)))(rawEventsResponse.result);
 }
 
diff --git a/packages/pipeline/src/entities/ExchangeFillEvent.ts b/packages/pipeline/src/entities/ExchangeFillEvent.ts
new file mode 100644
index 000000000..1e9e8d986
--- /dev/null
+++ b/packages/pipeline/src/entities/ExchangeFillEvent.ts
@@ -0,0 +1,24 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+@Entity()
+export class ExchangeFillEvent {
+    @PrimaryColumn() public logIndex!: number;
+
+    @Column() public address!: string;
+    @Column() public rawData!: string;
+    @Column() public blockNumber!: number;
+
+    @Column() public makerAddress!: string;
+    @Column() public takerAddress!: string;
+    @Column() public feeRecepientAddress!: string;
+    @Column() public senderAddress!: string;
+    @Column() public makerAssetFilledAmount!: string;
+    @Column() public takerAssetFilledAmount!: string;
+    @Column() public makerFeePaid!: string;
+    @Column() public takerFeePaid!: string;
+    @Column() public orderHash!: string;
+    // TODO(albrow): Decode asset data.
+    @Column() public makerAssetData!: string;
+    @Column() public takerAssetData!: string;
+    // TODO(albrow): Include topics?
+}
diff --git a/packages/pipeline/src/index.ts b/packages/pipeline/src/index.ts
index eccb24d18..d70ec3e3e 100644
--- a/packages/pipeline/src/index.ts
+++ b/packages/pipeline/src/index.ts
@@ -1,13 +1,45 @@
-import { Etherscan } from './data-sources/etherscan';
+import { ExchangeFillEventArgs } from '@0xproject/contract-wrappers';
+import { LogWithDecodedArgs } from 'ethereum-types';
+import 'reflect-metadata';
+import { createConnection } from 'typeorm';
 
 import { artifacts } from './artifacts';
+import { Etherscan } from './data-sources/etherscan';
+import { ExchangeFillEvent } from './entities/ExchangeFillEvent';
+import { config } from './ormconfig';
 
 const etherscan = new Etherscan(process.env.ETHERSCAN_API_KEY as string);
 
 (async () => {
-    const events = await etherscan.getContractEventsAsync(
+    const connection = await createConnection(config);
+    const repository = connection.getRepository(ExchangeFillEvent);
+    console.log(`found ${await repository.count()} existing fill events`);
+    const eventLogs = await etherscan.getContractEventsAsync(
         '0x4f833a24e1f95d70f028921e27040ca56e09ab0b',
         artifacts.Exchange.compilerOutput.abi,
     );
-    console.log(events);
+    for (const eventLog of eventLogs) {
+        if (eventLog.event !== 'Fill') {
+            continue;
+        }
+        const fillEventLog = eventLog as LogWithDecodedArgs<ExchangeFillEventArgs>;
+        const exchangeFillEvent = new ExchangeFillEvent();
+        exchangeFillEvent.logIndex = fillEventLog.logIndex as number;
+        exchangeFillEvent.address = fillEventLog.address as string;
+        exchangeFillEvent.rawData = fillEventLog.data as string;
+        exchangeFillEvent.blockNumber = fillEventLog.blockNumber as number;
+        exchangeFillEvent.makerAddress = fillEventLog.args.makerAddress.toString();
+        exchangeFillEvent.takerAddress = fillEventLog.args.takerAddress.toString();
+        exchangeFillEvent.feeRecepientAddress = fillEventLog.args.feeRecipientAddress;
+        exchangeFillEvent.senderAddress = fillEventLog.args.senderAddress;
+        exchangeFillEvent.makerAssetFilledAmount = fillEventLog.args.makerAssetFilledAmount.toString();
+        exchangeFillEvent.takerAssetFilledAmount = fillEventLog.args.takerAssetFilledAmount.toString();
+        exchangeFillEvent.makerFeePaid = fillEventLog.args.makerFeePaid.toString();
+        exchangeFillEvent.takerFeePaid = fillEventLog.args.takerFeePaid.toString();
+        exchangeFillEvent.orderHash = fillEventLog.args.orderHash;
+        exchangeFillEvent.makerAssetData = fillEventLog.args.makerAssetData;
+        exchangeFillEvent.takerAssetData = fillEventLog.args.takerAssetData;
+        await repository.save(exchangeFillEvent);
+    }
+    console.log(`now ${await repository.count()} total fill events`);
 })();
diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts
new file mode 100644
index 000000000..48e316078
--- /dev/null
+++ b/packages/pipeline/src/ormconfig.ts
@@ -0,0 +1,14 @@
+import { ConnectionOptions } from 'typeorm';
+
+export const config: ConnectionOptions = {
+    type: 'sqlite',
+    database: 'database.sqlite',
+    synchronize: true,
+    logging: false,
+    entities: ['./lib/src/entities/**/*.js'],
+    migrations: ['./lib/src/migrations/**/*.js'],
+    cli: {
+        entitiesDir: 'lib/src/entities',
+        migrationsDir: 'lib/src/migrations',
+    },
+};
diff --git a/packages/pipeline/tsconfig.json b/packages/pipeline/tsconfig.json
index 2ee711adc..918d469ed 100644
--- a/packages/pipeline/tsconfig.json
+++ b/packages/pipeline/tsconfig.json
@@ -2,7 +2,9 @@
     "extends": "../../tsconfig",
     "compilerOptions": {
         "outDir": "lib",
-        "rootDir": "."
+        "rootDir": ".",
+        "emitDecoratorMetadata": true,
+        "experimentalDecorators": true
     },
     "include": ["./src/**/*", "./test/**/*"]
 }
diff --git a/yarn.lock b/yarn.lock
index d912ea7b4..f7f060dbc 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2322,6 +2322,10 @@ anymatch@^2.0.0:
     micromatch "^3.1.4"
     normalize-path "^2.1.1"
 
+app-root-path@^2.0.1:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/app-root-path/-/app-root-path-2.1.0.tgz#98bf6599327ecea199309866e8140368fd2e646a"
+
 append-buffer@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/append-buffer/-/append-buffer-1.0.2.tgz#d8220cf466081525efea50614f3de6514dfa58f1"
@@ -3797,7 +3801,7 @@ buffer@^5.0.3, buffer@^5.0.5:
     base64-js "^1.0.2"
     ieee754 "^1.1.4"
 
-buffer@^5.2.1:
+buffer@^5.1.0, buffer@^5.2.1:
   version "5.2.1"
   resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.2.1.tgz#dd57fa0f109ac59c602479044dca7b8b3d0b71d6"
   dependencies:
@@ -4250,6 +4254,16 @@ cli-cursor@^2.1.0:
   dependencies:
     restore-cursor "^2.0.0"
 
+cli-highlight@^1.2.3:
+  version "1.2.3"
+  resolved "https://registry.yarnpkg.com/cli-highlight/-/cli-highlight-1.2.3.tgz#b200f97ed0e43d24633e89de0f489a48bb87d2bf"
+  dependencies:
+    chalk "^2.3.0"
+    highlight.js "^9.6.0"
+    mz "^2.4.0"
+    parse5 "^3.0.3"
+    yargs "^10.0.3"
+
 cli-spinners@^0.1.2:
   version "0.1.2"
   resolved "https://registry.yarnpkg.com/cli-spinners/-/cli-spinners-0.1.2.tgz#bb764d88e185fb9e1e6a2a1f19772318f605e31c"
@@ -5668,6 +5682,10 @@ dotenv@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-4.0.0.tgz#864ef1379aced55ce6f95debecdce179f7a0cd1d"
 
+dotenv@^5.0.1:
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-5.0.1.tgz#a5317459bd3d79ab88cff6e44057a6a3fbb1fcef"
+
 drbg.js@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/drbg.js/-/drbg.js-1.0.1.tgz#3e36b6c42b37043823cdbc332d58f31e2445480b"
@@ -6810,6 +6828,10 @@ figgy-pudding@^3.1.0, figgy-pudding@^3.2.1, figgy-pudding@^3.4.1, figgy-pudding@
   version "3.5.1"
   resolved "https://registry.yarnpkg.com/figgy-pudding/-/figgy-pudding-3.5.1.tgz#862470112901c727a0e495a80744bd5baa1d6790"
 
+figlet@^1.1.1:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/figlet/-/figlet-1.2.0.tgz#6c46537378fab649146b5a6143dda019b430b410"
+
 figures@^1.3.5, figures@^1.7.0:
   version "1.7.0"
   resolved "https://registry.yarnpkg.com/figures/-/figures-1.7.0.tgz#cbe1e3affcf1cd44b80cadfed28dc793a9701d2e"
@@ -7973,7 +7995,7 @@ heap@~0.2.6:
   version "0.2.6"
   resolved "https://registry.yarnpkg.com/heap/-/heap-0.2.6.tgz#087e1f10b046932fc8594dd9e6d378afc9d1e5ac"
 
-highlight.js@^9.0.0, highlight.js@^9.11.0, highlight.js@~9.12.0:
+highlight.js@^9.0.0, highlight.js@^9.11.0, highlight.js@^9.6.0, highlight.js@~9.12.0:
   version "9.12.0"
   resolved "https://registry.yarnpkg.com/highlight.js/-/highlight.js-9.12.0.tgz#e6d9dbe57cbefe60751f02af336195870c90c01e"
 
@@ -9365,7 +9387,7 @@ js-yaml@3.x, js-yaml@^3.4.2, js-yaml@^3.6.1, js-yaml@^3.7.0:
     argparse "^1.0.7"
     esprima "^4.0.0"
 
-js-yaml@^3.12.0, js-yaml@^3.9.0, js-yaml@^3.9.1:
+js-yaml@^3.11.0, js-yaml@^3.12.0, js-yaml@^3.9.0, js-yaml@^3.9.1:
   version "3.12.0"
   resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.12.0.tgz#eaed656ec8344f10f527c6bfa1b6e2244de167d1"
   dependencies:
@@ -10965,7 +10987,7 @@ mute-stream@0.0.7, mute-stream@~0.0.4:
   version "0.0.7"
   resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.7.tgz#3075ce93bc21b8fab43e1bc4da7e8115ed1e7bab"
 
-mz@^2.6.0:
+mz@^2.4.0, mz@^2.6.0:
   version "2.7.0"
   resolved "https://registry.yarnpkg.com/mz/-/mz-2.7.0.tgz#95008057a56cafadc2bc63dde7f9ff6955948e32"
   dependencies:
@@ -10973,7 +10995,7 @@ mz@^2.6.0:
     object-assign "^4.0.1"
     thenify-all "^1.0.0"
 
-nan@2.10.0, nan@>=2.5.1, nan@^2.0.8, nan@^2.2.1, nan@^2.3.0, nan@^2.3.3, nan@^2.6.2, nan@^2.9.2:
+nan@2.10.0, nan@>=2.5.1, nan@^2.0.8, nan@^2.2.1, nan@^2.3.0, nan@^2.3.3, nan@^2.6.2, nan@^2.9.2, nan@~2.10.0:
   version "2.10.0"
   resolved "https://registry.yarnpkg.com/nan/-/nan-2.10.0.tgz#96d0cd610ebd58d4b4de9cc0c6828cda99c7548f"
 
@@ -11155,7 +11177,7 @@ node-notifier@^5.2.1:
     shellwords "^0.1.1"
     which "^1.3.0"
 
-node-pre-gyp@^0.10.0:
+node-pre-gyp@^0.10.0, node-pre-gyp@^0.10.3:
   version "0.10.3"
   resolved "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.10.3.tgz#3070040716afdc778747b61b6887bf78880b80fc"
   dependencies:
@@ -11889,6 +11911,10 @@ param-case@^2.1.0:
   dependencies:
     no-case "^2.2.0"
 
+parent-require@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/parent-require/-/parent-require-1.0.0.tgz#746a167638083a860b0eef6732cb27ed46c32977"
+
 parse-asn1@^5.0.0:
   version "5.1.1"
   resolved "https://registry.yarnpkg.com/parse-asn1/-/parse-asn1-5.1.1.tgz#f6bf293818332bd0dab54efb16087724745e6ca8"
@@ -11970,7 +11996,7 @@ parse5@4.0.0:
   version "4.0.0"
   resolved "https://registry.npmjs.org/parse5/-/parse5-4.0.0.tgz#6d78656e3da8d78b4ec0b906f7c08ef1dfe3f608"
 
-parse5@^3.0.1:
+parse5@^3.0.1, parse5@^3.0.3:
   version "3.0.3"
   resolved "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz#042f792ffdd36851551cf4e9e066b3874ab45b5c"
   dependencies:
@@ -13493,6 +13519,10 @@ redux@^3.6.0:
     loose-envify "^1.1.0"
     symbol-observable "^1.0.3"
 
+reflect-metadata@^0.1.12:
+  version "0.1.12"
+  resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.1.12.tgz#311bf0c6b63cd782f228a81abe146a2bfa9c56f2"
+
 refractor@^2.4.1:
   version "2.6.2"
   resolved "https://registry.npmjs.org/refractor/-/refractor-2.6.2.tgz#8e0877ab8864165275aafeea5d9c8eebe871552f"
@@ -14809,6 +14839,14 @@ sprintf-js@~1.0.2:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c"
 
+sqlite3@^4.0.2:
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-4.0.2.tgz#1bbeb68b03ead5d499e42a3a1b140064791c5a64"
+  dependencies:
+    nan "~2.10.0"
+    node-pre-gyp "^0.10.3"
+    request "^2.87.0"
+
 sshpk@^1.7.0:
   version "1.14.1"
   resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.14.1.tgz#130f5975eddad963f1d56f92b9ac6c51fa9f83eb"
@@ -15923,6 +15961,24 @@ typedoc@0.13.0:
     typedoc-default-themes "^0.5.0"
     typescript "3.1.x"
 
+typeorm@^0.2.7:
+  version "0.2.7"
+  resolved "https://registry.yarnpkg.com/typeorm/-/typeorm-0.2.7.tgz#4bbbace80dc91b1303be13f42d44ebf01d1b2558"
+  dependencies:
+    app-root-path "^2.0.1"
+    buffer "^5.1.0"
+    chalk "^2.3.2"
+    cli-highlight "^1.2.3"
+    debug "^3.1.0"
+    dotenv "^5.0.1"
+    glob "^7.1.2"
+    js-yaml "^3.11.0"
+    mkdirp "^0.5.1"
+    reflect-metadata "^0.1.12"
+    xml2js "^0.4.17"
+    yargonaut "^1.1.2"
+    yargs "^11.1.0"
+
 types-bn@^0.0.1:
   version "0.0.1"
   resolved "https://registry.yarnpkg.com/types-bn/-/types-bn-0.0.1.tgz#4253c7c7251b14e1d77cdca6f58800e5e2b82c4b"
@@ -17205,7 +17261,7 @@ xml2js@0.4.17:
     sax ">=0.6.0"
     xmlbuilder "^4.1.0"
 
-xml2js@0.4.19:
+xml2js@0.4.19, xml2js@^0.4.17:
   version "0.4.19"
   resolved "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7"
   dependencies:
@@ -17264,6 +17320,14 @@ yallist@^3.0.0, yallist@^3.0.2:
   version "3.0.2"
   resolved "https://registry.npmjs.org/yallist/-/yallist-3.0.2.tgz#8452b4bb7e83c7c188d8041c1a837c773d6d8bb9"
 
+yargonaut@^1.1.2:
+  version "1.1.4"
+  resolved "https://registry.yarnpkg.com/yargonaut/-/yargonaut-1.1.4.tgz#c64f56432c7465271221f53f5cc517890c3d6e0c"
+  dependencies:
+    chalk "^1.1.1"
+    figlet "^1.1.1"
+    parent-require "^1.0.0"
+
 yargs-parser@10.x, yargs-parser@^10.0.0, yargs-parser@^10.1.0:
   version "10.1.0"
   resolved "https://registry.npmjs.org/yargs-parser/-/yargs-parser-10.1.0.tgz#7202265b89f7e9e9f2e5765e0fe735a905edbaa8"
@@ -17301,7 +17365,7 @@ yargs-parser@^9.0.2:
   dependencies:
     camelcase "^4.1.0"
 
-yargs@11.1.0:
+yargs@11.1.0, yargs@^11.1.0:
   version "11.1.0"
   resolved "https://registry.yarnpkg.com/yargs/-/yargs-11.1.0.tgz#90b869934ed6e871115ea2ff58b03f4724ed2d77"
   dependencies: