aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.gitignore4
-rw-r--r--packages/pipeline/package.json13
-rw-r--r--packages/pipeline/src/data-sources/etherscan/events.ts9
-rw-r--r--packages/pipeline/src/entities/ExchangeFillEvent.ts24
-rw-r--r--packages/pipeline/src/index.ts38
-rw-r--r--packages/pipeline/src/ormconfig.ts14
-rw-r--r--packages/pipeline/tsconfig.json4
-rw-r--r--yarn.lock82
8 files changed, 166 insertions, 22 deletions
diff --git a/.gitignore b/.gitignore
index db53d8639..390d5d166 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,6 +14,10 @@ pids
# OSX folder metadata
.DS_Store
+# SQLite database files
+*.db
+*.sqlite
+
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json
index cdcd5ed18..d81ea2668 100644
--- a/packages/pipeline/package.json
+++ b/packages/pipeline/package.json
@@ -30,13 +30,13 @@
"license": "Apache-2.0",
"devDependencies": {
"@0xproject/tslint-config": "^1.0.7",
- "tslint": "5.11.0",
- "typescript": "3.0.1",
- "mocha": "^5.2.0",
"chai": "^4.1.2",
"chai-as-promised": "^7.1.1",
"chai-bignumber": "^2.0.2",
- "dirty-chai": "^2.0.1"
+ "dirty-chai": "^2.0.1",
+ "mocha": "^5.2.0",
+ "tslint": "5.11.0",
+ "typescript": "3.0.1"
},
"dependencies": {
"@0xproject/contract-wrappers": "^1.0.1",
@@ -45,6 +45,9 @@
"@types/ramda": "^0.25.38",
"axios": "^0.18.0",
"ethereum-types": "^1.0.6",
- "ramda": "^0.25.0"
+ "ramda": "^0.25.0",
+ "reflect-metadata": "^0.1.12",
+ "sqlite3": "^4.0.2",
+ "typeorm": "^0.2.7"
}
}
diff --git a/packages/pipeline/src/data-sources/etherscan/events.ts b/packages/pipeline/src/data-sources/etherscan/events.ts
index 89b3ffac1..edc8cde7b 100644
--- a/packages/pipeline/src/data-sources/etherscan/events.ts
+++ b/packages/pipeline/src/data-sources/etherscan/events.ts
@@ -1,5 +1,6 @@
+import { ExchangeEventArgs } from '@0xproject/contract-wrappers';
import { AbiDecoder } from '@0xproject/utils';
-import { AbiDefinition, DecodedLogArgs, LogEntry, LogWithDecodedArgs } from 'ethereum-types';
+import { AbiDefinition, LogEntry, LogWithDecodedArgs } from 'ethereum-types';
import * as R from 'ramda';
// Raw events response from etherescan.io
@@ -47,12 +48,12 @@ export function _convertResponseToLogEntry(result: EventsResponseResult): LogEnt
// Decodes a LogEntry into a LogWithDecodedArgs
// tslint:disable-next-line:completed-docs
export const _decodeLogEntry = R.curry((contractAbi: AbiDefinition[], log: LogEntry): LogWithDecodedArgs<
- DecodedLogArgs
+ ExchangeEventArgs
> => {
const abiDecoder = new AbiDecoder([contractAbi]);
const logWithDecodedArgs = abiDecoder.tryToDecodeLogOrNoop(log);
// tslint:disable-next-line:no-unnecessary-type-assertion
- return logWithDecodedArgs as LogWithDecodedArgs<DecodedLogArgs>;
+ return logWithDecodedArgs as LogWithDecodedArgs<ExchangeEventArgs>;
});
/**
@@ -64,7 +65,7 @@ export const _decodeLogEntry = R.curry((contractAbi: AbiDefinition[], log: LogEn
export function parseRawEventsResponse(
contractAbi: AbiDefinition[],
rawEventsResponse: EventsResponse,
-): Array<LogWithDecodedArgs<DecodedLogArgs>> {
+): Array<LogWithDecodedArgs<ExchangeEventArgs>> {
return R.pipe(R.map(_convertResponseToLogEntry), R.map(_decodeLogEntry(contractAbi)))(rawEventsResponse.result);
}
diff --git a/packages/pipeline/src/entities/ExchangeFillEvent.ts b/packages/pipeline/src/entities/ExchangeFillEvent.ts
new file mode 100644
index 000000000..1e9e8d986
--- /dev/null
+++ b/packages/pipeline/src/entities/ExchangeFillEvent.ts
@@ -0,0 +1,24 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+@Entity()
+export class ExchangeFillEvent {
+ @PrimaryColumn() public logIndex!: number;
+
+ @Column() public address!: string;
+ @Column() public rawData!: string;
+ @Column() public blockNumber!: number;
+
+ @Column() public makerAddress!: string;
+ @Column() public takerAddress!: string;
+ @Column() public feeRecepientAddress!: string;
+ @Column() public senderAddress!: string;
+ @Column() public makerAssetFilledAmount!: string;
+ @Column() public takerAssetFilledAmount!: string;
+ @Column() public makerFeePaid!: string;
+ @Column() public takerFeePaid!: string;
+ @Column() public orderHash!: string;
+ // TODO(albrow): Decode asset data.
+ @Column() public makerAssetData!: string;
+ @Column() public takerAssetData!: string;
+ // TODO(albrow): Include topics?
+}
diff --git a/packages/pipeline/src/index.ts b/packages/pipeline/src/index.ts
index eccb24d18..d70ec3e3e 100644
--- a/packages/pipeline/src/index.ts
+++ b/packages/pipeline/src/index.ts
@@ -1,13 +1,45 @@
-import { Etherscan } from './data-sources/etherscan';
+import { ExchangeFillEventArgs } from '@0xproject/contract-wrappers';
+import { LogWithDecodedArgs } from 'ethereum-types';
+import 'reflect-metadata';
+import { createConnection } from 'typeorm';
import { artifacts } from './artifacts';
+import { Etherscan } from './data-sources/etherscan';
+import { ExchangeFillEvent } from './entities/ExchangeFillEvent';
+import { config } from './ormconfig';
const etherscan = new Etherscan(process.env.ETHERSCAN_API_KEY as string);
(async () => {
- const events = await etherscan.getContractEventsAsync(
+ const connection = await createConnection(config);
+ const repository = connection.getRepository(ExchangeFillEvent);
+ console.log(`found ${await repository.count()} existing fill events`);
+ const eventLogs = await etherscan.getContractEventsAsync(
'0x4f833a24e1f95d70f028921e27040ca56e09ab0b',
artifacts.Exchange.compilerOutput.abi,
);
- console.log(events);
+ for (const eventLog of eventLogs) {
+ if (eventLog.event !== 'Fill') {
+ continue;
+ }
+ const fillEventLog = eventLog as LogWithDecodedArgs<ExchangeFillEventArgs>;
+ const exchangeFillEvent = new ExchangeFillEvent();
+ exchangeFillEvent.logIndex = fillEventLog.logIndex as number;
+ exchangeFillEvent.address = fillEventLog.address as string;
+ exchangeFillEvent.rawData = fillEventLog.data as string;
+ exchangeFillEvent.blockNumber = fillEventLog.blockNumber as number;
+ exchangeFillEvent.makerAddress = fillEventLog.args.makerAddress.toString();
+ exchangeFillEvent.takerAddress = fillEventLog.args.takerAddress.toString();
+ exchangeFillEvent.feeRecepientAddress = fillEventLog.args.feeRecipientAddress;
+ exchangeFillEvent.senderAddress = fillEventLog.args.senderAddress;
+ exchangeFillEvent.makerAssetFilledAmount = fillEventLog.args.makerAssetFilledAmount.toString();
+ exchangeFillEvent.takerAssetFilledAmount = fillEventLog.args.takerAssetFilledAmount.toString();
+ exchangeFillEvent.makerFeePaid = fillEventLog.args.makerFeePaid.toString();
+ exchangeFillEvent.takerFeePaid = fillEventLog.args.takerFeePaid.toString();
+ exchangeFillEvent.orderHash = fillEventLog.args.orderHash;
+ exchangeFillEvent.makerAssetData = fillEventLog.args.makerAssetData;
+ exchangeFillEvent.takerAssetData = fillEventLog.args.takerAssetData;
+ await repository.save(exchangeFillEvent);
+ }
+ console.log(`now ${await repository.count()} total fill events`);
})();
diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts
new file mode 100644
index 000000000..48e316078
--- /dev/null
+++ b/packages/pipeline/src/ormconfig.ts
@@ -0,0 +1,14 @@
+import { ConnectionOptions } from 'typeorm';
+
+export const config: ConnectionOptions = {
+ type: 'sqlite',
+ database: 'database.sqlite',
+ synchronize: true,
+ logging: false,
+ entities: ['./lib/src/entities/**/*.js'],
+ migrations: ['./lib/src/migrations/**/*.js'],
+ cli: {
+ entitiesDir: 'lib/src/entities',
+ migrationsDir: 'lib/src/migrations',
+ },
+};
diff --git a/packages/pipeline/tsconfig.json b/packages/pipeline/tsconfig.json
index 2ee711adc..918d469ed 100644
--- a/packages/pipeline/tsconfig.json
+++ b/packages/pipeline/tsconfig.json
@@ -2,7 +2,9 @@
"extends": "../../tsconfig",
"compilerOptions": {
"outDir": "lib",
- "rootDir": "."
+ "rootDir": ".",
+ "emitDecoratorMetadata": true,
+ "experimentalDecorators": true
},
"include": ["./src/**/*", "./test/**/*"]
}
diff --git a/yarn.lock b/yarn.lock
index d912ea7b4..f7f060dbc 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2322,6 +2322,10 @@ anymatch@^2.0.0:
micromatch "^3.1.4"
normalize-path "^2.1.1"
+app-root-path@^2.0.1:
+ version "2.1.0"
+ resolved "https://registry.yarnpkg.com/app-root-path/-/app-root-path-2.1.0.tgz#98bf6599327ecea199309866e8140368fd2e646a"
+
append-buffer@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/append-buffer/-/append-buffer-1.0.2.tgz#d8220cf466081525efea50614f3de6514dfa58f1"
@@ -3797,7 +3801,7 @@ buffer@^5.0.3, buffer@^5.0.5:
base64-js "^1.0.2"
ieee754 "^1.1.4"
-buffer@^5.2.1:
+buffer@^5.1.0, buffer@^5.2.1:
version "5.2.1"
resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.2.1.tgz#dd57fa0f109ac59c602479044dca7b8b3d0b71d6"
dependencies:
@@ -4250,6 +4254,16 @@ cli-cursor@^2.1.0:
dependencies:
restore-cursor "^2.0.0"
+cli-highlight@^1.2.3:
+ version "1.2.3"
+ resolved "https://registry.yarnpkg.com/cli-highlight/-/cli-highlight-1.2.3.tgz#b200f97ed0e43d24633e89de0f489a48bb87d2bf"
+ dependencies:
+ chalk "^2.3.0"
+ highlight.js "^9.6.0"
+ mz "^2.4.0"
+ parse5 "^3.0.3"
+ yargs "^10.0.3"
+
cli-spinners@^0.1.2:
version "0.1.2"
resolved "https://registry.yarnpkg.com/cli-spinners/-/cli-spinners-0.1.2.tgz#bb764d88e185fb9e1e6a2a1f19772318f605e31c"
@@ -5668,6 +5682,10 @@ dotenv@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-4.0.0.tgz#864ef1379aced55ce6f95debecdce179f7a0cd1d"
+dotenv@^5.0.1:
+ version "5.0.1"
+ resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-5.0.1.tgz#a5317459bd3d79ab88cff6e44057a6a3fbb1fcef"
+
drbg.js@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/drbg.js/-/drbg.js-1.0.1.tgz#3e36b6c42b37043823cdbc332d58f31e2445480b"
@@ -6810,6 +6828,10 @@ figgy-pudding@^3.1.0, figgy-pudding@^3.2.1, figgy-pudding@^3.4.1, figgy-pudding@
version "3.5.1"
resolved "https://registry.yarnpkg.com/figgy-pudding/-/figgy-pudding-3.5.1.tgz#862470112901c727a0e495a80744bd5baa1d6790"
+figlet@^1.1.1:
+ version "1.2.0"
+ resolved "https://registry.yarnpkg.com/figlet/-/figlet-1.2.0.tgz#6c46537378fab649146b5a6143dda019b430b410"
+
figures@^1.3.5, figures@^1.7.0:
version "1.7.0"
resolved "https://registry.yarnpkg.com/figures/-/figures-1.7.0.tgz#cbe1e3affcf1cd44b80cadfed28dc793a9701d2e"
@@ -7973,7 +7995,7 @@ heap@~0.2.6:
version "0.2.6"
resolved "https://registry.yarnpkg.com/heap/-/heap-0.2.6.tgz#087e1f10b046932fc8594dd9e6d378afc9d1e5ac"
-highlight.js@^9.0.0, highlight.js@^9.11.0, highlight.js@~9.12.0:
+highlight.js@^9.0.0, highlight.js@^9.11.0, highlight.js@^9.6.0, highlight.js@~9.12.0:
version "9.12.0"
resolved "https://registry.yarnpkg.com/highlight.js/-/highlight.js-9.12.0.tgz#e6d9dbe57cbefe60751f02af336195870c90c01e"
@@ -9365,7 +9387,7 @@ js-yaml@3.x, js-yaml@^3.4.2, js-yaml@^3.6.1, js-yaml@^3.7.0:
argparse "^1.0.7"
esprima "^4.0.0"
-js-yaml@^3.12.0, js-yaml@^3.9.0, js-yaml@^3.9.1:
+js-yaml@^3.11.0, js-yaml@^3.12.0, js-yaml@^3.9.0, js-yaml@^3.9.1:
version "3.12.0"
resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.12.0.tgz#eaed656ec8344f10f527c6bfa1b6e2244de167d1"
dependencies:
@@ -10965,7 +10987,7 @@ mute-stream@0.0.7, mute-stream@~0.0.4:
version "0.0.7"
resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.7.tgz#3075ce93bc21b8fab43e1bc4da7e8115ed1e7bab"
-mz@^2.6.0:
+mz@^2.4.0, mz@^2.6.0:
version "2.7.0"
resolved "https://registry.yarnpkg.com/mz/-/mz-2.7.0.tgz#95008057a56cafadc2bc63dde7f9ff6955948e32"
dependencies:
@@ -10973,7 +10995,7 @@ mz@^2.6.0:
object-assign "^4.0.1"
thenify-all "^1.0.0"
-nan@2.10.0, nan@>=2.5.1, nan@^2.0.8, nan@^2.2.1, nan@^2.3.0, nan@^2.3.3, nan@^2.6.2, nan@^2.9.2:
+nan@2.10.0, nan@>=2.5.1, nan@^2.0.8, nan@^2.2.1, nan@^2.3.0, nan@^2.3.3, nan@^2.6.2, nan@^2.9.2, nan@~2.10.0:
version "2.10.0"
resolved "https://registry.yarnpkg.com/nan/-/nan-2.10.0.tgz#96d0cd610ebd58d4b4de9cc0c6828cda99c7548f"
@@ -11155,7 +11177,7 @@ node-notifier@^5.2.1:
shellwords "^0.1.1"
which "^1.3.0"
-node-pre-gyp@^0.10.0:
+node-pre-gyp@^0.10.0, node-pre-gyp@^0.10.3:
version "0.10.3"
resolved "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.10.3.tgz#3070040716afdc778747b61b6887bf78880b80fc"
dependencies:
@@ -11889,6 +11911,10 @@ param-case@^2.1.0:
dependencies:
no-case "^2.2.0"
+parent-require@^1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/parent-require/-/parent-require-1.0.0.tgz#746a167638083a860b0eef6732cb27ed46c32977"
+
parse-asn1@^5.0.0:
version "5.1.1"
resolved "https://registry.yarnpkg.com/parse-asn1/-/parse-asn1-5.1.1.tgz#f6bf293818332bd0dab54efb16087724745e6ca8"
@@ -11970,7 +11996,7 @@ parse5@4.0.0:
version "4.0.0"
resolved "https://registry.npmjs.org/parse5/-/parse5-4.0.0.tgz#6d78656e3da8d78b4ec0b906f7c08ef1dfe3f608"
-parse5@^3.0.1:
+parse5@^3.0.1, parse5@^3.0.3:
version "3.0.3"
resolved "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz#042f792ffdd36851551cf4e9e066b3874ab45b5c"
dependencies:
@@ -13493,6 +13519,10 @@ redux@^3.6.0:
loose-envify "^1.1.0"
symbol-observable "^1.0.3"
+reflect-metadata@^0.1.12:
+ version "0.1.12"
+ resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.1.12.tgz#311bf0c6b63cd782f228a81abe146a2bfa9c56f2"
+
refractor@^2.4.1:
version "2.6.2"
resolved "https://registry.npmjs.org/refractor/-/refractor-2.6.2.tgz#8e0877ab8864165275aafeea5d9c8eebe871552f"
@@ -14809,6 +14839,14 @@ sprintf-js@~1.0.2:
version "1.0.3"
resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c"
+sqlite3@^4.0.2:
+ version "4.0.2"
+ resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-4.0.2.tgz#1bbeb68b03ead5d499e42a3a1b140064791c5a64"
+ dependencies:
+ nan "~2.10.0"
+ node-pre-gyp "^0.10.3"
+ request "^2.87.0"
+
sshpk@^1.7.0:
version "1.14.1"
resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.14.1.tgz#130f5975eddad963f1d56f92b9ac6c51fa9f83eb"
@@ -15923,6 +15961,24 @@ typedoc@0.13.0:
typedoc-default-themes "^0.5.0"
typescript "3.1.x"
+typeorm@^0.2.7:
+ version "0.2.7"
+ resolved "https://registry.yarnpkg.com/typeorm/-/typeorm-0.2.7.tgz#4bbbace80dc91b1303be13f42d44ebf01d1b2558"
+ dependencies:
+ app-root-path "^2.0.1"
+ buffer "^5.1.0"
+ chalk "^2.3.2"
+ cli-highlight "^1.2.3"
+ debug "^3.1.0"
+ dotenv "^5.0.1"
+ glob "^7.1.2"
+ js-yaml "^3.11.0"
+ mkdirp "^0.5.1"
+ reflect-metadata "^0.1.12"
+ xml2js "^0.4.17"
+ yargonaut "^1.1.2"
+ yargs "^11.1.0"
+
types-bn@^0.0.1:
version "0.0.1"
resolved "https://registry.yarnpkg.com/types-bn/-/types-bn-0.0.1.tgz#4253c7c7251b14e1d77cdca6f58800e5e2b82c4b"
@@ -17205,7 +17261,7 @@ xml2js@0.4.17:
sax ">=0.6.0"
xmlbuilder "^4.1.0"
-xml2js@0.4.19:
+xml2js@0.4.19, xml2js@^0.4.17:
version "0.4.19"
resolved "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7"
dependencies:
@@ -17264,6 +17320,14 @@ yallist@^3.0.0, yallist@^3.0.2:
version "3.0.2"
resolved "https://registry.npmjs.org/yallist/-/yallist-3.0.2.tgz#8452b4bb7e83c7c188d8041c1a837c773d6d8bb9"
+yargonaut@^1.1.2:
+ version "1.1.4"
+ resolved "https://registry.yarnpkg.com/yargonaut/-/yargonaut-1.1.4.tgz#c64f56432c7465271221f53f5cc517890c3d6e0c"
+ dependencies:
+ chalk "^1.1.1"
+ figlet "^1.1.1"
+ parent-require "^1.0.0"
+
yargs-parser@10.x, yargs-parser@^10.0.0, yargs-parser@^10.1.0:
version "10.1.0"
resolved "https://registry.npmjs.org/yargs-parser/-/yargs-parser-10.1.0.tgz#7202265b89f7e9e9f2e5765e0fe735a905edbaa8"
@@ -17301,7 +17365,7 @@ yargs-parser@^9.0.2:
dependencies:
camelcase "^4.1.0"
-yargs@11.1.0:
+yargs@11.1.0, yargs@^11.1.0:
version "11.1.0"
resolved "https://registry.yarnpkg.com/yargs/-/yargs-11.1.0.tgz#90b869934ed6e871115ea2ff58b03f4724ed2d77"
dependencies:
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-05 20:39:41 +0800