aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAlex Browne <stephenalexbrowne@gmail.com>2018-09-18 02:27:38 +0800
committerAlex Browne <stephenalexbrowne@gmail.com>2018-12-05 06:21:42 +0800
commit57e7119c0d4f1ab7dd1d4c0118e72dc1706e2151 (patch)
tree68596be050d6f84e3b7e6f51f91d09a542ac3129
parentc43ba6b3c7082544f8aa8f1c297eb271ca079ea4 (diff)
downloaddexon-0x-contracts-57e7119c0d4f1ab7dd1d4c0118e72dc1706e2151.tar
dexon-0x-contracts-57e7119c0d4f1ab7dd1d4c0118e72dc1706e2151.tar.gz
dexon-0x-contracts-57e7119c0d4f1ab7dd1d4c0118e72dc1706e2151.tar.bz2
dexon-0x-contracts-57e7119c0d4f1ab7dd1d4c0118e72dc1706e2151.tar.lz
dexon-0x-contracts-57e7119c0d4f1ab7dd1d4c0118e72dc1706e2151.tar.xz
dexon-0x-contracts-57e7119c0d4f1ab7dd1d4c0118e72dc1706e2151.tar.zst
dexon-0x-contracts-57e7119c0d4f1ab7dd1d4c0118e72dc1706e2151.zip
Rebase pipeline branch off development
-rw-r--r--packages/pipeline/.npmignore7
-rw-r--r--packages/pipeline/README.md54
-rw-r--r--packages/pipeline/package.json52
-rw-r--r--packages/pipeline/src/global.d.ts6
-rw-r--r--packages/pipeline/src/models/block.ts22
-rw-r--r--packages/pipeline/src/models/event.ts84
-rw-r--r--packages/pipeline/src/models/historical_prices.ts43
-rw-r--r--packages/pipeline/src/models/order.ts30
-rw-r--r--packages/pipeline/src/models/price.ts15
-rw-r--r--packages/pipeline/src/models/relayer.ts75
-rw-r--r--packages/pipeline/src/models/tokens.ts24
-rw-r--r--packages/pipeline/src/models/transaction.ts36
-rw-r--r--packages/pipeline/src/postgres.ts12
-rw-r--r--packages/pipeline/src/run_jobs.ts87
-rw-r--r--packages/pipeline/src/scrape_prices.ts10
-rw-r--r--packages/pipeline/src/scripts/create_tables.ts258
-rw-r--r--packages/pipeline/src/scripts/join_tables.ts234
-rw-r--r--packages/pipeline/src/scripts/query_data.ts87
-rw-r--r--packages/pipeline/src/scripts/scrape_data.ts649
-rw-r--r--packages/pipeline/src/types.ts6
-rw-r--r--packages/pipeline/src/utils.ts181
-rw-r--r--packages/pipeline/src/zrx.ts11
-rw-r--r--packages/pipeline/tsconfig.json8
-rw-r--r--packages/pipeline/tslint.json3
-rw-r--r--packages/pipeline/typedoc-tsconfig.json7
-rw-r--r--yarn.lock732
26 files changed, 2666 insertions, 67 deletions
diff --git a/packages/pipeline/.npmignore b/packages/pipeline/.npmignore
new file mode 100644
index 000000000..89302c908
--- /dev/null
+++ b/packages/pipeline/.npmignore
@@ -0,0 +1,7 @@
+.*
+yarn-error.log
+/scripts/
+/generated_docs/
+/src/
+tsconfig.json
+/lib/monorepo_scripts/
diff --git a/packages/pipeline/README.md b/packages/pipeline/README.md
new file mode 100644
index 000000000..594454bd0
--- /dev/null
+++ b/packages/pipeline/README.md
@@ -0,0 +1,54 @@
+## @0xproject/pipeline
+
+This repository contains scripts used for scraping data from the Ethereum blockchain into SQL tables for analysis by the 0x team.
+
+## Contributing
+
+We strongly recommend that the community help us make improvements and determine the future direction of the protocol. To report bugs within this package, please create an issue in this repository.
+
+Please read our [contribution guidelines](../../CONTRIBUTING.md) before getting started.
+
+## Local Dev Setup
+
+Requires Node version 6.9.5 or higher.
+
+Add the following to your `.env` file:
+
+```
+REDSHIFT_USER
+REDSHIFT_DB
+REDSHIFT_PASSWORD
+REDSHIFT_PORT
+REDSHIFT_HOST
+WEB3_PROVIDER_URL
+```
+
+Running a script example:
+
+```
+node ./lib/scripts/scrape_data.js --type tokens
+```
+
+### Install dependencies:
+
+```bash
+yarn install
+```
+
+### Build
+
+```bash
+yarn build
+```
+
+### Clean
+
+```bash
+yarn clean
+```
+
+### Lint
+
+```bash
+yarn lint
+```
diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json
new file mode 100644
index 000000000..9d47f69f3
--- /dev/null
+++ b/packages/pipeline/package.json
@@ -0,0 +1,52 @@
+{
+ "name": "@0xproject/pipeline",
+ "version": "0.0.1",
+ "private": true,
+ "description": "Data pipeline for offline analysis",
+ "scripts": {
+ "build": "yarn tsc -b",
+ "test": "yarn run_mocha",
+ "rebuild_and_test": "run-s build test",
+ "test:circleci": "yarn test:coverage",
+ "run_mocha": "mocha --require source-map-support/register --require make-promises-safe lib/test/**/*_test.js --bail --exit",
+ "test:coverage": "nyc npm run test --all && yarn coverage:report:lcov",
+ "coverage:report:lcov": "nyc report --reporter=text-lcov > coverage/lcov.info",
+ "clean": "shx rm -rf lib",
+ "lint": "tslint --project . --exclude **/src/generated_contract_wrappers/**/*"
+ },
+ "repository": {
+ "type": "git",
+ "url": "https://github.com/0xProject/0x.js.git"
+ },
+ "license": "Apache-2.0",
+ "bugs": {
+ "url": "https://github.com/0xProject/0x.js/issues"
+ },
+ "homepage": "https://github.com/0xProject/0x.js/packages/pipeline/README.md",
+ "devDependencies": {
+ "@0xproject/tslint-config": "^1.0.7",
+ "@types/command-line-args": "^4.0.2",
+ "@types/dotenv": "^4.0.2",
+ "@types/glob": "^5.0.33",
+ "@types/lodash": "4.14.104",
+ "@types/node": "^8.0.53",
+ "@types/pg": "^7.4.1",
+ "@types/request": "^2.0.13",
+ "shx": "^0.2.2",
+ "tslint": "5.11.0",
+ "typescript": "3.0.1"
+ },
+ "dependencies": {
+ "0x.js": "^0.38.3",
+ "@0xproject/connect": "^0.6.14",
+ "airtable": "^0.5.2",
+ "command-line-args": "^4.0.7",
+ "dotenv": "^4.0.0",
+ "lodash": "^4.17.4",
+ "node-redshift": "^0.1.5",
+ "pg": "^7.4.1",
+ "queue": "^4.4.2",
+ "request": "^2.83.0",
+ "web3": "^0.20.0"
+ }
+}
diff --git a/packages/pipeline/src/global.d.ts b/packages/pipeline/src/global.d.ts
new file mode 100644
index 000000000..0af4ddb30
--- /dev/null
+++ b/packages/pipeline/src/global.d.ts
@@ -0,0 +1,6 @@
+declare module 'queue';
+declare module 'airtable';
+declare module '*.json' {
+ const value: any;
+ export default value;
+}
diff --git a/packages/pipeline/src/models/block.ts b/packages/pipeline/src/models/block.ts
new file mode 100644
index 000000000..a81cdb293
--- /dev/null
+++ b/packages/pipeline/src/models/block.ts
@@ -0,0 +1,22 @@
+const block = {
+ tableName: 'blocks',
+ tableProperties: {
+ id: {
+ type: 'key',
+ },
+ timestamp: {
+ type: 'timestamp',
+ required: true,
+ },
+ block_number: {
+ type: 'bigint',
+ required: true,
+ },
+ },
+};
+const logToBlockSchemaMapping: any = {
+ number: 'block_number',
+ hash: 'block_hash',
+ timestamp: 'timestamp',
+};
+export { block, logToBlockSchemaMapping };
diff --git a/packages/pipeline/src/models/event.ts b/packages/pipeline/src/models/event.ts
new file mode 100644
index 000000000..cb1c939e5
--- /dev/null
+++ b/packages/pipeline/src/models/event.ts
@@ -0,0 +1,84 @@
+const event = {
+ tableName: 'events',
+ tableProperties: {
+ id: {
+ type: 'key',
+ },
+ timestamp: {
+ type: 'timestamp',
+ required: true,
+ },
+ event_type: {
+ type: 'varchar',
+ required: true,
+ },
+ error_id: {
+ type: 'varchar',
+ },
+ order_hash: {
+ type: 'char(66)',
+ },
+ maker: {
+ type: 'char(42)',
+ },
+ maker_amount: {
+ type: 'varchar',
+ },
+ maker_fee: {
+ type: 'varchar',
+ },
+ maker_token: {
+ type: 'char(42)',
+ },
+ taker_amount: {
+ type: 'varchar',
+ },
+ taker_fee: {
+ type: 'varchar',
+ },
+ taker_token: {
+ type: 'char(42)',
+ },
+ txn_hash: {
+ type: 'char(66)',
+ },
+ gas_used: {
+ type: 'varchar',
+ },
+ gas_price: {
+ type: 'varchar',
+ },
+ fee_recipient: {
+ type: 'char(42)',
+ },
+ method_id: {
+ type: 'char(10)',
+ },
+ salt: {
+ type: 'varchar',
+ },
+ block_number: {
+ type: 'bigint',
+ },
+ },
+};
+const logToEventSchemaMapping: any = {
+ blockNumber: 'block_number',
+ transactionHash: 'txn_hash',
+ event: 'event_type',
+ logIndex: 'log_index',
+ 'args.maker': 'maker',
+ 'args.taker': 'taker',
+ 'args.feeRecipient': 'fee_recipient',
+ 'args.makerToken': 'maker_token',
+ 'args.takerToken': 'taker_token',
+ 'args.filledMakerTokenAmount': 'maker_amount',
+ 'args.filledTakerTokenAmount': 'taker_amount',
+ 'args.paidMakerFee': 'maker_fee',
+ 'args.paidTakerFee': 'taker_fee',
+ 'args.orderHash': 'order_hash',
+ 'args.cancelledMakerTokenAmount': 'maker_amount',
+ 'args.cancelledTakerTokenAmount': 'taker_amount',
+ 'args.errorId': 'error_id',
+};
+export { event, logToEventSchemaMapping };
diff --git a/packages/pipeline/src/models/historical_prices.ts b/packages/pipeline/src/models/historical_prices.ts
new file mode 100644
index 000000000..cf49b579e
--- /dev/null
+++ b/packages/pipeline/src/models/historical_prices.ts
@@ -0,0 +1,43 @@
+const historicalPrices = {
+ tableName: 'historical_prices',
+ tableProperties: {
+ token: {
+ type: 'varchar',
+ },
+ base: {
+ type: 'varchar',
+ },
+ timestamp: {
+ type: 'timestamp',
+ },
+ close: {
+ type: 'numeric(50)',
+ },
+ high: {
+ type: 'numeric(50)',
+ },
+ low: {
+ type: 'numeric(50)',
+ },
+ open: {
+ type: 'numeric(50)',
+ },
+ volume_from: {
+ type: 'numeric(50)',
+ },
+ volume_to: {
+ type: 'numeric(50)',
+ },
+ },
+};
+const logToHistoricalPricesSchema: { [log: string]: string } = {
+ token: 'token',
+ time: 'timestamp',
+ close: 'close',
+ high: 'high',
+ low: 'low',
+ open: 'open',
+ volumefrom: 'volume_from',
+ volumeto: 'volume_to',
+};
+export { historicalPrices, logToHistoricalPricesSchema };
diff --git a/packages/pipeline/src/models/order.ts b/packages/pipeline/src/models/order.ts
new file mode 100644
index 000000000..dadae2883
--- /dev/null
+++ b/packages/pipeline/src/models/order.ts
@@ -0,0 +1,30 @@
+const order = {
+ tableName: 'orders',
+ tableProperties: {
+ id: {
+ type: 'key',
+ },
+ timestamp: {
+ type: 'timestamp',
+ required: true,
+ },
+ block_number: {
+ type: 'bigint',
+ required: true,
+ },
+ },
+};
+const logToOrderSchemaMapping: any = {
+ exchangeContractAddress: 'exchange_contract_address',
+ maker: 'maker',
+ makerTokenAddress: 'maker_token',
+ makerTokenAmount: 'maker_amount',
+ makerFee: 'maker_fee',
+ taker: 'taker',
+ takerTokenAddress: 'taker_token',
+ takerTokenAmount: 'taker_amount',
+ takerFee: 'taker_fee',
+ expirationUnixTimestampSec: 'expiration_unix_timestamp_sec',
+ salt: 'salt',
+};
+export { order, logToOrderSchemaMapping };
diff --git a/packages/pipeline/src/models/price.ts b/packages/pipeline/src/models/price.ts
new file mode 100644
index 000000000..9e45cc2a9
--- /dev/null
+++ b/packages/pipeline/src/models/price.ts
@@ -0,0 +1,15 @@
+const price = {
+ tableName: 'prices',
+ tableProperties: {
+ address: {
+ type: 'char(42)',
+ },
+ timestamp: {
+ type: 'timestamp',
+ },
+ price: {
+ type: 'numeric(50)',
+ },
+ },
+};
+export { price };
diff --git a/packages/pipeline/src/models/relayer.ts b/packages/pipeline/src/models/relayer.ts
new file mode 100644
index 000000000..77a123e8b
--- /dev/null
+++ b/packages/pipeline/src/models/relayer.ts
@@ -0,0 +1,75 @@
+// const relayer = {
+// tableName: 'relayers',
+// tableProperties: {
+// id: {
+// type: 'integer',
+// },
+// name: {
+// type: 'varchar',
+// },
+// url : {
+// type: 'varchar',
+// },
+// model: {
+// type: 'varchar[]',
+// },
+// status: {
+// type: 'varchar',
+// },
+// sra_status: {
+// type: 'varchar',
+// },
+// sra_http_url: {
+// type: 'varchar',
+// },
+// known_fee_addresses: {
+// type: 'char(42)[]',
+// },
+// known_taker_addresses: {
+// type: 'char(42)[]',
+// },
+// relayer_type: {
+// type: 'varchar',
+// },
+// },
+// };
+const relayer = {
+ tableName: 'relayers',
+ tableProperties: {
+ name: {
+ type: 'varchar',
+ },
+ url: {
+ type: 'varchar',
+ },
+ sra_http_endpoint: {
+ type: 'varchar',
+ },
+ sra_ws_endpoint: {
+ type: 'varchar',
+ },
+ fee_recipient_addresses: {
+ type: 'char(42)[]',
+ },
+ taker_addresses: {
+ type: 'char(42)[]',
+ },
+ },
+};
+// const logToRelayerSchemaMapping: any = {
+// 'id' : 'id',
+// 'fields[\'Name\']': 'name',
+// 'fields[\'URL\']': 'url',
+// 'fields[\'Model\']': 'model',
+// 'fields[\'Status\']': 'status',
+// 'fields[\'SRA Status\']': 'sra_status',
+// 'fields[\'SRA HTTP URL\']': 'sra_http_url',
+// 'fields[\'Known Fee Addresses\']': 'known_fee_addresses',
+// 'fields[\'Known Taker Addresses\']': 'known_taker_addresses',
+// 'fields[\'Relayer Type\']': 'relayer_type',
+// };
+const logToRelayerSchemaMapping: any = {
+ name: 'name',
+ homepage_url: 'url',
+};
+export { relayer, logToRelayerSchemaMapping };
diff --git a/packages/pipeline/src/models/tokens.ts b/packages/pipeline/src/models/tokens.ts
new file mode 100644
index 000000000..96e8a31af
--- /dev/null
+++ b/packages/pipeline/src/models/tokens.ts
@@ -0,0 +1,24 @@
+const token = {
+ tableName: 'tokens',
+ tableProperties: {
+ address: {
+ type: 'char(66)',
+ },
+ decimals: {
+ type: 'bigint',
+ },
+ name: {
+ type: 'varchar',
+ },
+ symbol: {
+ type: 'varchar',
+ },
+ },
+};
+const logToTokenSchemaMapping: any = {
+ address: 'address',
+ decimals: 'decimals',
+ name: 'name',
+ symbol: 'symbol',
+};
+export { token, logToTokenSchemaMapping };
diff --git a/packages/pipeline/src/models/transaction.ts b/packages/pipeline/src/models/transaction.ts
new file mode 100644
index 000000000..715cc9480
--- /dev/null
+++ b/packages/pipeline/src/models/transaction.ts
@@ -0,0 +1,36 @@
+const transaction = {
+ tableName: 'transactions',
+ tableProperties: {
+ txn_hash: {
+ type: 'char(66)',
+ },
+ block_hash: {
+ type: 'char(66)',
+ },
+ block_number: {
+ type: 'bigint',
+ },
+ gas_used: {
+ type: 'varchar',
+ },
+ gas_price: {
+ type: 'varchar',
+ },
+ method_id: {
+ type: 'char(10)',
+ },
+ salt: {
+ type: 'varchar',
+ },
+ },
+};
+const logToTransactionSchemaMapping: any = {
+ hash: 'txn_hash',
+ gas: 'gas_used',
+ gasPrice: 'gas_price',
+ blockHash: 'block_hash',
+ blockNumber: 'block_number',
+ method_id: 'method_id',
+ salt: 'salt',
+};
+export { transaction, logToTransactionSchemaMapping };
diff --git a/packages/pipeline/src/postgres.ts b/packages/pipeline/src/postgres.ts
new file mode 100644
index 000000000..d095e5c9e
--- /dev/null
+++ b/packages/pipeline/src/postgres.ts
@@ -0,0 +1,12 @@
+import * as dotenv from 'dotenv';
+import { Pool, PoolConfig } from 'pg';
+dotenv.config();
+const client: PoolConfig = {
+ user: process.env.AURORA_USER,
+ database: process.env.AURORA_DB,
+ password: process.env.AURORA_PASSWORD,
+ port: parseInt(process.env.AURORA_PORT || '5432', 10),
+ host: process.env.AURORA_HOST,
+};
+const postgresClient = new Pool(client);
+export { postgresClient };
diff --git a/packages/pipeline/src/run_jobs.ts b/packages/pipeline/src/run_jobs.ts
new file mode 100644
index 000000000..4d82d4e2d
--- /dev/null
+++ b/packages/pipeline/src/run_jobs.ts
@@ -0,0 +1,87 @@
+import { exec } from 'child_process';
+
+import { postgresClient } from './postgres.js';
+import { dataFetchingQueries } from './scripts/query_data.js';
+import { web3, zrx } from './zrx.js';
+const CUR_BLOCK_OFFSET = 20;
+postgresClient.query(dataFetchingQueries.get_max_block, []).then((data: any) => {
+ const maxBlockNumber = data.rows[0].max;
+ const safeCurBlockNumber = web3.eth.blockNumber - CUR_BLOCK_OFFSET;
+ console.log('Scraping ' + maxBlockNumber + ' to ' + safeCurBlockNumber);
+ exec(
+ 'node ./lib/scripts/scrape_data --type events --from ' + maxBlockNumber + ' --to ' + safeCurBlockNumber,
+ (error, stdout, stderr) => {
+ if (error) {
+ console.log(error);
+ return;
+ }
+ console.log('Scraped events');
+ console.log('Scraping blocks');
+ exec(
+ 'node ./lib/scripts/scrape_data --type blocks --from ' + maxBlockNumber + ' --to ' + safeCurBlockNumber,
+ (error, stdout, stderr) => {
+ if (error) {
+ console.log(error);
+ return;
+ }
+ console.log('Scraped blocks');
+ console.log('Scraping transactions');
+ exec(
+ 'node ./lib/scripts/scrape_data --type transactions --from ' +
+ maxBlockNumber +
+ ' --to ' +
+ safeCurBlockNumber,
+ (error, stdout, stderr) => {
+ if (error) {
+ console.log(error);
+ return;
+ }
+ console.log('Scraped transactions');
+ console.log('Joining events_staging');
+ exec(
+ 'node ./lib/scripts/join_tables --name events_staging --from ' +
+ maxBlockNumber +
+ ' --to ' +
+ safeCurBlockNumber,
+ (error, stdout, stderr) => {
+ if (error) {
+ console.log(error);
+ return;
+ }
+ console.log('Joined events_staging');
+ console.log('Joining events');
+ exec(
+ 'node ./lib/scripts/join_tables --name events --from ' +
+ maxBlockNumber +
+ ' --to ' +
+ safeCurBlockNumber,
+ (error, stdout, stderr) => {
+ if (error) {
+ console.log(error);
+ return;
+ }
+ console.log('Joined events');
+ console.log('Joining events_full');
+ exec(
+ 'node ./lib/scripts/join_tables --name events_full --from ' +
+ maxBlockNumber +
+ ' --to ' +
+ safeCurBlockNumber,
+ (error, stdout, stderr) => {
+ if (error) {
+ console.log(error);
+ return;
+ }
+ },
+ );
+ },
+ );
+ },
+ );
+ },
+ );
+ },
+ );
+ },
+ );
+});
diff --git a/packages/pipeline/src/scrape_prices.ts b/packages/pipeline/src/scrape_prices.ts
new file mode 100644
index 000000000..c26062fac
--- /dev/null
+++ b/packages/pipeline/src/scrape_prices.ts
@@ -0,0 +1,10 @@
+import { postgresClient } from './postgres.js';
+import { dataFetchingQueries } from './scripts/query_data.js';
+import { scrapeDataScripts } from './scripts/scrape_data.js';
+import { web3, zrx } from './zrx.js';
+const CUR_BLOCK_OFFSET = 20;
+postgresClient.query(dataFetchingQueries.get_most_recent_pricing_date, []).then((data: any) => {
+ const curMaxScrapedDate = new Date(data.rows[0].max);
+ const curDate = new Date();
+ scrapeDataScripts.scrapeAllPricesToDB(curMaxScrapedDate.getTime(), curDate.getTime());
+});
diff --git a/packages/pipeline/src/scripts/create_tables.ts b/packages/pipeline/src/scripts/create_tables.ts
new file mode 100644
index 000000000..fd0d2b78b
--- /dev/null
+++ b/packages/pipeline/src/scripts/create_tables.ts
@@ -0,0 +1,258 @@
+import * as commandLineArgs from 'command-line-args';
+
+import { postgresClient } from '../postgres';
+import { formatters } from '../utils';
+const tableQueries: any = {
+ events_full: `CREATE TABLE IF NOT EXISTS events_full (
+ timestamp TIMESTAMP WITH TIME ZONE,
+ event_type VARCHAR,
+ error_id VARCHAR,
+ order_hash CHAR(66),
+ maker CHAR(42),
+ maker_amount NUMERIC(78),
+ maker_fee NUMERIC(78),
+ maker_token CHAR(42),
+ taker CHAR(42),
+ taker_amount NUMERIC(78),
+ taker_fee NUMERIC(78),
+ taker_token CHAR(42),
+ txn_hash CHAR(66),
+ gas_used NUMERIC(78),
+ gas_price NUMERIC(78),
+ fee_recipient CHAR(42),
+ method_id CHAR(10),
+ salt VARCHAR,
+ block_number BIGINT,
+ log_index BIGINT,
+ taker_symbol VARCHAR,
+ taker_name VARCHAR,
+ taker_decimals BIGINT,
+ taker_usd_price NUMERIC(78),
+ taker_txn_usd_value NUMERIC(78),
+ maker_symbol VARCHAR,
+ maker_name VARCHAR,
+ maker_decimals BIGINT,
+ maker_usd_price NUMERIC(78),
+ maker_txn_usd_value NUMERIC(78),
+ PRIMARY KEY (txn_hash, order_hash, log_index)
+ )`,
+ events: `CREATE TABLE IF NOT EXISTS events (
+ timestamp TIMESTAMP WITH TIME ZONE,
+ event_type VARCHAR,
+ error_id VARCHAR,
+ order_hash CHAR(66),
+ maker CHAR(42),
+ maker_amount NUMERIC(78),
+ maker_fee NUMERIC(78),
+ maker_token CHAR(42),
+ taker CHAR(42),
+ taker_amount NUMERIC(78),
+ taker_fee NUMERIC(78),
+ taker_token CHAR(42),
+ txn_hash CHAR(66),
+ gas_used NUMERIC(78),
+ gas_price NUMERIC(78),
+ fee_recipient CHAR(42),
+ method_id CHAR(10),
+ salt VARCHAR,
+ block_number BIGINT,
+ log_index BIGINT,
+ PRIMARY KEY (txn_hash, order_hash, log_index)
+ )`,
+ events_staging: `CREATE TABLE IF NOT EXISTS events_staging (
+ timestamp TIMESTAMP WITH TIME ZONE,
+ event_type VARCHAR,
+ error_id VARCHAR,
+ order_hash CHAR(66),
+ maker CHAR(42),
+ maker_amount NUMERIC(78),
+ maker_fee NUMERIC(78),
+ maker_token CHAR(42),
+ taker CHAR(42),
+ taker_amount NUMERIC(78),
+ taker_fee NUMERIC(78),
+ taker_token CHAR(42),
+ txn_hash CHAR(66),
+ fee_recipient CHAR(42),
+ block_number BIGINT,
+ log_index BIGINT,
+ PRIMARY KEY (txn_hash, order_hash, log_index)
+ )`,
+ events_raw: `CREATE TABLE IF NOT EXISTS events_raw (
+ event_type VARCHAR,
+ error_id VARCHAR,
+ order_hash CHAR(66),
+ maker CHAR(42),
+ maker_amount NUMERIC(78),
+ maker_fee NUMERIC(78),
+ maker_token CHAR(42),
+ taker CHAR(42),
+ taker_amount NUMERIC(78),
+ taker_fee NUMERIC(78),
+ taker_token CHAR(42),
+ txn_hash CHAR(66),
+ fee_recipient CHAR(42),
+ block_number BIGINT,
+ log_index BIGINT,
+ PRIMARY KEY (txn_hash, order_hash, log_index)
+ )`,
+ blocks: `CREATE TABLE IF NOT EXISTS blocks (
+ timestamp TIMESTAMP WITH TIME ZONE,
+ block_hash CHAR(66) UNIQUE,
+ block_number BIGINT,
+ PRIMARY KEY (block_hash)
+ )`,
+ transactions: `CREATE TABLE IF NOT EXISTS transactions (
+ txn_hash CHAR(66) UNIQUE,
+ block_hash CHAR(66),
+ block_number BIGINT,
+ gas_used NUMERIC(78),
+ gas_price NUMERIC(78),
+ method_id CHAR(10),
+ salt VARCHAR,
+ PRIMARY KEY (txn_hash)
+ )`,
+ tokens: `CREATE TABLE IF NOT EXISTS tokens (
+ address CHAR(42) UNIQUE,
+ name VARCHAR,
+ symbol VARCHAR,
+ decimals INT,
+ PRIMARY KEY (address)
+ )`,
+ prices: `CREATE TABLE IF NOT EXISTS prices (
+ address CHAR(42) UNIQUE,
+ timestamp TIMESTAMP WITH TIME ZONE,
+ price NUMERIC(78, 18),
+ PRIMARY KEY (address, timestamp)
+ )`,
+ relayers: `CREATE TABLE IF NOT EXISTS relayers (
+ name VARCHAR UNIQUE,
+ url VARCHAR DEFAULT '',
+ sra_http_endpoint VARCHAR DEFAULT '',
+ sra_ws_endpoint VARCHAR DEFAULT '',
+ fee_recipient_addresses CHAR(42)[] DEFAULT '{}',
+ taker_addresses CHAR(42)[] DEFAULT '{}',
+ PRIMARY KEY(name)`,
+ historical_prices: `CREATE TABLE IF NOT EXISTS historical_prices (
+ token VARCHAR,
+ base VARCHAR,
+ timestamp TIMESTAMP WITH TIME ZONE,
+ close NUMERIC(78, 18),
+ high NUMERIC(78, 18),
+ low NUMERIC(78, 18),
+ open NUMERIC(78, 18),
+ volume_from NUMERIC(78, 18),
+ volume_to NUMERIC(78, 18),
+ PRIMARY KEY (token, base, timestamp)
+ )`,
+ orders: `CREATE TABLE IF NOT EXISTS orders (
+ relayer_id VARCHAR,
+ exchange_contract_address CHAR(42),
+ maker CHAR(42),
+ maker_amount NUMERIC(78),
+ maker_fee NUMERIC(78),
+ maker_token CHAR(42),
+ taker CHAR(42),
+ taker_amount NUMERIC(78),
+ taker_fee NUMERIC(78),
+ taker_token CHAR(42),
+ fee_recipient CHAR(42),
+ expiration_unix_timestamp_sec NUMERIC(78),
+ salt VARCHAR,
+ order_hash CHAR(66),
+ PRIMARY KEY (relayer_id, order_hash)
+ )`,
+};
+function _safeQuery(query: string): any {
+ return new Promise((resolve, reject) => {
+ postgresClient
+ .query(query)
+ .then((data: any) => {
+ resolve(data);
+ })
+ .catch((err: any) => {
+ reject(err);
+ });
+ });
+}
+export const tableScripts = {
+ createTable(query: string): any {
+ return _safeQuery(query);
+ },
+ createAllTables(): any {
+ for (const tableName of tableQueries) {
+ _safeQuery(tableQueries[tableName]);
+ }
+ },
+};
+export const insertDataScripts = {
+ insertSingleRow(table: string, object: any): any {
+ return new Promise((resolve, reject) => {
+ const columns = Object.keys(object);
+ const safeArray: any = [];
+ for (const key of columns) {
+ if (key in object) {
+ if (key === 'timestamp') {
+ safeArray.push('to_timestamp(' + object[key] + ')');
+ } else if (typeof object[key] === 'string' || object[key] instanceof String) {
+ safeArray.push(formatters.escapeSQLParam(object[key]));
+ } else {
+ safeArray.push(object[key]);
+ }
+ } else {
+ safeArray.push('default');
+ }
+ }
+ const queryString = `INSERT INTO ${table} (${columns}) VALUES (${safeArray}) ON CONFLICT DO NOTHING`;
+ console.log(queryString);
+ postgresClient
+ .query(queryString)
+ .then((data: any) => {
+ resolve(data);
+ })
+ .catch((err: any) => {
+ reject(err);
+ });
+ });
+ },
+ insertMultipleRows(table: string, rows: any[], columns: any[]): any {
+ return new Promise((resolve, reject) => {
+ if (rows.length > 0) {
+ const rowsSplit = rows.map((value, index) => {
+ const safeArray: any = [];
+ for (const key of columns) {
+ if (key in value) {
+ if (key === 'timestamp') {
+ safeArray.push('to_timestamp(' + value[key] + ')');
+ } else if (typeof value[key] === 'string' || value[key] instanceof String) {
+ safeArray.push(formatters.escapeSQLParam(value[key]));
+ } else if (value[key] instanceof Array) {
+ const escapedArray = value[key].map((subValue: string, subIndex: number) => {
+ return formatters.escapeSQLParam(subValue);
+ });
+ safeArray.push('ARRAY[' + escapedArray.toString() + ']');
+ } else {
+ safeArray.push(value[key]);
+ }
+ } else {
+ safeArray.push('default');
+ }
+ }
+ return '(' + safeArray + ')';
+ });
+ const queryString = `INSERT INTO ${table} (${columns}) VALUES ${rowsSplit} ON CONFLICT DO NOTHING`;
+ postgresClient
+ .query(queryString)
+ .then((data: any) => {
+ resolve(data);
+ })
+ .catch((err: any) => {
+ // console.log(err);
+ reject(err);
+ });
+ } else {
+ resolve({});
+ }
+ });
+ },
+};
diff --git a/packages/pipeline/src/scripts/join_tables.ts b/packages/pipeline/src/scripts/join_tables.ts
new file mode 100644
index 000000000..e7c05b39a
--- /dev/null
+++ b/packages/pipeline/src/scripts/join_tables.ts
@@ -0,0 +1,234 @@
+import * as commandLineArgs from 'command-line-args';
+
+import { postgresClient } from '../postgres';
+import { formatters } from '../utils';
+const optionDefinitions = [
+ { name: 'name', alias: 'n', type: String },
+ { name: 'from', alias: 'f', type: Number },
+ { name: 'to', alias: 't', type: Number },
+];
+const cli = commandLineArgs(optionDefinitions);
+const dataInsertionQueries: any = {
+ events_staging: `INSERT INTO events_staging (
+ timestamp,
+ event_type,
+ error_id,
+ order_hash,
+ maker,
+ maker_amount,
+ maker_fee,
+ maker_token,
+ taker,
+ taker_amount,
+ taker_fee,
+ taker_token,
+ txn_hash,
+ fee_recipient,
+ block_number,
+ log_index
+ )
+ (SELECT
+ b.timestamp,
+ a.event_type,
+ a.error_id,
+ a.order_hash,
+ a.maker,
+ a.maker_amount,
+ a.maker_fee,
+ a.maker_token,
+ a.taker,
+ a.taker_amount,
+ a.taker_fee,
+ a.taker_token,
+ a.txn_hash,
+ a.fee_recipient,
+ a.block_number,
+ a.log_index
+ FROM
+ events_raw a
+ JOIN
+ blocks b
+ ON
+ a.block_number = b.block_number
+ AND
+ b.block_number >= $1
+ AND
+ b.block_number <= $2
+ ) ON CONFLICT (order_hash, txn_hash, log_index) DO NOTHING`,
+ events: `INSERT INTO events (
+ timestamp,
+ event_type,
+ error_id,
+ order_hash,
+ maker,
+ maker_amount,
+ maker_fee,
+ maker_token,
+ taker,
+ taker_amount,
+ taker_fee,
+ taker_token,
+ txn_hash,
+ fee_recipient,
+ block_number,
+ log_index,
+ gas_used,
+ gas_price,
+ method_id,
+ salt
+ )
+ (SELECT
+ a.timestamp,
+ a.event_type,
+ a.error_id,
+ a.order_hash,
+ a.maker,
+ a.maker_amount,
+ a.maker_fee,
+ a.maker_token,
+ a.taker,
+ a.taker_amount,
+ a.taker_fee,
+ a.taker_token,
+ a.txn_hash,
+ a.fee_recipient,
+ a.block_number,
+ a.log_index,
+ b.gas_used,
+ b.gas_price,
+ b.method_id,
+ b.salt
+ FROM
+ events_staging a
+ JOIN
+ transactions b
+ ON
+ a.txn_hash = b.txn_hash
+ AND
+ a.block_number >= $1
+ AND
+ a.block_number <= $2
+ ) ON CONFLICT (order_hash, txn_hash, log_index) DO NOTHING`,
+ events_full: `
+ INSERT INTO events_full (
+ timestamp,
+ event_type,
+ error_id,
+ order_hash,
+ maker,
+ maker_amount,
+ maker_fee,
+ maker_token,
+ taker,
+ taker_amount,
+ taker_fee,
+ taker_token,
+ txn_hash,
+ fee_recipient,
+ block_number,
+ log_index,
+ gas_used,
+ gas_price,
+ method_id,
+ salt,
+ taker_symbol,
+ taker_name,
+ taker_decimals,
+ taker_usd_price,
+ taker_txn_usd_value,
+ maker_symbol,
+ maker_name,
+ maker_decimals,
+ maker_usd_price,
+ maker_txn_usd_value
+ )
+ (SELECT
+ events.timestamp,
+ events.event_type,
+ events.error_id,
+ events.order_hash,
+ events.maker,
+ events.maker_amount,
+ events.maker_fee,
+ events.maker_token,
+ events.taker,
+ events.taker_amount,
+ events.taker_fee,
+ events.taker_token,
+ events.txn_hash,
+ events.fee_recipient,
+ events.block_number,
+ events.log_index,
+ events.gas_used,
+ events.gas_price,
+ events.method_id,
+ events.salt,
+ taker_token_prices.symbol,
+ taker_token_prices.name,
+ taker_token_prices.decimals,
+ taker_token_prices.price,
+ (events.taker_amount / (10 ^ taker_token_prices.decimals)) * taker_token_prices.price,
+ maker_token_prices.symbol,
+ maker_token_prices.name,
+ maker_token_prices.decimals,
+ maker_token_prices.price,
+ (events.maker_amount / (10 ^ maker_token_prices.decimals)) * maker_token_prices.price
+ FROM
+ events
+ LEFT JOIN
+ (SELECT
+ tokens.address,
+ tokens.name,
+ tokens.symbol,
+ tokens.decimals,
+ prices.timestamp,
+ prices.price
+ FROM
+ tokens
+ LEFT JOIN
+ prices
+ ON
+ tokens.symbol = prices.symbol) taker_token_prices
+ ON
+ (events.taker_token = taker_token_prices.address
+ AND
+ (DATE(events.timestamp) = DATE(taker_token_prices.timestamp) OR taker_token_prices.timestamp IS NULL))
+ LEFT JOIN
+ (SELECT
+ tokens.address,
+ tokens.name,
+ tokens.symbol,
+ tokens.decimals,
+ prices.timestamp,
+ prices.price
+ FROM
+ tokens
+ LEFT JOIN
+ prices
+ ON
+ tokens.symbol = prices.symbol) maker_token_prices
+ ON
+ (events.maker_token = maker_token_prices.address
+ AND
+ (DATE(events.timestamp) = DATE(maker_token_prices.timestamp) OR maker_token_prices.timestamp IS NULL))
+ WHERE
+ events.block_number >= $1
+ AND
+ events.block_number <= $2
+ ) ON CONFLICT (order_hash, txn_hash, log_index) DO NOTHING`,
+};
+if (cli.name) {
+ const query = dataInsertionQueries[cli.name];
+ if (query && cli.from) {
+ const fromBlock = cli.from;
+ const toBlock = cli.to ? cli.to : cli.from + 1;
+ postgresClient
+ .query(query, [fromBlock, toBlock])
+ .then((data: any) => {
+ console.log(data);
+ })
+ .catch((err: any) => {
+ console.error(err);
+ });
+ }
+}
diff --git a/packages/pipeline/src/scripts/query_data.ts b/packages/pipeline/src/scripts/query_data.ts
new file mode 100644
index 000000000..97e3749ea
--- /dev/null
+++ b/packages/pipeline/src/scripts/query_data.ts
@@ -0,0 +1,87 @@
+import { formatters } from '../utils';
+export const dataFetchingQueries: any = {
+ get_missing_txn_hashes: `
+ SELECT
+ a.txn_hash
+ FROM
+ events_raw a
+ WHERE NOT EXISTS
+ (
+ SELECT
+ *
+ FROM
+ transactions b
+ WHERE
+ b.txn_hash = a.txn_hash
+ )
+ AND
+ a.block_number >= $1
+ AND
+ a.block_number < $2`,
+ get_used_block_numbers: `
+ SELECT DISTINCT
+ a.block_number
+ FROM
+ events_raw a
+ WHERE NOT EXISTS
+ (
+ SELECT
+ *
+ FROM
+ blocks b
+ WHERE
+ b.block_number = a.block_number
+ )
+ AND
+ a.block_number >= $1
+ AND
+ a.block_number < $2`,
+ get_token_registry: `
+ SELECT
+ *
+ FROM
+ tokens`,
+ get_max_block: `
+ SELECT
+ MAX(block_number)
+ FROM
+ events_raw`,
+ get_relayers: `
+ SELECT
+ *
+ FROM
+ relayers`,
+ get_most_recent_pricing_date: `
+ SELECT
+ MAX(DATE(timestamp))
+ FROM
+ prices
+ `,
+ get_top_unknown_token_addresses: `
+ SELECT a.token_address as address, a.txn_value / 2 as total_txn_value
+FROM
+(SELECT token_address, SUM(txn_value) as txn_value
+FROM
+(select a.timestamp, a.maker_token as token_address, (CASE WHEN a.taker_txn_usd_value > a.maker_txn_usd_value OR a.maker_txn_usd_value IS NULL
+ THEN a.taker_txn_usd_value
+ ELSE a.maker_txn_usd_value END) as txn_value
+ from events_full a
+ where a.event_type = 'LogFill'
+ and a.timestamp > (NOW() + INTERVAL '-24 hours')
+ union
+ select a.timestamp, a.taker_token as token_address, (CASE WHEN a.taker_txn_usd_value > a.maker_txn_usd_value OR a.maker_txn_usd_value IS NULL
+ THEN a.taker_txn_usd_value
+ ELSE a.maker_txn_usd_value END) as txn_value
+ from events_full a
+ where a.event_type = 'LogFill'
+ and a.timestamp > (NOW() + INTERVAL '-24 hours')) token_txn_values
+WHERE token_address IS NOT NULL
+AND txn_value > 0
+GROUP BY 1
+ORDER BY 2 DESC) a
+LEFT JOIN tokens b
+ON a.token_address = b.address
+WHERE symbol is NULL
+ORDER BY 2 DESC
+`,
+};
diff --git a/packages/pipeline/src/scripts/scrape_data.ts b/packages/pipeline/src/scripts/scrape_data.ts
new file mode 100644
index 000000000..963670b47
--- /dev/null
+++ b/packages/pipeline/src/scripts/scrape_data.ts
@@ -0,0 +1,649 @@
+import { ExchangeEvents, ZeroEx } from '0x.js';
+import { HttpClient, Order, OrderbookRequest, OrderbookResponse, TokenPairsItem } from '@0xproject/connect';
+import * as Airtable from 'airtable';
+import * as commandLineArgs from 'command-line-args';
+import * as _ from 'lodash';
+import * as querystring from 'querystring';
+import * as queue from 'queue';
+import * as request from 'request';
+import * as rpn from 'request-promise-native';
+
+import { HttpRequestOptions } from '../../../connect/lib/src/types.js';
+import { relayer } from '../models/relayer.js';
+import { token } from '../models/tokens.js';
+import { postgresClient } from '../postgres.js';
+import { typeConverters } from '../utils.js';
+import { web3, zrx } from '../zrx.js';
+
+import { insertDataScripts } from './create_tables.js';
+import { dataFetchingQueries } from './query_data.js';
+const optionDefinitions = [
+ { name: 'from', alias: 'f', type: Number },
+ { name: 'to', alias: 't', type: Number },
+ { name: 'type', type: String },
+ { name: 'id', type: String },
+ { name: 'force', type: Boolean },
+ { name: 'token', type: String },
+];
+const cli = commandLineArgs(optionDefinitions);
+const q = queue({ concurrency: 6, autostart: true });
+const airtableBase = new Airtable({ apiKey: process.env.AIRTABLE_API_KEY }).base(process.env.AIRTABLE_0X_BASE);
+const BLOCK_INCREMENTS = 1000;
+const BASE_SYMBOL = 'USD'; // use USD as base currency against
+const API_HIST_LIMIT = 2000; // cryptocompare API limits histoday price query to 2000 days
+const SECONDS_PER_DAY = 86400;
+const PRICE_API_ENDPOINT = 'https://min-api.cryptocompare.com/data/pricehistorical';
+const RELAYER_REGISTRY_JSON = 'https://raw.githubusercontent.com/0xProject/0x-relayer-registry/master/relayers.json';
+const METAMASK_ETH_CONTRACT_METADATA_JSON =
+ 'https://raw.githubusercontent.com/MetaMask/eth-contract-metadata/master/contract-map.json';
+const ETHPLORER_BASE_URL = 'http://api.ethplorer.io';
+const ETHPLORER_TOP_TOKENS_JSON = `${ETHPLORER_BASE_URL}/getTopTokens?apiKey=dyijm5418TjOJe34`;
+// const HIST_PRICE_API_ENDPOINT = 'https://min-api.cryptocompare.com/data/histoday';
+const AIRTABLE_RELAYER_INFO = 'Relayer Info';
+export const pullDataScripts = {
+ getAllEvents(fromBlockNumber: number, toBlockNumber: number): any {
+ return new Promise((resolve, reject) => {
+ const getLogsPromises: any[] = [];
+ getLogsPromises.push(
+ zrx.exchange.getLogsAsync(
+ ExchangeEvents.LogFill,
+ { fromBlock: fromBlockNumber, toBlock: toBlockNumber },
+ {},
+ ),
+ zrx.exchange.getLogsAsync(
+ ExchangeEvents.LogCancel,
+ { fromBlock: fromBlockNumber, toBlock: toBlockNumber },
+ {},
+ ),
+ zrx.exchange.getLogsAsync(
+ ExchangeEvents.LogError,
+ { fromBlock: fromBlockNumber, toBlock: toBlockNumber },
+ {},
+ ),
+ );
+ Promise.all(getLogsPromises)
+ .then((data: any[]) => {
+ resolve(data);
+ })
+ .catch((err: any) => {
+ reject(err);
+ });
+ });
+ },
+ getBlockInfo(blockNumber: number): any {
+ return new Promise((resolve, reject) => {
+ web3.eth.getBlock(blockNumber, (err, result) => {
+ if (err) {
+ reject(err);
+ } else {
+ resolve(result);
+ }
+ });
+ });
+ },
+ getTransactionInfo(transactionHash: string): any {
+ return new Promise((resolve, reject) => {
+ web3.eth.getTransaction(transactionHash, (err, result) => {
+ if (err) {
+ reject(err);
+ } else {
+ resolve(result);
+ }
+ });
+ });
+ },
+ getTokenRegistry(): any {
+ return new Promise((resolve, reject) => {
+ zrx.tokenRegistry
+ .getTokensAsync()
+ .then((data: any) => {
+ resolve(data);
+ })
+ .catch((err: any) => {
+ reject(err);
+ });
+ });
+ },
+ getMetaMaskTokens(): any {
+ return new Promise((resolve, reject) => {
+ request(METAMASK_ETH_CONTRACT_METADATA_JSON, (error, response, body) => {
+ if (error) {
+ reject(error);
+ } else {
+ resolve(JSON.parse(body));
+ }
+ });
+ });
+ },
+ getEthplorerTopTokens(): any {
+ return new Promise((resolve, reject) => {
+ request(ETHPLORER_TOP_TOKENS_JSON, (error, response, body) => {
+ if (error) {
+ reject(error);
+ } else {
+ resolve(JSON.parse(body));
+ }
+ });
+ });
+ },
+ getEthplorerToken(tokenAddress: string): any {
+ return new Promise((resolve, reject) => {
+ const url = `${ETHPLORER_BASE_URL}/getTokenInfo/${tokenAddress}?apiKey=dyijm5418TjOJe34`;
+ request(url, (error, response, body) => {
+ if (error) {
+ reject(error);
+ } else {
+ try {
+ const json = JSON.parse(body);
+ resolve(json);
+ } catch (err) {
+ resolve({ error: 'error' });
+ }
+ }
+ });
+ });
+ },
+ getPriceData(symbol: string, timestamp: number, timeDelay?: number): any {
+ return new Promise((resolve, reject) => {
+ if (symbol === 'WETH') {
+ symbol = 'ETH';
+ }
+ let parsedParams = querystring.stringify({
+ fsym: symbol,
+ tsyms: 'USD',
+ ts: timestamp / 1000,
+ });
+ console.debug(parsedParams);
+ setTimeout(() => {
+ request(PRICE_API_ENDPOINT + '?' + parsedParams, (error, response, body) => {
+ if (error) {
+ reject(error);
+ } else {
+ resolve(JSON.parse(body));
+ }
+ });
+ }, timeDelay);
+ });
+ },
+ getRelayers(): any {
+ return new Promise((resolve, reject) => {
+ request(RELAYER_REGISTRY_JSON, (error, response, body) => {
+ if (error) {
+ reject(error);
+ } else {
+ resolve(JSON.parse(body));
+ }
+ });
+ });
+ },
+ async getOrderBook(sraEndpoint: string): Promise<Object> {
+ const relayerClient = new HttpClient(sraEndpoint);
+ const tokenResponse: TokenPairsItem[] = await relayerClient.getTokenPairsAsync();
+ const fullOrderBook: OrderbookResponse[] = [];
+ for (const tokenPair of tokenResponse) {
+ const orderBookRequest: OrderbookRequest = {
+ baseTokenAddress: tokenPair.tokenA.address,
+ quoteTokenAddress: tokenPair.tokenB.address,
+ };
+ const orderBook: OrderbookResponse = await relayerClient.getOrderbookAsync(orderBookRequest);
+ fullOrderBook.push(orderBook);
+ }
+ return fullOrderBook;
+ },
+ // async getHistoricalPrices(
+ // fromSymbol: string,
+ // toSymbol: string,
+ // fromTimestamp: number,
+ // toTimestamp: number,
+ // ): Promise<HistoricalPriceResponse> {
+ // const daysInQueryPeriod = Math.round((toTimestamp - fromTimestamp) / (SECONDS_PER_DAY));
+ // if(fromSymbol === 'WETH') {
+ // fromSymbol = 'ETH';
+ // }
+ // var parsedParams = {
+ // fsym: fromSymbol,
+ // tsym: toSymbol,
+ // limit: Math.min(daysInQueryPeriod, API_HIST_LIMIT),
+ // toTs: toTimestamp,
+ // };
+ // var options = {
+ // uri: HIST_PRICE_API_ENDPOINT,
+ // qs: parsedParams,
+ // json: false,
+ // };
+ // try {
+ // const response = await rpn(options);
+ // return Promise.resolve(JSON.parse(response));
+ // } catch (error) {
+ // console.debug(error);
+ // return Promise.reject(error);
+ // }
+ // },
+};
+export const scrapeDataScripts = {
+ scrapeAllPricesToDB(fromTime: number, toTime: number) {
+ const fromDate = new Date(fromTime);
+ fromDate.setUTCHours(0);
+ fromDate.setUTCMinutes(0);
+ fromDate.setUTCSeconds(0);
+ fromDate.setUTCMilliseconds(0);
+ const toDate = new Date(toTime);
+ postgresClient
+ .query(dataFetchingQueries.get_token_registry, [])
+ .then((result: any) => {
+ for (const curDate = fromDate; curDate < toDate; curDate.setDate(curDate.getDate() + 1)) {
+ for (const token of Object.values(result.rows)) {
+ console.debug('Scraping ' + curDate + ' ' + token);
+ q.push(_scrapePriceToDB(curDate.getTime(), token, 500));
+ }
+ }
+ })
+ .catch((err: any) => {
+ console.debug(err);
+ });
+ },
+};
+function _scrapeEventsToDB(fromBlock: number, toBlock: number): any {
+ return (cb: () => void) => {
+ pullDataScripts
+ .getAllEvents(fromBlock, toBlock)
+ .then((data: any) => {
+ const parsedEvents: any = {};
+ parsedEvents[ExchangeEvents.LogFill] = [];
+ parsedEvents[ExchangeEvents.LogCancel] = [];
+ parsedEvents[ExchangeEvents.LogError] = [];
+ for (const index in data) {
+ for (const datum of data[index]) {
+ const event = typeConverters.convertLogEventToEventObject(datum);
+ parsedEvents[event.event_type].push(event);
+ }
+ }
+ console.log(fromBlock + ' : ' + toBlock + ' ' + parsedEvents[ExchangeEvents.LogFill].length);
+ for (const event_type in parsedEvents) {
+ if (parsedEvents[event_type].length > 0) {
+ insertDataScripts
+ .insertMultipleRows(
+ 'events_raw',
+ parsedEvents[event_type],
+ Object.keys(parsedEvents[event_type][0]),
+ )
+ .then(() => {})
+ .catch((error: any) => {});
+ }
+ }
+ cb();
+ })
+ .catch((err: any) => {
+ cb();
+ });
+ };
+}
+function _scrapeBlockToDB(block: number): any {
+ return (cb: () => void) => {
+ pullDataScripts
+ .getBlockInfo(block)
+ .then((data: any) => {
+ const parsedBlock = typeConverters.convertLogBlockToBlockObject(data);
+ insertDataScripts
+ .insertSingleRow('blocks', parsedBlock)
+ .then((result: any) => {
+ cb();
+ })
+ .catch((err: any) => {
+ cb();
+ });
+ })
+ .catch((err: any) => {
+ cb();
+ });
+ };
+}
+// function _scrapeAllRelayersToDB(): any {
+// return (cb: () => void) => {
+// airtableBase(AIRTABLE_RELAYER_INFO)
+// .select()
+// .eachPage((records: any, fetchNextPage: () => void) => {
+// const parsedRelayers: any[] = [];
+// for(const record of records) {
+// parsedRelayers.push(typeConverters.convertRelayerToRelayerObject(record));
+// }
+// insertDataScripts.insertMultipleRows('relayers', parsedRelayers, Object.keys(parsedRelayers[0]))
+// .then((result: any) => {
+// cb();
+// })
+// .catch((err: any) => {
+// cb();
+// });
+// })
+// .catch((err: any) => {
+// cb();
+// });
+// };
+// }
+function _scrapeAllRelayersToDB(): any {
+ return (cb: () => void) => {
+ pullDataScripts
+ .getRelayers()
+ .then((relayers: any[]) => {
+ console.log(relayers);
+ const parsedRelayers: any[] = [];
+ for (const relayer of relayers) {
+ parsedRelayers.push(typeConverters.convertRelayerToRelayerObject(relayer));
+ }
+ console.log(parsedRelayers);
+ insertDataScripts
+ .insertMultipleRows('relayers', parsedRelayers, Object.keys(relayer.tableProperties))
+ .then((result: any) => {
+ console.log(result);
+ cb();
+ })
+ .catch((err: any) => {
+ console.log(err);
+ cb();
+ });
+ })
+ .catch((err: any) => {
+ cb();
+ });
+ };
+}
+function _scrapeTransactionToDB(transactionHash: string): any {
+ return (cb: () => void) => {
+ pullDataScripts
+ .getTransactionInfo(transactionHash)
+ .then((data: any) => {
+ const parsedTransaction = typeConverters.convertLogTransactionToTransactionObject(data);
+ insertDataScripts
+ .insertSingleRow('transactions', parsedTransaction)
+ .then((result: any) => {
+ cb();
+ })
+ .catch((err: any) => {
+ cb();
+ });
+ })
+ .catch((err: any) => {
+ cb();
+ });
+ };
+}
+function _scrapeTokenRegistryToDB(): any {
+ return (cb: () => void) => {
+ pullDataScripts
+ .getTokenRegistry()
+ .then((data: any) => {
+ const parsedTokens: any = [];
+ for (const token of data) {
+ parsedTokens.push(typeConverters.convertLogTokenToTokenObject(token));
+ }
+ insertDataScripts.insertMultipleRows('tokens', parsedTokens, Object.keys(parsedTokens[0]));
+ cb();
+ })
+ .catch((err: any) => {
+ cb();
+ });
+ };
+}
+function _scrapeMetaMaskEthContractMetadataToDB(): any {
+ return (cb: () => void) => {
+ pullDataScripts
+ .getMetaMaskTokens()
+ .then((data: any) => {
+ const parsedTokens: any = [];
+ const dataArray = _.map(_.keys(data), (tokenAddress: string) => {
+ const value = _.get(data, tokenAddress);
+ return {
+ address: tokenAddress,
+ ...value,
+ };
+ });
+ const erc20TokensOnly = _.filter(dataArray, entry => {
+ const isErc20 = _.get(entry, 'erc20');
+ return isErc20;
+ });
+ for (const token of erc20TokensOnly) {
+ parsedTokens.push(typeConverters.convertMetaMaskTokenToTokenObject(token));
+ }
+ insertDataScripts.insertMultipleRows('tokens', parsedTokens, Object.keys(parsedTokens[0]));
+ cb();
+ })
+ .catch((err: any) => {
+ cb();
+ });
+ };
+}
+function _scrapeEthplorerTopTokensToDB(): any {
+ return (cb: () => void) => {
+ pullDataScripts
+ .getEthplorerTopTokens()
+ .then((data: any) => {
+ const parsedTokens: any = [];
+ const tokens = _.get(data, 'tokens');
+ for (const token of tokens) {
+ parsedTokens.push(typeConverters.convertMetaMaskTokenToTokenObject(token));
+ }
+ insertDataScripts.insertMultipleRows('tokens', parsedTokens, Object.keys(parsedTokens[0]));
+ cb();
+ })
+ .catch((err: any) => {
+ cb();
+ });
+ };
+}
+function _scrapeUnknownTokenInformationToDB(): any {
+ return (cb: () => void) => {
+ postgresClient
+ .query(dataFetchingQueries.get_top_unknown_token_addresses)
+ .then(async (result: any) => {
+ const addresses = _.map(result.rows, row => _.get(row, 'address'));
+ const responses = await Promise.all(
+ _.map(addresses, address => pullDataScripts.getEthplorerToken(address)),
+ );
+ const tokens = _.filter(responses, response => _.isUndefined(_.get(response, 'error')));
+ const parsedTokens = _.map(tokens, tokenInfo =>
+ typeConverters.convertEthplorerTokenToTokenObject(tokenInfo),
+ );
+ insertDataScripts.insertMultipleRows('tokens', parsedTokens, Object.keys(parsedTokens[0]));
+ cb();
+ })
+ .catch((err: any) => {
+ cb();
+ });
+ };
+}
+function _scrapePriceToDB(timestamp: number, token: any, timeDelay?: number): any {
+ return (cb: () => void) => {
+ pullDataScripts
+ .getPriceData(token.symbol, timestamp, timeDelay)
+ .then((data: any) => {
+ const safeSymbol = token.symbol === 'WETH' ? 'ETH' : token.symbol;
+ const parsedPrice = {
+ timestamp: timestamp / 1000,
+ symbol: token.symbol,
+ base: 'USD',
+ price: _.has(data[safeSymbol], 'USD') ? data[safeSymbol].USD : 0,
+ };
+ console.debug('Inserting ' + timestamp);
+ console.debug(parsedPrice);
+ insertDataScripts.insertSingleRow('prices', parsedPrice);
+ cb();
+ })
+ .catch((err: any) => {
+ console.debug(err);
+ cb();
+ });
+ };
+}
+// function _scrapeHistoricalPricesToDB(token: any, fromTimestamp: number, toTimestamp: number): any {
+// return (cb: () => void) => {
+// pullDataScripts
+// .getHistoricalPrices(token, BASE_SYMBOL, fromTimestamp, toTimestamp)
+// .then((data: any) => {
+// const parsedHistoricalPrices: any = [];
+// for (const historicalPrice of data['Data']) {
+// const parsedHistoricalPrice = typeConverters.convertLogHistoricalPricesToHistoricalPricesObject(historicalPrice);
+// parsedHistoricalPrice['token'] = token;
+// parsedHistoricalPrice['base'] = BASE_SYMBOL;
+// parsedHistoricalPrices.push(parsedHistoricalPrice);
+// }
+// if (parsedHistoricalPrices.length > 0) {
+// insertDataScripts
+// .insertMultipleRows(
+// 'historical_prices',
+// parsedHistoricalPrices,
+// Object.keys(parsedHistoricalPrices[0]),
+// )
+// .catch((error: any) => {
+// console.error(error);
+// });
+// }
+// cb();
+// })
+// .catch((error: any) => {
+// console.error(error);
+// cb();
+// });
+// };
+// }
+function _scrapeOrderBookToDB(id: string, sraEndpoint: string): any {
+ return (cb: () => void) => {
+ pullDataScripts
+ .getOrderBook(sraEndpoint)
+ .then((data: any) => {
+ for (const book of data) {
+ for (const order of book.bids) {
+ console.debug(order);
+ const parsedOrder = typeConverters.convertLogOrderToOrderObject(order);
+ parsedOrder.relayer_id = id;
+ parsedOrder.order_hash = ZeroEx.getOrderHashHex(order);
+ insertDataScripts.insertSingleRow('orders', parsedOrder).catch((error: any) => {
+ console.error(error);
+ });
+ }
+ for (const order of book.asks) {
+ console.debug(order);
+ const parsedOrder = typeConverters.convertLogOrderToOrderObject(order);
+ parsedOrder.relayer_id = id;
+ parsedOrder.order_hash = ZeroEx.getOrderHashHex(order);
+ insertDataScripts.insertSingleRow('orders', parsedOrder).catch((error: any) => {
+ console.error(error);
+ });
+ }
+ }
+ cb();
+ })
+ .catch((error: any) => {
+ console.error(error);
+ cb();
+ });
+ };
+}
+if (cli.type === 'events') {
+ if (cli.from && cli.to) {
+ const destToBlock = cli.to ? cli.to : cli.from;
+ let curFromBlock = cli.from;
+ let curToBlock = curFromBlock;
+ do {
+ curToBlock += destToBlock - curToBlock < BLOCK_INCREMENTS ? destToBlock - curToBlock : BLOCK_INCREMENTS;
+ q.push(_scrapeEventsToDB(curFromBlock, curToBlock));
+ curFromBlock = curToBlock + 1;
+ } while (curToBlock < destToBlock);
+ }
+} else if (cli.type === 'blocks') {
+ if (cli.from && cli.to) {
+ if (cli.force) {
+ const destToBlock = cli.to ? cli.to : cli.from;
+ let curFromBlock = cli.from;
+ const curToBlock = curFromBlock;
+ for (; curFromBlock < destToBlock; curFromBlock++) {
+ q.push(_scrapeBlockToDB(curFromBlock));
+ }
+ } else {
+ const fetchFrom = cli.from;
+ const fetchTo = cli.to ? cli.to : cli.from + 1;
+ postgresClient
+ .query(dataFetchingQueries.get_used_block_numbers, [fetchFrom, fetchTo])
+ .then((data: any) => {
+ for (const row of data.rows) {
+ q.push(_scrapeBlockToDB(row.block_number));
+ }
+ })
+ .catch((err: any) => {
+ // console.debug(err);
+ });
+ }
+ }
+} else if (cli.type === 'transactions') {
+ if (cli.id) {
+ q.push(_scrapeTransactionToDB(cli.id));
+ } else if (cli.from) {
+ const fetchFrom = cli.from;
+ const fetchTo = cli.to ? cli.to : cli.from + 1;
+ postgresClient
+ .query(dataFetchingQueries.get_missing_txn_hashes, [fetchFrom, fetchTo])
+ .then((data: any) => {
+ for (const row of data.rows) {
+ q.push(_scrapeTransactionToDB(row.txn_hash));
+ }
+ })
+ .catch((err: any) => {
+ // console.debug(err);
+ });
+ }
+} else if (cli.type === 'tokens') {
+ q.push(_scrapeMetaMaskEthContractMetadataToDB());
+ q.push(_scrapeEthplorerTopTokensToDB());
+} else if (cli.type === 'unknown_tokens') {
+ q.push(_scrapeUnknownTokenInformationToDB());
+} else if (cli.type === 'prices' && cli.from && cli.to) {
+ const fromDate = new Date(cli.from);
+ console.debug(fromDate);
+ fromDate.setUTCHours(0);
+ fromDate.setUTCMinutes(0);
+ fromDate.setUTCSeconds(0);
+ fromDate.setUTCMilliseconds(0);
+ console.debug(fromDate);
+ const toDate = new Date(cli.to);
+ postgresClient
+ .query(dataFetchingQueries.get_token_registry, [])
+ .then((result: any) => {
+ for (const curDate = fromDate; curDate < toDate; curDate.setDate(curDate.getDate() + 1)) {
+ for (const token of Object.values(result.rows)) {
+ console.debug('Scraping ' + curDate + ' ' + token);
+ q.push(_scrapePriceToDB(curDate.getTime(), token));
+ }
+ }
+ })
+ .catch((err: any) => {
+ console.debug(err);
+ });
+ // } else if (cli.type === 'historical_prices') {
+ // if (cli.token && cli.from && cli.to) {
+ // q.push(_scrapeHistoricalPricesToDB(cli.token, cli.from, cli.to));
+ // }
+ // } else if (cli.type === 'all_historical_prices') {
+ // if (cli.from && cli.to) {
+ // postgresClient
+ // .query(dataFetchingQueries.get_token_registry, [])
+ // .then((result: any) => {
+ // const curTokens: any = result.rows.map((a: any): any => a.symbol);
+ // for (const curToken of curTokens) {
+ // console.debug('Historical data backfill: Pushing coin ' + curToken);
+ // q.push(_scrapeHistoricalPricesToDB(curToken, cli.from, cli.to));
+ // }
+ // })
+ // .catch((err: any) => {
+ // console.debug(err);
+ // });
+ // }
+} else if (cli.type === 'relayers') {
+ q.push(_scrapeAllRelayersToDB());
+} else if (cli.type === 'orders') {
+ postgresClient.query(dataFetchingQueries.get_relayers, []).then((result: any) => {
+ for (const relayer of result.rows) {
+ if (relayer.sra_http_url) {
+ q.push(_scrapeOrderBookToDB(relayer.id, relayer.sra_http_url));
+ }
+ }
+ });
+}
diff --git a/packages/pipeline/src/types.ts b/packages/pipeline/src/types.ts
new file mode 100644
index 000000000..1c76f12a7
--- /dev/null
+++ b/packages/pipeline/src/types.ts
@@ -0,0 +1,6 @@
+export interface HistoricalPriceResponse {
+ Response: string;
+ Type: number;
+ Aggregated: boolean;
+ Data: JSON[];
+}
diff --git a/packages/pipeline/src/utils.ts b/packages/pipeline/src/utils.ts
new file mode 100644
index 000000000..3474cd16b
--- /dev/null
+++ b/packages/pipeline/src/utils.ts
@@ -0,0 +1,181 @@
+import * as _ from 'lodash';
+
+import { block, logToBlockSchemaMapping } from './models/block';
+import { event, logToEventSchemaMapping } from './models/event';
+import { historicalPrices, logToHistoricalPricesSchema } from './models/historical_prices';
+import { logToOrderSchemaMapping, order } from './models/order';
+import { logToRelayerSchemaMapping } from './models/relayer';
+import { logToTokenSchemaMapping, token } from './models/tokens';
+import { logToTransactionSchemaMapping, transaction } from './models/transaction';
+export const typeConverters = {
+ convertLogEventToEventObject(log: any): any {
+ const newEvent: any = {};
+ for (const key in logToEventSchemaMapping) {
+ if (_.has(log, key)) {
+ newEvent[logToEventSchemaMapping[key]] = _.get(log, key);
+ if (newEvent[logToEventSchemaMapping[key]].constructor.name === 'BigNumber') {
+ newEvent[logToEventSchemaMapping[key]] = newEvent[logToEventSchemaMapping[key]].toString();
+ }
+ }
+ }
+ return newEvent;
+ },
+ convertLogBlockToBlockObject(logBlock: any): any {
+ const newBlock: any = {};
+ for (const key in logToBlockSchemaMapping) {
+ if (_.has(logBlock, key)) {
+ newBlock[logToBlockSchemaMapping[key]] = _.get(logBlock, key);
+ if (newBlock[logToBlockSchemaMapping[key]].constructor.name === 'BigNumber') {
+ newBlock[logToBlockSchemaMapping[key]] = newBlock[logToBlockSchemaMapping[key]].toString();
+ }
+ }
+ }
+ return newBlock;
+ },
+ convertLogTokenToTokenObject(logToken: any): any {
+ const newToken: any = {};
+ for (const key in logToTokenSchemaMapping) {
+ if (_.has(logToken, key)) {
+ newToken[logToTokenSchemaMapping[key]] = _.get(logToken, key);
+ if (newToken[logToTokenSchemaMapping[key]].constructor.name === 'BigNumber') {
+ newToken[logToTokenSchemaMapping[key]] = newToken[logToTokenSchemaMapping[key]].toString();
+ }
+ }
+ }
+ newToken[logToTokenSchemaMapping.address] = newToken[logToTokenSchemaMapping.address].toLowerCase();
+ return newToken;
+ },
+ convertMetaMaskTokenToTokenObject(metaMaskToken: any): any {
+ const newToken: any = {};
+ for (const key in logToTokenSchemaMapping) {
+ if (_.has(metaMaskToken, key)) {
+ newToken[logToTokenSchemaMapping[key]] = _.get(metaMaskToken, key);
+ }
+ }
+ newToken[logToTokenSchemaMapping.address] = newToken[logToTokenSchemaMapping.address].toLowerCase();
+ console.log(newToken);
+ return newToken;
+ },
+ convertEthplorerTokenToTokenObject(ethplorerToken: any): any {
+ const newToken: any = {};
+ for (const key in logToTokenSchemaMapping) {
+ if (_.has(ethplorerToken, key)) {
+ newToken[logToTokenSchemaMapping[key]] = _.get(ethplorerToken, key);
+ }
+ }
+ newToken[logToTokenSchemaMapping.address] = newToken[logToTokenSchemaMapping.address].toLowerCase();
+ return newToken;
+ },
+ convertLogTransactionToTransactionObject(logTransaction: any): any {
+ const newTransaction: any = {};
+ for (const key in logToTransactionSchemaMapping) {
+ if (_.has(logTransaction, key)) {
+ newTransaction[logToTransactionSchemaMapping[key]] = _.get(logTransaction, key);
+ if (newTransaction[logToTransactionSchemaMapping[key]].constructor.name === 'BigNumber') {
+ newTransaction[logToTransactionSchemaMapping[key]] = newTransaction[
+ logToTransactionSchemaMapping[key]
+ ].toString();
+ }
+ } else {
+ if (key === 'method_id') {
+ newTransaction[logToTransactionSchemaMapping[key]] = logTransaction.input.substring(0, 10);
+ } else if (key === 'salt') {
+ newTransaction[logToTransactionSchemaMapping[key]] =
+ '0x' + logTransaction.input.substring(714, 778); // Only God can judge me
+ }
+ }
+ }
+ return newTransaction;
+ },
+ // convertRelayerToRelayerObject(logRelayer: any): any {
+ // const newRelayer: any = {};
+ // for (const key in logToRelayerSchemaMapping) {
+ // if (_.has(logRelayer, key)) {
+ // newRelayer[logToRelayerSchemaMapping[key]] = _.get(logRelayer, key);
+ // if (newRelayer[logToRelayerSchemaMapping[key]].constructor.name === 'BigNumber') {
+ // newRelayer[logToRelayerSchemaMapping[key]] = newRelayer[logToRelayerSchemaMapping[key]].toString();
+ // }
+ // } else if((logToRelayerSchemaMapping[key] === 'known_fee_addresses' || logToRelayerSchemaMapping[key] === 'known_taker_addresses')) {
+ // newRelayer[logToRelayerSchemaMapping[key]] = '{}';
+ // } else {
+ // newRelayer[logToRelayerSchemaMapping[key]] = '';
+ // }
+ // }
+ // return newRelayer;
+ // },
+ convertRelayerToRelayerObject(logRelayer: any): any {
+ const newRelayer: any = {};
+ for (const key in logToRelayerSchemaMapping) {
+ if (_.has(logRelayer, key)) {
+ newRelayer[logToRelayerSchemaMapping[key]] = _.get(logRelayer, key);
+ if (newRelayer[logToRelayerSchemaMapping[key]].constructor.name === 'BigNumber') {
+ newRelayer[logToRelayerSchemaMapping[key]] = newRelayer[logToRelayerSchemaMapping[key]].toString();
+ }
+ } else if (
+ logToRelayerSchemaMapping[key] === 'known_fee_addresses' ||
+ logToRelayerSchemaMapping[key] === 'known_taker_addresses'
+ ) {
+ newRelayer[logToRelayerSchemaMapping[key]] = '{}';
+ } else {
+ newRelayer[logToRelayerSchemaMapping[key]] = '';
+ }
+ }
+ if (_.has(logRelayer, 'networks')) {
+ for (const network of logRelayer.networks) {
+ if (network.networkId === 1) {
+ if (_.has(network, 'sra_http_endpoint')) {
+ newRelayer.sra_http_endpoint = network.sra_http_endpoint;
+ }
+ if (_.has(network, 'sra_ws_endpoint')) {
+ newRelayer.sra_ws_endpoint = network.sra_ws_endpoint;
+ }
+ if (_.has(network, 'static_order_fields')) {
+ if (_.has(network, 'static_order_fields.fee_recipient_addresses')) {
+ newRelayer.fee_recipient_addresses = network.static_order_fields.fee_recipient_addresses;
+ }
+ if (_.has(network, 'static_order_fields.taker_addresses')) {
+ newRelayer.taker_addresses = network.static_order_fields.taker_addresses;
+ }
+ }
+ }
+ }
+ }
+ return newRelayer;
+ },
+ convertLogHistoricalPricesToHistoricalPricesObject(logHistoricalPrice: any): any {
+ const newHistoricalPrices: any = {};
+ for (const key in logToHistoricalPricesSchema) {
+ if (_.has(logHistoricalPrice, key)) {
+ newHistoricalPrices[logToHistoricalPricesSchema[key]] = _.get(logHistoricalPrice, key);
+ }
+ }
+ return newHistoricalPrices;
+ },
+ convertLogOrderToOrderObject(logOrder: any): any {
+ const newOrder: any = {};
+ for (const key in logToOrderSchemaMapping) {
+ if (_.has(logOrder, key)) {
+ console.log(key);
+ console.log(logOrder[key]);
+ newOrder[logToOrderSchemaMapping[key]] = _.get(logOrder, key);
+ if (newOrder[logToOrderSchemaMapping[key]].constructor.name === 'BigNumber') {
+ newOrder[logToOrderSchemaMapping[key]] = newOrder[logToOrderSchemaMapping[key]].toString();
+ }
+ }
+ }
+ console.log(newOrder);
+ return newOrder;
+ },
+};
+export const formatters = {
+ escapeSQLParams(params: any[]): string {
+ let escapedString = '';
+ for (const i in params) {
+ escapedString += "'" + params[i] + "',";
+ }
+ return escapedString.slice(0, -1);
+ },
+ escapeSQLParam(param: string): string {
+ return "'" + param + "'";
+ },
+};
diff --git a/packages/pipeline/src/zrx.ts b/packages/pipeline/src/zrx.ts
new file mode 100644
index 000000000..cbe10a55a
--- /dev/null
+++ b/packages/pipeline/src/zrx.ts
@@ -0,0 +1,11 @@
+import { ExchangeEvents, ZeroEx } from '0x.js';
+import * as dotenv from 'dotenv';
+import * as Web3 from 'web3';
+dotenv.config();
+const provider = new Web3.providers.HttpProvider(process.env.WEB3_PROVIDER_URL);
+const web3 = new Web3(provider);
+const MAINNET = 1;
+const zrx = new ZeroEx(provider, {
+ networkId: MAINNET,
+});
+export { web3, zrx };
diff --git a/packages/pipeline/tsconfig.json b/packages/pipeline/tsconfig.json
new file mode 100644
index 000000000..718e623c7
--- /dev/null
+++ b/packages/pipeline/tsconfig.json
@@ -0,0 +1,8 @@
+{
+ "extends": "../../tsconfig",
+ "compilerOptions": {
+ "outDir": "lib",
+ "rootDir": "."
+ },
+ "include": ["src/**/*", "test/**/*"]
+}
diff --git a/packages/pipeline/tslint.json b/packages/pipeline/tslint.json
new file mode 100644
index 000000000..ffaefe83a
--- /dev/null
+++ b/packages/pipeline/tslint.json
@@ -0,0 +1,3 @@
+{
+ "extends": ["@0xproject/tslint-config"]
+}
diff --git a/packages/pipeline/typedoc-tsconfig.json b/packages/pipeline/typedoc-tsconfig.json
new file mode 100644
index 000000000..b9c6b36f3
--- /dev/null
+++ b/packages/pipeline/typedoc-tsconfig.json
@@ -0,0 +1,7 @@
+{
+ "extends": "../../typedoc-tsconfig",
+ "compilerOptions": {
+ "outDir": "lib"
+ },
+ "include": ["src/**/*", "test/**/*"]
+}
diff --git a/yarn.lock b/yarn.lock
index e3f047296..cf9052f7c 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2,6 +2,23 @@
# yarn lockfile v1
+"0x.js@^0.38.3":
+ version "0.38.6"
+ resolved "https://registry.yarnpkg.com/0x.js/-/0x.js-0.38.6.tgz#f6cb2b6ef9aea2ff7696c923095e0b794c1d84fc"
+ dependencies:
+ "@0xproject/assert" "^0.2.14"
+ "@0xproject/base-contract" "^0.3.6"
+ "@0xproject/contract-wrappers" "^0.1.1"
+ "@0xproject/order-utils" "^0.0.9"
+ "@0xproject/order-watcher" "^0.0.8"
+ "@0xproject/sol-compiler" "^0.5.4"
+ "@0xproject/types" "^0.8.2"
+ "@0xproject/typescript-typings" "^0.4.3"
+ "@0xproject/utils" "^0.7.3"
+ "@0xproject/web3-wrapper" "^0.7.3"
+ ethers "3.0.22"
+ lodash "4.17.10"
+
"@0x-lerna-fork/add@^3.0.0-beta.22":
version "3.0.0-beta.22"
resolved "https://registry.yarnpkg.com/@0x-lerna-fork/add/-/add-3.0.0-beta.22.tgz#b69d7cbf6a28af6abb3aabb47fb617e7c6c7ac2c"
@@ -472,12 +489,220 @@
npmlog "^4.1.2"
write-file-atomic "^2.3.0"
+"@0xproject/assert@^0.2.14":
+ version "0.2.14"
+ resolved "https://registry.yarnpkg.com/@0xproject/assert/-/assert-0.2.14.tgz#7d5a373fedc8eb482716b730f4dddf3ef33bfa29"
+ dependencies:
+ "@0xproject/json-schemas" "^0.8.3"
+ "@0xproject/typescript-typings" "^0.4.3"
+ "@0xproject/utils" "^0.7.3"
+ lodash "4.17.10"
+ valid-url "1.0.9"
+
+"@0xproject/base-contract@^0.3.6":
+ version "0.3.6"
+ resolved "https://registry.yarnpkg.com/@0xproject/base-contract/-/base-contract-0.3.6.tgz#c6d742ce2904bdd9e77524d99ba2438d3aae8a40"
+ dependencies:
+ "@0xproject/types" "^0.8.2"
+ "@0xproject/typescript-typings" "^0.4.3"
+ "@0xproject/utils" "^0.7.3"
+ "@0xproject/web3-wrapper" "^0.7.3"
+ ethers "3.0.22"
+ lodash "4.17.10"
+
+"@0xproject/connect@^0.6.14":
+ version "0.6.17"
+ resolved "https://registry.yarnpkg.com/@0xproject/connect/-/connect-0.6.17.tgz#c281b1468d53b09adccbc87a9f1ee462dd6c0977"
+ dependencies:
+ "@0xproject/assert" "^0.2.14"
+ "@0xproject/json-schemas" "^0.8.3"
+ "@0xproject/types" "^0.8.2"
+ "@0xproject/typescript-typings" "^0.4.3"
+ "@0xproject/utils" "^0.7.3"
+ isomorphic-fetch "2.2.1"
+ lodash "4.17.10"
+ query-string "4.3.4"
+ websocket "1.0.25"
+
+"@0xproject/contract-wrappers@^0.1.1":
+ version "0.1.1"
+ resolved "https://registry.yarnpkg.com/@0xproject/contract-wrappers/-/contract-wrappers-0.1.1.tgz#8108d7ec051f202ef0cfa77c91c4ef994bf89881"
+ dependencies:
+ "@0xproject/assert" "^0.2.14"
+ "@0xproject/base-contract" "^0.3.6"
+ "@0xproject/fill-scenarios" "^0.0.6"
+ "@0xproject/json-schemas" "^0.8.3"
+ "@0xproject/order-utils" "^0.0.9"
+ "@0xproject/sol-compiler" "^0.5.4"
+ "@0xproject/types" "^0.8.2"
+ "@0xproject/typescript-typings" "^0.4.3"
+ "@0xproject/utils" "^0.7.3"
+ "@0xproject/web3-wrapper" "^0.7.3"
+ ethereumjs-blockstream "5.0.0"
+ ethereumjs-util "5.1.5"
+ ethers "3.0.22"
+ js-sha3 "0.7.0"
+ lodash "4.17.10"
+ uuid "3.2.1"
+
+"@0xproject/fill-scenarios@^0.0.6":
+ version "0.0.6"
+ resolved "https://registry.yarnpkg.com/@0xproject/fill-scenarios/-/fill-scenarios-0.0.6.tgz#969a2c49122650af5a1bf7fe7c6d286a0e6f2de2"
+ dependencies:
+ "@0xproject/base-contract" "^0.3.6"
+ "@0xproject/order-utils" "^0.0.9"
+ "@0xproject/types" "^0.8.2"
+ "@0xproject/typescript-typings" "^0.4.3"
+ "@0xproject/utils" "^0.7.3"
+ "@0xproject/web3-wrapper" "^0.7.3"
+ ethers "3.0.22"
+ lodash "4.17.10"
+
+"@0xproject/json-schemas@^0.8.3":
+ version "0.8.3"
+ resolved "https://registry.yarnpkg.com/@0xproject/json-schemas/-/json-schemas-0.8.3.tgz#455e6219a6bd05e990392165192a983a9ab89f26"
+ dependencies:
+ "@0xproject/typescript-typings" "^0.4.3"
+ "@types/node" "9.6.0"
+ jsonschema "1.2.2"
+ lodash.values "4.3.0"
+
"@0xproject/npm-cli-login@^0.0.11":
version "0.0.11"
resolved "https://registry.yarnpkg.com/@0xproject/npm-cli-login/-/npm-cli-login-0.0.11.tgz#3f1ec06112ce62aad300ff0575358f68aeecde2e"
dependencies:
npm-registry-client "7.0.9"
+"@0xproject/order-utils@^0.0.9":
+ version "0.0.9"
+ resolved "https://registry.yarnpkg.com/@0xproject/order-utils/-/order-utils-0.0.9.tgz#75225dfbd87335d18810abf995d8e077b9a84868"
+ dependencies:
+ "@0xproject/assert" "^0.2.14"
+ "@0xproject/json-schemas" "^0.8.3"
+ "@0xproject/types" "^0.8.2"
+ "@0xproject/typescript-typings" "^0.4.3"
+ "@0xproject/utils" "^0.7.3"
+ "@0xproject/web3-wrapper" "^0.7.3"
+ "@types/node" "9.6.0"
+ bn.js "4.11.7"
+ ethereumjs-abi "0.6.5"
+ ethereumjs-util "5.1.5"
+ lodash "4.17.10"
+
+"@0xproject/order-watcher@^0.0.8":
+ version "0.0.8"
+ resolved "https://registry.yarnpkg.com/@0xproject/order-watcher/-/order-watcher-0.0.8.tgz#cfba4c39ca8c5d39e076276479383d0aae022c6d"
+ dependencies:
+ "@0xproject/assert" "^0.2.14"
+ "@0xproject/base-contract" "^0.3.6"
+ "@0xproject/contract-wrappers" "^0.1.1"
+ "@0xproject/fill-scenarios" "^0.0.6"
+ "@0xproject/json-schemas" "^0.8.3"
+ "@0xproject/order-utils" "^0.0.9"
+ "@0xproject/types" "^0.8.2"
+ "@0xproject/typescript-typings" "^0.4.3"
+ "@0xproject/utils" "^0.7.3"
+ "@0xproject/web3-wrapper" "^0.7.3"
+ bintrees "1.0.2"
+ ethereum-types "^0.0.2"
+ ethereumjs-blockstream "5.0.0"
+ ethers "3.0.22"
+ lodash "4.17.10"
+
+"@0xproject/sol-compiler@^0.5.4":
+ version "0.5.4"
+ resolved "https://registry.yarnpkg.com/@0xproject/sol-compiler/-/sol-compiler-0.5.4.tgz#3e0b04b0c02c5ec046ebb962b5ed20978c6b4cdd"
+ dependencies:
+ "@0xproject/json-schemas" "^0.8.3"
+ "@0xproject/sol-resolver" "^0.0.8"
+ "@0xproject/types" "^0.8.2"
+ "@0xproject/typescript-typings" "^0.4.3"
+ "@0xproject/utils" "^0.7.3"
+ "@0xproject/web3-wrapper" "^0.7.3"
+ "@types/yargs" "10.0.2"
+ chalk "2.3.2"
+ ethereumjs-util "5.1.5"
+ isomorphic-fetch "2.2.1"
+ lodash "4.17.10"
+ mkdirp "0.5.1"
+ require-from-string "2.0.1"
+ semver "5.5.0"
+ solc "0.4.23"
+ web3 "0.20.6"
+ web3-eth-abi "1.0.0-beta.33"
+ yargs "11.1.0"
+
+"@0xproject/sol-resolver@^0.0.8":
+ version "0.0.8"
+ resolved "https://registry.yarnpkg.com/@0xproject/sol-resolver/-/sol-resolver-0.0.8.tgz#5496ccebc9afa9eedfb5b62104f6478aa852bf66"
+ dependencies:
+ "@0xproject/types" "^0.8.2"
+ "@0xproject/typescript-typings" "^0.0.3"
+ lodash "4.17.10"
+
+"@0xproject/tslint-config@^1.0.7":
+ version "1.0.8"
+ resolved "https://registry.yarnpkg.com/@0xproject/tslint-config/-/tslint-config-1.0.8.tgz#cfdce03b84dd2e108278b0024f89fe30218299e0"
+ dependencies:
+ lodash "^4.17.5"
+ tslint "5.11.0"
+ tslint-eslint-rules "5.4.0"
+ tslint-react "^3.2.0"
+ tsutils "3.0.0"
+
+"@0xproject/types@^0.5.0":
+ version "0.5.0"
+ resolved "https://registry.yarnpkg.com/@0xproject/types/-/types-0.5.0.tgz#ba3cfbc11a8c6344b57c9680aa7df2ea84b9bf05"
+ dependencies:
+ bignumber.js "~4.1.0"
+
+"@0xproject/types@^0.8.2":
+ version "0.8.2"
+ resolved "https://registry.yarnpkg.com/@0xproject/types/-/types-0.8.2.tgz#6f936b73bfb6f017b5102002d97da0881da92d1b"
+ dependencies:
+ "@types/node" "9.6.0"
+ bignumber.js "~4.1.0"
+
+"@0xproject/typescript-typings@^0.0.3":
+ version "0.0.3"
+ resolved "https://registry.yarnpkg.com/@0xproject/typescript-typings/-/typescript-typings-0.0.3.tgz#3272080bde00ade0a970b0d236686b483b08a1d0"
+ dependencies:
+ "@0xproject/types" "^0.5.0"
+ bignumber.js "~4.1.0"
+
+"@0xproject/typescript-typings@^0.4.3":
+ version "0.4.3"
+ resolved "https://registry.yarnpkg.com/@0xproject/typescript-typings/-/typescript-typings-0.4.3.tgz#f99f939a43f2764ad7182fcd78a71212a1d76d96"
+ dependencies:
+ "@0xproject/types" "^0.8.2"
+ bignumber.js "~4.1.0"
+ ethereum-types "^0.0.2"
+
+"@0xproject/utils@^0.7.3":
+ version "0.7.3"
+ resolved "https://registry.yarnpkg.com/@0xproject/utils/-/utils-0.7.3.tgz#ffa7c6da9bf0dd3e13694f185dcfc48a8981ff05"
+ dependencies:
+ "@0xproject/typescript-typings" "^0.4.3"
+ "@types/node" "9.6.0"
+ bignumber.js "~4.1.0"
+ ethereum-types "^0.0.2"
+ ethereumjs-util "^5.1.1"
+ ethers "3.0.22"
+ js-sha3 "0.7.0"
+ lodash "4.17.10"
+ web3 "0.20.6"
+
+"@0xproject/web3-wrapper@^0.7.3":
+ version "0.7.3"
+ resolved "https://registry.yarnpkg.com/@0xproject/web3-wrapper/-/web3-wrapper-0.7.3.tgz#9bd50b034b92fd505b6766b6e225f014b6d08b08"
+ dependencies:
+ "@0xproject/types" "^0.8.2"
+ "@0xproject/typescript-typings" "^0.4.3"
+ "@0xproject/utils" "^0.7.3"
+ ethers "3.0.22"
+ lodash "4.17.10"
+ web3 "0.20.6"
+
"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.0.0-beta.35":
version "7.0.0"
resolved "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.0.0.tgz#06e2ab19bdb535385559aabb5ba59729482800f8"
@@ -1202,10 +1427,18 @@
"@types/express" "*"
"@types/node" "*"
+"@types/caseless@*":
+ version "0.12.1"
+ resolved "https://registry.yarnpkg.com/@types/caseless/-/caseless-0.12.1.tgz#9794c69c8385d0192acc471a540d1f8e0d16218a"
+
"@types/cheerio@*":
version "0.22.9"
resolved "https://registry.npmjs.org/@types/cheerio/-/cheerio-0.22.9.tgz#b5990152604c2ada749b7f88cab3476f21f39d7b"
+"@types/command-line-args@^4.0.2":
+ version "4.0.3"
+ resolved "https://registry.yarnpkg.com/@types/command-line-args/-/command-line-args-4.0.3.tgz#6f09c9873beb00013496941f4d73b98410bc1802"
+
"@types/compare-versions@^3.0.0":
version "3.0.0"
resolved "https://registry.yarnpkg.com/@types/compare-versions/-/compare-versions-3.0.0.tgz#4a45dffe0ebbc00d0f2daef8a0e96ffc66cf5955"
@@ -1222,6 +1455,12 @@
version "2.0.0"
resolved "https://registry.npmjs.org/@types/detect-node/-/detect-node-2.0.0.tgz#696e024ddd105c72bbc6a2e3f97902a2886f2c3f"
+"@types/dotenv@^4.0.2":
+ version "4.0.3"
+ resolved "https://registry.yarnpkg.com/@types/dotenv/-/dotenv-4.0.3.tgz#ebcfc40da7bc0728b705945b7db48485ec5b4b67"
+ dependencies:
+ "@types/node" "*"
+
"@types/enzyme-adapter-react-16@^1.0.3":
version "1.0.3"
resolved "https://registry.npmjs.org/@types/enzyme-adapter-react-16/-/enzyme-adapter-react-16-1.0.3.tgz#0cf7025b036694ca8d596fe38f24162e7117acf1"
@@ -1286,6 +1525,12 @@
version "2.0.0"
resolved "https://registry.yarnpkg.com/@types/find-versions/-/find-versions-2.0.0.tgz#a976ca032ab8dd1161116604eded0620d2b85df2"
+"@types/form-data@*":
+ version "2.2.1"
+ resolved "https://registry.yarnpkg.com/@types/form-data/-/form-data-2.2.1.tgz#ee2b3b8eaa11c0938289953606b745b738c54b1e"
+ dependencies:
+ "@types/node" "*"
+
"@types/fs-extra@^5.0.3":
version "5.0.4"
resolved "https://registry.yarnpkg.com/@types/fs-extra/-/fs-extra-5.0.4.tgz#b971134d162cc0497d221adde3dbb67502225599"
@@ -1429,6 +1674,10 @@
version "9.6.5"
resolved "https://registry.yarnpkg.com/@types/node/-/node-9.6.5.tgz#ee700810fdf49ac1c399fc5980b7559b3e5a381d"
+"@types/node@9.6.0":
+ version "9.6.0"
+ resolved "http://registry.npmjs.org/@types/node/-/node-9.6.0.tgz#d3480ee666df9784b1001a1872a2f6ccefb6c2d7"
+
"@types/node@^10.3.2":
version "10.9.4"
resolved "https://registry.yarnpkg.com/@types/node/-/node-10.9.4.tgz#0f4cb2dc7c1de6096055357f70179043c33e9897"
@@ -1437,6 +1686,10 @@
version "10.5.7"
resolved "https://registry.npmjs.org/@types/node/-/node-10.5.7.tgz#960d9feb3ade2233bcc9843c918d740b4f78a7cf"
+"@types/node@^8.0.53":
+ version "8.10.36"
+ resolved "https://registry.yarnpkg.com/@types/node/-/node-8.10.36.tgz#eac05d576fbcd0b4ea3c912dc58c20475c08d9e4"
+
"@types/numeral@^0.0.22":
version "0.0.22"
resolved "https://registry.yarnpkg.com/@types/numeral/-/numeral-0.0.22.tgz#86bef1f0a2d743afdc2ef3168d45f2905e1a0b93"
@@ -1447,6 +1700,20 @@
dependencies:
"@types/node" "*"
+"@types/pg-types@*":
+ version "1.11.4"
+ resolved "https://registry.yarnpkg.com/@types/pg-types/-/pg-types-1.11.4.tgz#8d7c59fb509ce3dca3f8bae589252051c639a9a8"
+ dependencies:
+ moment ">=2.14.0"
+
+"@types/pg@^7.4.1":
+ version "7.4.10"
+ resolved "https://registry.yarnpkg.com/@types/pg/-/pg-7.4.10.tgz#1cf4d6c5d32e621566eca82981161b371fb90ace"
+ dependencies:
+ "@types/events" "*"
+ "@types/node" "*"
+ "@types/pg-types" "*"
+
"@types/prop-types@*":
version "15.5.5"
resolved "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.5.5.tgz#17038dd322c2325f5da650a94d5f9974943625e3"
@@ -1561,6 +1828,15 @@
dependencies:
redux "*"
+"@types/request@^2.0.13":
+ version "2.47.1"
+ resolved "https://registry.yarnpkg.com/@types/request/-/request-2.47.1.tgz#25410d3afbdac04c91a94ad9efc9824100735824"
+ dependencies:
+ "@types/caseless" "*"
+ "@types/form-data" "*"
+ "@types/node" "*"
+ "@types/tough-cookie" "*"
+
"@types/require-from-string@^1.2.0":
version "1.2.0"
resolved "https://registry.yarnpkg.com/@types/require-from-string/-/require-from-string-1.2.0.tgz#c18cfc8a2c1a0259e5841d1fef2b5e9d01c64242"
@@ -1620,6 +1896,10 @@
version "0.0.33"
resolved "https://registry.yarnpkg.com/@types/tmp/-/tmp-0.0.33.tgz#1073c4bc824754ae3d10cfab88ab0237ba964e4d"
+"@types/tough-cookie@*":
+ version "2.3.3"
+ resolved "https://registry.yarnpkg.com/@types/tough-cookie/-/tough-cookie-2.3.3.tgz#7f226d67d654ec9070e755f46daebf014628e9d9"
+
"@types/uuid@^3.4.2", "@types/uuid@^3.4.3":
version "3.4.3"
resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-3.4.3.tgz#121ace265f5569ce40f4f6d0ff78a338c732a754"
@@ -1643,7 +1923,7 @@
"@types/events" "*"
"@types/node" "*"
-"@types/yargs@^10.0.0":
+"@types/yargs@10.0.2", "@types/yargs@^10.0.0":
version "10.0.2"
resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-10.0.2.tgz#cc4ea921877874d1261e2c44b89807bc836e1b12"
@@ -1916,6 +2196,15 @@ agentkeepalive@^3.4.1:
dependencies:
humanize-ms "^1.2.1"
+airtable@^0.5.2:
+ version "0.5.6"
+ resolved "https://registry.yarnpkg.com/airtable/-/airtable-0.5.6.tgz#7f1a3cc038f77c9734c89deacdad500e25ad6fb1"
+ dependencies:
+ async "1.5.2"
+ lodash "4.17.10"
+ request "2.85.0"
+ xhr "2.3.3"
+
ajv-errors@^1.0.0:
version "1.0.0"
resolved "https://registry.npmjs.org/ajv-errors/-/ajv-errors-1.0.0.tgz#ecf021fa108fd17dfb5e6b383f2dd233e31ffc59"
@@ -2117,6 +2406,18 @@ arr-union@^3.1.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/arr-union/-/arr-union-3.1.0.tgz#e39b09aea9def866a8f206e288af63919bae39c4"
+array-back@^1.0.3, array-back@^1.0.4:
+ version "1.0.4"
+ resolved "https://registry.yarnpkg.com/array-back/-/array-back-1.0.4.tgz#644ba7f095f7ffcf7c43b5f0dc39d3c1f03c063b"
+ dependencies:
+ typical "^2.6.0"
+
+array-back@^2.0.0:
+ version "2.0.0"
+ resolved "https://registry.yarnpkg.com/array-back/-/array-back-2.0.0.tgz#6877471d51ecc9c9bfa6136fb6c7d5fe69748022"
+ dependencies:
+ typical "^2.6.1"
+
array-differ@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/array-differ/-/array-differ-1.0.0.tgz#eff52e3758249d33be402b8bb8e564bb2b5d4031"
@@ -2259,7 +2560,7 @@ async-limiter@~1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/async-limiter/-/async-limiter-1.0.0.tgz#78faed8c3d074ab81f22b4e985d79e8738f720f8"
-async@1.x, async@^1.4.0, async@^1.4.2, async@^1.5.2:
+async@1.5.2, async@1.x, async@^1.4.0, async@^1.4.2, async@^1.5.2:
version "1.5.2"
resolved "https://registry.yarnpkg.com/async/-/async-1.5.2.tgz#ec6a61ae56480c0c3cb241c95618e20892f9672a"
@@ -3051,7 +3352,7 @@ bindings@^1.2.1, bindings@^1.3.0:
version "1.3.0"
resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.3.0.tgz#b346f6ecf6a95f5a815c5839fc7cdb22502f1ed7"
-bintrees@^1.0.2:
+bintrees@1.0.2, bintrees@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/bintrees/-/bintrees-1.0.2.tgz#49f896d6e858a4a499df85c38fb399b9aff840f8"
@@ -3423,6 +3724,10 @@ buffer-to-arraybuffer@^0.0.5:
version "0.0.5"
resolved "https://registry.yarnpkg.com/buffer-to-arraybuffer/-/buffer-to-arraybuffer-0.0.5.tgz#6064a40fa76eb43c723aba9ef8f6e1216d10511a"
+buffer-writer@1.0.1:
+ version "1.0.1"
+ resolved "https://registry.yarnpkg.com/buffer-writer/-/buffer-writer-1.0.1.tgz#22a936901e3029afcd7547eb4487ceb697a3bf08"
+
buffer-xor@^1.0.3:
version "1.0.3"
resolved "https://registry.yarnpkg.com/buffer-xor/-/buffer-xor-1.0.3.tgz#26e61ed1422fb70dd42e6e36729ed51d855fe8d9"
@@ -3701,6 +4006,14 @@ chain-function@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/chain-function/-/chain-function-1.0.0.tgz#0d4ab37e7e18ead0bdc47b920764118ce58733dc"
+chalk@2.3.2:
+ version "2.3.2"
+ resolved "http://registry.npmjs.org/chalk/-/chalk-2.3.2.tgz#250dc96b07491bfd601e648d66ddf5f60c7a5c65"
+ dependencies:
+ ansi-styles "^3.2.1"
+ escape-string-regexp "^1.0.5"
+ supports-color "^5.3.0"
+
chalk@^1.0.0, chalk@^1.1.1, chalk@^1.1.3:
version "1.1.3"
resolved "https://registry.yarnpkg.com/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98"
@@ -4090,6 +4403,14 @@ combined-stream@1.0.6, combined-stream@^1.0.5, combined-stream@~1.0.5, combined-
dependencies:
delayed-stream "~1.0.0"
+command-line-args@^4.0.7:
+ version "4.0.7"
+ resolved "https://registry.yarnpkg.com/command-line-args/-/command-line-args-4.0.7.tgz#f8d1916ecb90e9e121eda6428e41300bfb64cc46"
+ dependencies:
+ array-back "^2.0.0"
+ find-replace "^1.0.3"
+ typical "^2.6.1"
+
commander@2.11.0:
version "2.11.0"
resolved "https://registry.yarnpkg.com/commander/-/commander-2.11.0.tgz#157152fd1e7a6c8d98a5b715cf376df928004563"
@@ -4098,7 +4419,7 @@ commander@2.15.1, commander@^2.12.1, commander@^2.8.1:
version "2.15.1"
resolved "https://registry.yarnpkg.com/commander/-/commander-2.15.1.tgz#df46e867d0fc2aec66a34662b406a9ccafff5b0f"
-commander@2.18.0:
+commander@2.18.0, commander@^2.9.0:
version "2.18.0"
resolved "https://registry.yarnpkg.com/commander/-/commander-2.18.0.tgz#2bf063ddee7c7891176981a2cc798e5754bc6970"
@@ -4403,6 +4724,10 @@ cookie@0.3.1:
version "0.3.1"
resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.3.1.tgz#e7e0a1f9ef43b4c8ba925c5c5a96e806d16873bb"
+cookiejar@^2.1.1:
+ version "2.1.2"
+ resolved "https://registry.yarnpkg.com/cookiejar/-/cookiejar-2.1.2.tgz#dd8a235530752f988f9a0844f3fc589e3111125c"
+
copy-concurrently@^1.0.0:
version "1.0.5"
resolved "https://registry.yarnpkg.com/copy-concurrently/-/copy-concurrently-1.0.5.tgz#92297398cae34937fcafd6ec8139c18051f0b5e0"
@@ -4793,6 +5118,13 @@ date-now@^0.1.4:
version "0.1.4"
resolved "https://registry.yarnpkg.com/date-now/-/date-now-0.1.4.tgz#eaf439fd4d4848ad74e5cc7dbef200672b9e345b"
+dateformat@^1.0.12:
+ version "1.0.12"
+ resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-1.0.12.tgz#9f124b67594c937ff706932e4a642cca8dbbfee9"
+ dependencies:
+ get-stdin "^4.0.1"
+ meow "^3.3.0"
+
dateformat@^2.0.0:
version "2.2.0"
resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-2.2.0.tgz#4065e2013cf9fb916ddfd82efb506ad4c6769062"
@@ -5850,6 +6182,13 @@ ethereum-common@^0.0.18:
version "0.0.18"
resolved "https://registry.yarnpkg.com/ethereum-common/-/ethereum-common-0.0.18.tgz#2fdc3576f232903358976eb39da783213ff9523f"
+ethereum-types@^0.0.2:
+ version "0.0.2"
+ resolved "https://registry.yarnpkg.com/ethereum-types/-/ethereum-types-0.0.2.tgz#6ef6faf46a24697cbf66b6c8a0ecf2095ce58c38"
+ dependencies:
+ "@types/node" "^8.0.53"
+ bignumber.js "~4.1.0"
+
ethereumjs-abi@0.6.5, ethereumjs-abi@^0.6.5:
version "0.6.5"
resolved "https://registry.yarnpkg.com/ethereumjs-abi/-/ethereumjs-abi-0.6.5.tgz#5a637ef16ab43473fa72a29ad90871405b3f5241"
@@ -5891,6 +6230,14 @@ ethereumjs-block@~1.2.2:
ethereumjs-util "^4.0.1"
merkle-patricia-tree "^2.1.2"
+ethereumjs-blockstream@5.0.0:
+ version "5.0.0"
+ resolved "https://registry.yarnpkg.com/ethereumjs-blockstream/-/ethereumjs-blockstream-5.0.0.tgz#63bfe9185757329a32822d5815562eb1dcd75d71"
+ dependencies:
+ immutable "3.8.2"
+ source-map-support "0.5.6"
+ uuid "3.2.1"
+
ethereumjs-blockstream@6.0.0:
version "6.0.0"
resolved "https://registry.yarnpkg.com/ethereumjs-blockstream/-/ethereumjs-blockstream-6.0.0.tgz#79d726d1f358935eb65195e91d40344c31e87eff"
@@ -5913,17 +6260,7 @@ ethereumjs-tx@^1.0.0, ethereumjs-tx@^1.1.1, ethereumjs-tx@^1.2.0, ethereumjs-tx@
ethereum-common "^0.0.18"
ethereumjs-util "^5.0.0"
-ethereumjs-util@^4.0.1, ethereumjs-util@^4.3.0, ethereumjs-util@^4.4.0:
- version "4.5.0"
- resolved "https://registry.yarnpkg.com/ethereumjs-util/-/ethereumjs-util-4.5.0.tgz#3e9428b317eebda3d7260d854fddda954b1f1bc6"
- dependencies:
- bn.js "^4.8.0"
- create-hash "^1.1.2"
- keccakjs "^0.2.0"
- rlp "^2.0.0"
- secp256k1 "^3.0.1"
-
-ethereumjs-util@^5.0.0, ethereumjs-util@^5.0.1, ethereumjs-util@^5.1.1, ethereumjs-util@^5.1.2, ethereumjs-util@^5.1.3, ethereumjs-util@^5.1.5:
+ethereumjs-util@5.1.5, ethereumjs-util@^5.0.0, ethereumjs-util@^5.0.1, ethereumjs-util@^5.1.1, ethereumjs-util@^5.1.2, ethereumjs-util@^5.1.3, ethereumjs-util@^5.1.5:
version "5.1.5"
resolved "https://registry.yarnpkg.com/ethereumjs-util/-/ethereumjs-util-5.1.5.tgz#2f02575852627d45622426f25ee4a0b5f377f27a"
dependencies:
@@ -5935,6 +6272,16 @@ ethereumjs-util@^5.0.0, ethereumjs-util@^5.0.1, ethereumjs-util@^5.1.1, ethereum
safe-buffer "^5.1.1"
secp256k1 "^3.0.1"
+ethereumjs-util@^4.0.1, ethereumjs-util@^4.3.0, ethereumjs-util@^4.4.0:
+ version "4.5.0"
+ resolved "https://registry.yarnpkg.com/ethereumjs-util/-/ethereumjs-util-4.5.0.tgz#3e9428b317eebda3d7260d854fddda954b1f1bc6"
+ dependencies:
+ bn.js "^4.8.0"
+ create-hash "^1.1.2"
+ keccakjs "^0.2.0"
+ rlp "^2.0.0"
+ secp256k1 "^3.0.1"
+
ethereumjs-util@^5.2.0:
version "5.2.0"
resolved "https://registry.yarnpkg.com/ethereumjs-util/-/ethereumjs-util-5.2.0.tgz#3e0c0d1741471acf1036052d048623dee54ad642"
@@ -6004,6 +6351,21 @@ ethereumjs-wallet@~0.6.0:
utf8 "^3.0.0"
uuid "^3.3.2"
+ethers@3.0.22:
+ version "3.0.22"
+ resolved "https://registry.yarnpkg.com/ethers/-/ethers-3.0.22.tgz#7fab1ea16521705837aa43c15831877b2716b436"
+ dependencies:
+ aes-js "3.0.0"
+ bn.js "^4.4.0"
+ elliptic "6.3.3"
+ hash.js "^1.0.0"
+ inherits "2.0.1"
+ js-sha3 "0.5.7"
+ scrypt-js "2.0.3"
+ setimmediate "1.0.4"
+ uuid "2.0.1"
+ xmlhttprequest "1.8.0"
+
ethers@~4.0.4:
version "4.0.4"
resolved "https://registry.yarnpkg.com/ethers/-/ethers-4.0.4.tgz#d3f85e8b27f4b59537e06526439b0fb15b44dc65"
@@ -6523,6 +6885,13 @@ find-index@^0.1.1:
version "0.1.1"
resolved "https://registry.yarnpkg.com/find-index/-/find-index-0.1.1.tgz#675d358b2ca3892d795a1ab47232f8b6e2e0dde4"
+find-replace@^1.0.3:
+ version "1.0.3"
+ resolved "https://registry.yarnpkg.com/find-replace/-/find-replace-1.0.3.tgz#b88e7364d2d9c959559f388c66670d6130441fa0"
+ dependencies:
+ array-back "^1.0.4"
+ test-value "^2.1.0"
+
find-up@^1.0.0:
version "1.1.2"
resolved "https://registry.yarnpkg.com/find-up/-/find-up-1.1.2.tgz#6b2e9822b1a2ce0a60ab64d610eccad53cb24d0f"
@@ -6827,7 +7196,7 @@ ganache-core@0xProject/ganache-core#monorepo-dep:
ethereumjs-tx "0xProject/ethereumjs-tx#fake-tx-include-signature-by-default"
ethereumjs-util "^5.2.0"
ethereumjs-vm "2.3.5"
- ethereumjs-wallet "~0.6.0"
+ ethereumjs-wallet "0.6.0"
fake-merkle-patricia-tree "~1.0.1"
heap "~0.2.6"
js-scrypt "^0.2.0"
@@ -6917,6 +7286,10 @@ gaze@^0.5.1:
dependencies:
globule "~0.1.0"
+generic-pool@2.4.3:
+ version "2.4.3"
+ resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.3.tgz#780c36f69dfad05a5a045dd37be7adca11a4f6ff"
+
generic-pool@~2.0.4:
version "2.0.4"
resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.0.4.tgz#f9718deda82fa125ed5c43e341c9a215a766d9a3"
@@ -8933,13 +9306,17 @@ js-sha3@0.5.7:
version "0.5.7"
resolved "https://registry.yarnpkg.com/js-sha3/-/js-sha3-0.5.7.tgz#0d4ffd8002d5333aabaf4a23eed2f6374c9f28e7"
+js-sha3@0.7.0, js-sha3@^0.7.0:
+ version "0.7.0"
+ resolved "https://registry.yarnpkg.com/js-sha3/-/js-sha3-0.7.0.tgz#0a5c57b36f79882573b2d84051f8bb85dd1bd63a"
+
js-sha3@^0.3.1:
version "0.3.1"
resolved "https://registry.yarnpkg.com/js-sha3/-/js-sha3-0.3.1.tgz#86122802142f0828502a0d1dee1d95e253bb0243"
-js-sha3@^0.7.0:
- version "0.7.0"
- resolved "https://registry.yarnpkg.com/js-sha3/-/js-sha3-0.7.0.tgz#0a5c57b36f79882573b2d84051f8bb85dd1bd63a"
+js-string-escape@1.0.1:
+ version "1.0.1"
+ resolved "https://registry.yarnpkg.com/js-string-escape/-/js-string-escape-1.0.1.tgz#e2625badbc0d67c7533e9edc1068c587ae4137ef"
js-tokens@^3.0.0, js-tokens@^3.0.2:
version "3.0.2"
@@ -9124,6 +9501,10 @@ jsonschema@*, jsonschema@1.2.4, jsonschema@^1.2.0:
version "1.2.4"
resolved "https://registry.yarnpkg.com/jsonschema/-/jsonschema-1.2.4.tgz#a46bac5d3506a254465bc548876e267c6d0d6464"
+jsonschema@1.2.2:
+ version "1.2.2"
+ resolved "https://registry.yarnpkg.com/jsonschema/-/jsonschema-1.2.2.tgz#83ab9c63d65bf4d596f91d81195e78772f6452bc"
+
jsprim@^1.2.2:
version "1.4.1"
resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2"
@@ -9814,7 +10195,7 @@ lodash.uniq@^4.5.0:
version "4.5.0"
resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773"
-lodash.values@^4.3.0:
+lodash.values@4.3.0, lodash.values@^4.3.0:
version "4.3.0"
resolved "https://registry.yarnpkg.com/lodash.values/-/lodash.values-4.3.0.tgz#a3a6c2b0ebecc5c2cba1c17e6e620fe81b53d347"
@@ -9824,6 +10205,10 @@ lodash.words@^3.0.0:
dependencies:
lodash._root "^3.0.0"
+lodash@4.17.10, lodash@^4.17.10:
+ version "4.17.10"
+ resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.10.tgz#1b7793cf7259ea38fb3661d4d38b3260af8ae4e7"
+
lodash@=4.17.4:
version "4.17.4"
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae"
@@ -9840,10 +10225,6 @@ lodash@^4.14.0, lodash@^4.17.4, lodash@^4.17.5, lodash@^4.2.1, lodash@^4.3.0, lo
version "4.17.5"
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.5.tgz#99a92d65c0272debe8c96b6057bc8fbfa3bed511"
-lodash@^4.17.10:
- version "4.17.10"
- resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.10.tgz#1b7793cf7259ea38fb3661d4d38b3260af8ae4e7"
-
lodash@~1.0.1:
version "1.0.2"
resolved "https://registry.yarnpkg.com/lodash/-/lodash-1.0.2.tgz#8f57560c83b59fc270bd3d561b690043430e2551"
@@ -10260,6 +10641,12 @@ micromatch@^3.0.4, micromatch@^3.1.10, micromatch@^3.1.4, micromatch@^3.1.8, mic
snapdragon "^0.8.1"
to-regex "^3.0.2"
+migrate@^0.2.2:
+ version "0.2.4"
+ resolved "https://registry.yarnpkg.com/migrate/-/migrate-0.2.4.tgz#30a54dac5c269b7321cec56b2aff4cccaea6418d"
+ dependencies:
+ dateformat "^1.0.12"
+
miller-rabin@^4.0.0:
version "4.0.1"
resolved "https://registry.yarnpkg.com/miller-rabin/-/miller-rabin-4.0.1.tgz#f080351c865b0dc562a8462966daa53543c78a4d"
@@ -10479,7 +10866,7 @@ moment@2.21.0:
version "2.21.0"
resolved "https://registry.yarnpkg.com/moment/-/moment-2.21.0.tgz#2a114b51d2a6ec9e6d83cf803f838a878d8a023a"
-moment@^2.6.0:
+moment@>=2.14.0, moment@^2.6.0:
version "2.22.2"
resolved "http://registry.yarnpkg.com/moment/-/moment-2.22.2.tgz#3c257f9839fc0e93ff53149632239eb90783ff66"
@@ -10769,6 +11156,16 @@ node-pre-gyp@^0.6.39:
tar "^2.2.1"
tar-pack "^3.4.0"
+node-redshift@^0.1.5:
+ version "0.1.5"
+ resolved "https://registry.yarnpkg.com/node-redshift/-/node-redshift-0.1.5.tgz#c6c7cff16d230a148094225f20072939b66582f0"
+ dependencies:
+ bluebird "^3.5.0"
+ commander "^2.9.0"
+ migrate "^0.2.2"
+ pg "^6.1.2"
+ sql-bricks "^1.2.3"
+
nodemon@^1.11.0:
version "1.17.3"
resolved "https://registry.yarnpkg.com/nodemon/-/nodemon-1.17.3.tgz#3b0bbc2ee05ccb43b1aef15ba05c63c7bc9b8530"
@@ -11049,6 +11446,10 @@ oauth-sign@~0.9.0:
version "0.9.0"
resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455"
+object-assign@4.1.0:
+ version "4.1.0"
+ resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.0.tgz#7a3b3d0e98063d43f4c03f2e8ae6cd51a86883a0"
+
object-assign@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-3.0.0.tgz#9bedd5ca0897949bca47e7ff408062d549f587f2"
@@ -11422,6 +11823,10 @@ package-json@^4.0.0, package-json@^4.0.1:
registry-url "^3.0.3"
semver "^5.1.0"
+packet-reader@0.3.1:
+ version "0.3.1"
+ resolved "https://registry.yarnpkg.com/packet-reader/-/packet-reader-0.3.1.tgz#cd62e60af8d7fea8a705ec4ff990871c46871f27"
+
pacote@^9.1.0:
version "9.1.0"
resolved "https://registry.yarnpkg.com/pacote/-/pacote-9.1.0.tgz#59810859bbd72984dcb267269259375d32f391e5"
@@ -11683,6 +12088,75 @@ performance-now@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b"
+pg-connection-string@0.1.3:
+ version "0.1.3"
+ resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-0.1.3.tgz#da1847b20940e42ee1492beaf65d49d91b245df7"
+
+pg-int8@1.0.1:
+ version "1.0.1"
+ resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c"
+
+pg-pool@1.*:
+ version "1.8.0"
+ resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-1.8.0.tgz#f7ec73824c37a03f076f51bfdf70e340147c4f37"
+ dependencies:
+ generic-pool "2.4.3"
+ object-assign "4.1.0"
+
+pg-pool@~2.0.3:
+ version "2.0.3"
+ resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-2.0.3.tgz#c022032c8949f312a4f91fb6409ce04076be3257"
+
+pg-types@1.*:
+ version "1.13.0"
+ resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-1.13.0.tgz#75f490b8a8abf75f1386ef5ec4455ecf6b345c63"
+ dependencies:
+ pg-int8 "1.0.1"
+ postgres-array "~1.0.0"
+ postgres-bytea "~1.0.0"
+ postgres-date "~1.0.0"
+ postgres-interval "^1.1.0"
+
+pg-types@~1.12.1:
+ version "1.12.1"
+ resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-1.12.1.tgz#d64087e3903b58ffaad279e7595c52208a14c3d2"
+ dependencies:
+ postgres-array "~1.0.0"
+ postgres-bytea "~1.0.0"
+ postgres-date "~1.0.0"
+ postgres-interval "^1.1.0"
+
+pg@^6.1.2:
+ version "6.4.2"
+ resolved "https://registry.yarnpkg.com/pg/-/pg-6.4.2.tgz#c364011060eac7a507a2ae063eb857ece910e27f"
+ dependencies:
+ buffer-writer "1.0.1"
+ js-string-escape "1.0.1"
+ packet-reader "0.3.1"
+ pg-connection-string "0.1.3"
+ pg-pool "1.*"
+ pg-types "1.*"
+ pgpass "1.*"
+ semver "4.3.2"
+
+pg@^7.4.1:
+ version "7.4.3"
+ resolved "https://registry.yarnpkg.com/pg/-/pg-7.4.3.tgz#f7b6f93f5340ecc2596afbb94a13e3d6b609834b"
+ dependencies:
+ buffer-writer "1.0.1"
+ packet-reader "0.3.1"
+ pg-connection-string "0.1.3"
+ pg-pool "~2.0.3"
+ pg-types "~1.12.1"
+ pgpass "1.x"
+ semver "4.3.2"
+
+pgpass@1.*, pgpass@1.x:
+ version "1.0.2"
+ resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.2.tgz#2a7bb41b6065b67907e91da1b07c1847c877b306"
+ dependencies:
+ split "^1.0.0"
+
pify@^2.0.0, pify@^2.2.0, pify@^2.3.0:
version "2.3.0"
resolved "https://registry.yarnpkg.com/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c"
@@ -12011,6 +12485,24 @@ postcss@^6.0.1:
source-map "^0.6.1"
supports-color "^5.3.0"
+postgres-array@~1.0.0:
+ version "1.0.2"
+ resolved "https://registry.yarnpkg.com/postgres-array/-/postgres-array-1.0.2.tgz#8e0b32eb03bf77a5c0a7851e0441c169a256a238"
+
+postgres-bytea@~1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/postgres-bytea/-/postgres-bytea-1.0.0.tgz#027b533c0aa890e26d172d47cf9ccecc521acd35"
+
+postgres-date@~1.0.0:
+ version "1.0.3"
+ resolved "https://registry.yarnpkg.com/postgres-date/-/postgres-date-1.0.3.tgz#e2d89702efdb258ff9d9cee0fe91bd06975257a8"
+
+postgres-interval@^1.1.0:
+ version "1.1.2"
+ resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.1.2.tgz#bf71ff902635f21cb241a013fc421d81d1db15a9"
+ dependencies:
+ xtend "^4.0.0"
+
prebuild-install@^2.2.2:
version "2.5.3"
resolved "https://registry.yarnpkg.com/prebuild-install/-/prebuild-install-2.5.3.tgz#9f65f242782d370296353710e9bc843490c19f69"
@@ -12368,7 +12860,7 @@ qs@~6.5.2:
version "6.5.2"
resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36"
-query-string@^4.1.0:
+query-string@4.3.4, query-string@^4.1.0:
version "4.3.4"
resolved "https://registry.yarnpkg.com/query-string/-/query-string-4.3.4.tgz#bbb693b9ca915c232515b228b1a02b609043dbeb"
dependencies:
@@ -12406,6 +12898,12 @@ querystringify@~1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/querystringify/-/querystringify-1.0.0.tgz#6286242112c5b712fa654e526652bf6a13ff05cb"
+queue@^4.4.2:
+ version "4.5.0"
+ resolved "https://registry.yarnpkg.com/queue/-/queue-4.5.0.tgz#0f125191a983e3c38fcc0c0c75087d358d0857f4"
+ dependencies:
+ inherits "~2.0.0"
+
quick-lru@^1.0.0:
version "1.1.0"
resolved "http://registry.yarnpkg.com/quick-lru/-/quick-lru-1.1.0.tgz#4360b17c61136ad38078397ff11416e186dcfbb8"
@@ -13169,32 +13667,7 @@ request@2.81.0, "request@>=2.9.0 <2.82.0":
tunnel-agent "^0.6.0"
uuid "^3.0.0"
-request@^2.47.0, request@^2.87.0:
- version "2.88.0"
- resolved "https://registry.yarnpkg.com/request/-/request-2.88.0.tgz#9c2fca4f7d35b592efe57c7f0a55e81052124fef"
- dependencies:
- aws-sign2 "~0.7.0"
- aws4 "^1.8.0"
- caseless "~0.12.0"
- combined-stream "~1.0.6"
- extend "~3.0.2"
- forever-agent "~0.6.1"
- form-data "~2.3.2"
- har-validator "~5.1.0"
- http-signature "~1.2.0"
- is-typedarray "~1.0.0"
- isstream "~0.1.2"
- json-stringify-safe "~5.0.1"
- mime-types "~2.1.19"
- oauth-sign "~0.9.0"
- performance-now "^2.1.0"
- qs "~6.5.2"
- safe-buffer "^5.1.2"
- tough-cookie "~2.4.3"
- tunnel-agent "^0.6.0"
- uuid "^3.3.2"
-
-request@^2.54.0, request@^2.67.0, request@^2.79.0:
+request@2.85.0, request@^2.54.0, request@^2.67.0, request@^2.79.0:
version "2.85.0"
resolved "https://registry.yarnpkg.com/request/-/request-2.85.0.tgz#5a03615a47c61420b3eb99b7dba204f83603e1fa"
dependencies:
@@ -13221,6 +13694,31 @@ request@^2.54.0, request@^2.67.0, request@^2.79.0:
tunnel-agent "^0.6.0"
uuid "^3.1.0"
+request@^2.47.0, request@^2.83.0, request@^2.87.0:
+ version "2.88.0"
+ resolved "https://registry.yarnpkg.com/request/-/request-2.88.0.tgz#9c2fca4f7d35b592efe57c7f0a55e81052124fef"
+ dependencies:
+ aws-sign2 "~0.7.0"
+ aws4 "^1.8.0"
+ caseless "~0.12.0"
+ combined-stream "~1.0.6"
+ extend "~3.0.2"
+ forever-agent "~0.6.1"
+ form-data "~2.3.2"
+ har-validator "~5.1.0"
+ http-signature "~1.2.0"
+ is-typedarray "~1.0.0"
+ isstream "~0.1.2"
+ json-stringify-safe "~5.0.1"
+ mime-types "~2.1.19"
+ oauth-sign "~0.9.0"
+ performance-now "^2.1.0"
+ qs "~6.5.2"
+ safe-buffer "^5.1.2"
+ tough-cookie "~2.4.3"
+ tunnel-agent "^0.6.0"
+ uuid "^3.3.2"
+
request@^2.85.0:
version "2.87.0"
resolved "https://registry.npmjs.org/request/-/request-2.87.0.tgz#32f00235cd08d482b4d0d68db93a829c0ed5756e"
@@ -13250,6 +13748,10 @@ require-directory@^2.1.1:
version "2.1.1"
resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42"
+require-from-string@2.0.1:
+ version "2.0.1"
+ resolved "https://registry.yarnpkg.com/require-from-string/-/require-from-string-2.0.1.tgz#c545233e9d7da6616e9d59adfb39fc9f588676ff"
+
require-from-string@^1.1.0:
version "1.2.1"
resolved "https://registry.yarnpkg.com/require-from-string/-/require-from-string-1.2.1.tgz#529c9ccef27380adfec9a2f965b649bbee636418"
@@ -13598,6 +14100,10 @@ scrypt-async@^1.2.0:
version "1.3.1"
resolved "https://registry.yarnpkg.com/scrypt-async/-/scrypt-async-1.3.1.tgz#a11fd6fac981b4b823ee01dee0221169500ddae9"
+scrypt-js@2.0.3:
+ version "2.0.3"
+ resolved "http://registry.npmjs.org/scrypt-js/-/scrypt-js-2.0.3.tgz#bb0040be03043da9a012a2cea9fc9f852cfc87d4"
+
scrypt-js@2.0.4:
version "2.0.4"
resolved "https://registry.yarnpkg.com/scrypt-js/-/scrypt-js-2.0.4.tgz#32f8c5149f0797672e551c07e230f834b6af5f16"
@@ -13695,6 +14201,10 @@ semver-sort@0.0.4:
version "5.5.0"
resolved "https://registry.yarnpkg.com/semver/-/semver-5.5.0.tgz#dc4bbc7a6ca9d916dee5d43516f0092b58f7b8ab"
+semver@4.3.2:
+ version "4.3.2"
+ resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.2.tgz#c7a07158a80bedd052355b770d82d6640f803be7"
+
semver@^4.1.0:
version "4.3.6"
resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.6.tgz#300bc6e0e86374f7ba61068b5b1ecd57fc6532da"
@@ -14042,9 +14552,9 @@ socks@~2.2.0:
ip "^1.1.5"
smart-buffer "^4.0.1"
-solc@0.4.24:
- version "0.4.24"
- resolved "https://registry.yarnpkg.com/solc/-/solc-0.4.24.tgz#354f14b269b38cbaa82a47d1ff151723502b954e"
+solc@0.4.23, solc@^0.4.23:
+ version "0.4.23"
+ resolved "https://registry.yarnpkg.com/solc/-/solc-0.4.23.tgz#54a0ff4015827b32fddb62c0a418b5247310a58e"
dependencies:
fs-extra "^0.30.0"
memorystream "^0.3.1"
@@ -14052,9 +14562,9 @@ solc@0.4.24:
semver "^5.3.0"
yargs "^4.7.1"
-solc@^0.4.2:
- version "0.4.21"
- resolved "https://registry.yarnpkg.com/solc/-/solc-0.4.21.tgz#6a7ecd505bfa0fc268330d5de6b9ae65c8c68264"
+solc@0.4.24:
+ version "0.4.24"
+ resolved "https://registry.yarnpkg.com/solc/-/solc-0.4.24.tgz#354f14b269b38cbaa82a47d1ff151723502b954e"
dependencies:
fs-extra "^0.30.0"
memorystream "^0.3.1"
@@ -14062,9 +14572,9 @@ solc@^0.4.2:
semver "^5.3.0"
yargs "^4.7.1"
-solc@^0.4.23:
- version "0.4.23"
- resolved "https://registry.yarnpkg.com/solc/-/solc-0.4.23.tgz#54a0ff4015827b32fddb62c0a418b5247310a58e"
+solc@^0.4.2:
+ version "0.4.21"
+ resolved "https://registry.yarnpkg.com/solc/-/solc-0.4.21.tgz#6a7ecd505bfa0fc268330d5de6b9ae65c8c68264"
dependencies:
fs-extra "^0.30.0"
memorystream "^0.3.1"
@@ -14295,6 +14805,12 @@ sprintf-js@~1.0.2:
version "1.0.3"
resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c"
+sql-bricks@^1.2.3:
+ version "1.5.1"
+ resolved "https://registry.yarnpkg.com/sql-bricks/-/sql-bricks-1.5.1.tgz#68dc6d77b21a696f4885184a031b861ea5575981"
+ dependencies:
+ underscore "1.4.x"
+
sshpk@^1.7.0:
version "1.14.1"
resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.14.1.tgz#130f5975eddad963f1d56f92b9ac6c51fa9f83eb"
@@ -14915,6 +15431,13 @@ test-exclude@^4.2.1:
read-pkg-up "^1.0.1"
require-main-filename "^1.0.1"
+test-value@^2.1.0:
+ version "2.1.0"
+ resolved "https://registry.yarnpkg.com/test-value/-/test-value-2.1.0.tgz#11da6ff670f3471a73b625ca4f3fdcf7bb748291"
+ dependencies:
+ array-back "^1.0.3"
+ typical "^2.6.0"
+
text-encoding@^0.6.4:
version "0.6.4"
resolved "https://registry.yarnpkg.com/text-encoding/-/text-encoding-0.6.4.tgz#e399a982257a276dae428bb92845cb71bdc26d19"
@@ -15419,6 +15942,10 @@ typewiselite@~1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/typewiselite/-/typewiselite-1.0.0.tgz#c8882fa1bb1092c06005a97f34ef5c8508e3664e"
+typical@^2.6.0, typical@^2.6.1:
+ version "2.6.1"
+ resolved "https://registry.yarnpkg.com/typical/-/typical-2.6.1.tgz#5c080e5d661cbbe38259d2e70a3c7253e873881d"
+
u2f-api@0.2.7:
version "0.2.7"
resolved "https://registry.yarnpkg.com/u2f-api/-/u2f-api-0.2.7.tgz#17bf196b242f6bf72353d9858e6a7566cc192720"
@@ -15509,14 +16036,14 @@ undefsafe@^2.0.2:
dependencies:
debug "^2.2.0"
+underscore@1.4.x, underscore@~1.4.4:
+ version "1.4.4"
+ resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.4.4.tgz#61a6a32010622afa07963bf325203cf12239d604"
+
underscore@1.8.3:
version "1.8.3"
resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.8.3.tgz#4f3fb53b106e6097fcf9cb4109f2a5e9bdfa5022"
-underscore@~1.4.4:
- version "1.4.4"
- resolved "https://registry.npmjs.org/underscore/-/underscore-1.4.4.tgz#61a6a32010622afa07963bf325203cf12239d604"
-
unherit@^1.0.4:
version "1.1.0"
resolved "https://registry.yarnpkg.com/unherit/-/unherit-1.1.0.tgz#6b9aaedfbf73df1756ad9e316dd981885840cd7d"
@@ -15806,7 +16333,7 @@ v8flags@^2.0.2:
dependencies:
user-home "^1.1.1"
-valid-url@^1.0.9:
+valid-url@1.0.9, valid-url@^1.0.9:
version "1.0.9"
resolved "https://registry.yarnpkg.com/valid-url/-/valid-url-1.0.9.tgz#1c14479b40f1397a75782f115e4086447433a200"
@@ -16012,6 +16539,14 @@ web3-bzz@1.0.0-beta.34:
swarm-js "0.1.37"
underscore "1.8.3"
+web3-core-helpers@1.0.0-beta.33:
+ version "1.0.0-beta.33"
+ resolved "https://registry.yarnpkg.com/web3-core-helpers/-/web3-core-helpers-1.0.0-beta.33.tgz#2af733e504db05e7c3648c1dacf577b0ec15dc43"
+ dependencies:
+ underscore "1.8.3"
+ web3-eth-iban "1.0.0-beta.33"
+ web3-utils "1.0.0-beta.33"
+
web3-core-helpers@1.0.0-beta.34:
version "1.0.0-beta.34"
resolved "https://registry.yarnpkg.com/web3-core-helpers/-/web3-core-helpers-1.0.0-beta.34.tgz#b168da00d3e19e156bc15ae203203dd4dfee2d03"
@@ -16064,6 +16599,15 @@ web3-core@1.0.0-beta.34:
web3-core-requestmanager "1.0.0-beta.34"
web3-utils "1.0.0-beta.34"
+web3-eth-abi@1.0.0-beta.33:
+ version "1.0.0-beta.33"
+ resolved "https://registry.yarnpkg.com/web3-eth-abi/-/web3-eth-abi-1.0.0-beta.33.tgz#2221f7151643660032a4df340f612349168c824a"
+ dependencies:
+ bn.js "4.11.6"
+ underscore "1.8.3"
+ web3-core-helpers "1.0.0-beta.33"
+ web3-utils "1.0.0-beta.33"
+
web3-eth-abi@1.0.0-beta.34, web3-eth-abi@^1.0.0-beta.24:
version "1.0.0-beta.34"
resolved "https://registry.yarnpkg.com/web3-eth-abi/-/web3-eth-abi-1.0.0-beta.34.tgz#034533e3aa2f7e59ff31793eaea685c0ed5af67a"
@@ -16101,6 +16645,13 @@ web3-eth-contract@1.0.0-beta.34:
web3-eth-abi "1.0.0-beta.34"
web3-utils "1.0.0-beta.34"
+web3-eth-iban@1.0.0-beta.33:
+ version "1.0.0-beta.33"
+ resolved "https://registry.yarnpkg.com/web3-eth-iban/-/web3-eth-iban-1.0.0-beta.33.tgz#1d73d0c5288a4565b1754a75b5fb3ea0b77a532f"
+ dependencies:
+ bn.js "4.11.6"
+ web3-utils "1.0.0-beta.33"
+
web3-eth-iban@1.0.0-beta.34:
version "1.0.0-beta.34"
resolved "https://registry.yarnpkg.com/web3-eth-iban/-/web3-eth-iban-1.0.0-beta.34.tgz#9af458605867ccf74ea979aaf326b38ba6a5ba0c"
@@ -16231,6 +16782,18 @@ web3-typescript-typings@^0.10.2:
dependencies:
bignumber.js "~4.1.0"
+web3-utils@1.0.0-beta.33:
+ version "1.0.0-beta.33"
+ resolved "https://registry.yarnpkg.com/web3-utils/-/web3-utils-1.0.0-beta.33.tgz#e091b7994f09b714b0198a4057d3ad2eb8cbe238"
+ dependencies:
+ bn.js "4.11.6"
+ eth-lib "0.1.27"
+ ethjs-unit "0.1.6"
+ number-to-bn "1.7.0"
+ randomhex "0.1.5"
+ underscore "1.8.3"
+ utf8 "2.1.1"
+
web3-utils@1.0.0-beta.34:
version "1.0.0-beta.34"
resolved "https://registry.yarnpkg.com/web3-utils/-/web3-utils-1.0.0-beta.34.tgz#9411fc39aaef39ca4e06169f762297d9ff020970"
@@ -16253,6 +16816,16 @@ web3@0.20.2:
xhr2 "*"
xmlhttprequest "*"
+web3@0.20.6:
+ version "0.20.6"
+ resolved "http://registry.npmjs.org/web3/-/web3-0.20.6.tgz#3e97306ae024fb24e10a3d75c884302562215120"
+ dependencies:
+ bignumber.js "git+https://github.com/frozeman/bignumber.js-nolookahead.git"
+ crypto-js "^3.1.4"
+ utf8 "^2.1.1"
+ xhr2 "*"
+ xmlhttprequest "*"
+
web3@^0.18.0:
version "0.18.4"
resolved "https://registry.yarnpkg.com/web3/-/web3-0.18.4.tgz#81ec1784145491f2eaa8955b31c06049e07c5e7d"
@@ -16263,6 +16836,16 @@ web3@^0.18.0:
xhr2 "*"
xmlhttprequest "*"
+web3@^0.20.0:
+ version "0.20.7"
+ resolved "https://registry.yarnpkg.com/web3/-/web3-0.20.7.tgz#1605e6d81399ed6f85a471a4f3da0c8be57df2f7"
+ dependencies:
+ bignumber.js "git+https://github.com/frozeman/bignumber.js-nolookahead.git"
+ crypto-js "^3.1.4"
+ utf8 "^2.1.1"
+ xhr2-cookies "^1.1.0"
+ xmlhttprequest "*"
+
web3@^1.0.0-beta.34:
version "1.0.0-beta.34"
resolved "https://registry.yarnpkg.com/web3/-/web3-1.0.0-beta.34.tgz#347e561b784098cb5563315f490479a1d91f2ab1"
@@ -16410,7 +16993,7 @@ websocket-extensions@>=0.1.1:
version "0.1.3"
resolved "https://registry.yarnpkg.com/websocket-extensions/-/websocket-extensions-0.1.3.tgz#5d2ff22977003ec687a4b87073dfbbac146ccf29"
-websocket@^1.0.24, websocket@^1.0.25:
+websocket@1.0.25, websocket@^1.0.24, websocket@^1.0.25:
version "1.0.25"
resolved "https://registry.yarnpkg.com/websocket/-/websocket-1.0.25.tgz#998ec790f0a3eacb8b08b50a4350026692a11958"
dependencies:
@@ -16653,10 +17236,25 @@ xhr-request@^1.0.1:
url-set-query "^1.0.0"
xhr "^2.0.4"
+xhr2-cookies@^1.1.0:
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/xhr2-cookies/-/xhr2-cookies-1.1.0.tgz#7d77449d0999197f155cb73b23df72505ed89d48"
+ dependencies:
+ cookiejar "^2.1.1"
+
xhr2@*, xhr2@0.1.4:
version "0.1.4"
resolved "https://registry.yarnpkg.com/xhr2/-/xhr2-0.1.4.tgz#7f87658847716db5026323812f818cadab387a5f"
+xhr@2.3.3:
+ version "2.3.3"
+ resolved "https://registry.yarnpkg.com/xhr/-/xhr-2.3.3.tgz#ad6b810e0917ce72b5ec704f5d41f1503b8e7524"
+ dependencies:
+ global "~4.3.0"
+ is-function "^1.0.1"
+ parse-headers "^2.0.0"
+ xtend "^4.0.0"
+
xhr@^2.0.4, xhr@^2.2.0, xhr@^2.3.3:
version "2.4.1"
resolved "https://registry.yarnpkg.com/xhr/-/xhr-2.4.1.tgz#ba982cced205ae5eec387169ac9dc77ca4853d38"