From 57e7119c0d4f1ab7dd1d4c0118e72dc1706e2151 Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Mon, 17 Sep 2018 11:27:38 -0700 Subject: Rebase pipeline branch off development --- packages/pipeline/package.json | 52 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 packages/pipeline/package.json (limited to 'packages/pipeline/package.json') diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json new file mode 100644 index 000000000..9d47f69f3 --- /dev/null +++ b/packages/pipeline/package.json @@ -0,0 +1,52 @@ +{ + "name": "@0xproject/pipeline", + "version": "0.0.1", + "private": true, + "description": "Data pipeline for offline analysis", + "scripts": { + "build": "yarn tsc -b", + "test": "yarn run_mocha", + "rebuild_and_test": "run-s build test", + "test:circleci": "yarn test:coverage", + "run_mocha": "mocha --require source-map-support/register --require make-promises-safe lib/test/**/*_test.js --bail --exit", + "test:coverage": "nyc npm run test --all && yarn coverage:report:lcov", + "coverage:report:lcov": "nyc report --reporter=text-lcov > coverage/lcov.info", + "clean": "shx rm -rf lib", + "lint": "tslint --project . --exclude **/src/generated_contract_wrappers/**/*" + }, + "repository": { + "type": "git", + "url": "https://github.com/0xProject/0x.js.git" + }, + "license": "Apache-2.0", + "bugs": { + "url": "https://github.com/0xProject/0x.js/issues" + }, + "homepage": "https://github.com/0xProject/0x.js/packages/pipeline/README.md", + "devDependencies": { + "@0xproject/tslint-config": "^1.0.7", + "@types/command-line-args": "^4.0.2", + "@types/dotenv": "^4.0.2", + "@types/glob": "^5.0.33", + "@types/lodash": "4.14.104", + "@types/node": "^8.0.53", + "@types/pg": "^7.4.1", + "@types/request": "^2.0.13", + "shx": "^0.2.2", + "tslint": "5.11.0", + "typescript": "3.0.1" + }, + "dependencies": { + "0x.js": "^0.38.3", + "@0xproject/connect": "^0.6.14", + "airtable": "^0.5.2", + "command-line-args": "^4.0.7", + "dotenv": "^4.0.0", + "lodash": "^4.17.4", + "node-redshift": "^0.1.5", + "pg": "^7.4.1", + "queue": "^4.4.2", + "request": "^2.83.0", + "web3": "^0.20.0" + } +} -- cgit v1.2.3 From cd73a047efbc1b7afa884ef27a1cc4991e20efd8 Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Wed, 19 Sep 2018 10:51:12 -0700 Subject: Remove old code. Create function for getting contract events via etherscan --- packages/pipeline/package.json | 45 ++++++++++++++++++------------------------ 1 file changed, 19 insertions(+), 26 deletions(-) (limited to 'packages/pipeline/package.json') diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index 9d47f69f3..b4ed33d8d 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -4,49 +4,42 @@ "private": true, "description": "Data pipeline for offline analysis", "scripts": { - "build": "yarn tsc -b", + "build": "yarn pre_build && tsc -b", + "pre_build": "run-s update_artifacts copy_artifacts", + "copy_artifacts": "copyfiles -u 2 './src/artifacts/**/*.json' ./lib/src/artifacts", + "update_artifacts": "for i in ${npm_package_config_contracts_v2}; do copyfiles -u 4 ../migrations/artifacts/2.0.0/$i.json src/artifacts; done;", "test": "yarn run_mocha", "rebuild_and_test": "run-s build test", "test:circleci": "yarn test:coverage", "run_mocha": "mocha --require source-map-support/register --require make-promises-safe lib/test/**/*_test.js --bail --exit", "test:coverage": "nyc npm run test --all && yarn coverage:report:lcov", "coverage:report:lcov": "nyc report --reporter=text-lcov > coverage/lcov.info", - "clean": "shx rm -rf lib", + "clean": "shx rm -rf lib src/artifacts", "lint": "tslint --project . --exclude **/src/generated_contract_wrappers/**/*" }, + "config": { + "contracts_v2": "Exchange", + "postpublish": { + "assets": [] + } + }, "repository": { "type": "git", - "url": "https://github.com/0xProject/0x.js.git" + "url": "https://github.com/0xProject/0x-monorepo" }, "license": "Apache-2.0", - "bugs": { - "url": "https://github.com/0xProject/0x.js/issues" - }, - "homepage": "https://github.com/0xProject/0x.js/packages/pipeline/README.md", "devDependencies": { "@0xproject/tslint-config": "^1.0.7", - "@types/command-line-args": "^4.0.2", - "@types/dotenv": "^4.0.2", - "@types/glob": "^5.0.33", - "@types/lodash": "4.14.104", - "@types/node": "^8.0.53", - "@types/pg": "^7.4.1", - "@types/request": "^2.0.13", - "shx": "^0.2.2", "tslint": "5.11.0", "typescript": "3.0.1" }, "dependencies": { - "0x.js": "^0.38.3", - "@0xproject/connect": "^0.6.14", - "airtable": "^0.5.2", - "command-line-args": "^4.0.7", - "dotenv": "^4.0.0", - "lodash": "^4.17.4", - "node-redshift": "^0.1.5", - "pg": "^7.4.1", - "queue": "^4.4.2", - "request": "^2.83.0", - "web3": "^0.20.0" + "@0xproject/contract-wrappers": "^1.0.1", + "@0xproject/subproviders": "^2.0.2", + "@0xproject/utils": "^1.0.8", + "@types/ramda": "^0.25.38", + "axios": "^0.18.0", + "ethereum-types": "^1.0.6", + "ramda": "^0.25.0" } } -- cgit v1.2.3 From 75d3f24835fc68a758cfb44c6bc05095c3e87ad3 Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Wed, 19 Sep 2018 11:56:41 -0700 Subject: Add tests for etherscan events --- packages/pipeline/package.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'packages/pipeline/package.json') diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index b4ed33d8d..cdcd5ed18 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -31,7 +31,12 @@ "devDependencies": { "@0xproject/tslint-config": "^1.0.7", "tslint": "5.11.0", - "typescript": "3.0.1" + "typescript": "3.0.1", + "mocha": "^5.2.0", + "chai": "^4.1.2", + "chai-as-promised": "^7.1.1", + "chai-bignumber": "^2.0.2", + "dirty-chai": "^2.0.1" }, "dependencies": { "@0xproject/contract-wrappers": "^1.0.1", -- cgit v1.2.3 From 4e6919a6e58d76766098262664c78f92d888337d Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Wed, 19 Sep 2018 17:30:34 -0700 Subject: Add typeorm and begin inserting database records --- packages/pipeline/package.json | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'packages/pipeline/package.json') diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index cdcd5ed18..d81ea2668 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -30,13 +30,13 @@ "license": "Apache-2.0", "devDependencies": { "@0xproject/tslint-config": "^1.0.7", - "tslint": "5.11.0", - "typescript": "3.0.1", - "mocha": "^5.2.0", "chai": "^4.1.2", "chai-as-promised": "^7.1.1", "chai-bignumber": "^2.0.2", - "dirty-chai": "^2.0.1" + "dirty-chai": "^2.0.1", + "mocha": "^5.2.0", + "tslint": "5.11.0", + "typescript": "3.0.1" }, "dependencies": { "@0xproject/contract-wrappers": "^1.0.1", @@ -45,6 +45,9 @@ "@types/ramda": "^0.25.38", "axios": "^0.18.0", "ethereum-types": "^1.0.6", - "ramda": "^0.25.0" + "ramda": "^0.25.0", + "reflect-metadata": "^0.1.12", + "sqlite3": "^4.0.2", + "typeorm": "^0.2.7" } } -- cgit v1.2.3 From 9e9104578c8526ff48ecdda8b87d61ccb3d66a2d Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Thu, 20 Sep 2018 17:25:48 -0700 Subject: Add support for decoding asset data --- packages/pipeline/package.json | 2 ++ 1 file changed, 2 insertions(+) (limited to 'packages/pipeline/package.json') diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index d81ea2668..6670938f2 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -40,7 +40,9 @@ }, "dependencies": { "@0xproject/contract-wrappers": "^1.0.1", + "@0xproject/order-utils": "^1.0.2", "@0xproject/subproviders": "^2.0.2", + "@0xproject/types": "^1.0.1", "@0xproject/utils": "^1.0.8", "@types/ramda": "^0.25.38", "axios": "^0.18.0", -- cgit v1.2.3 From 9ab55ccec0478e65f9d605aa2da7ed1fa13e01ac Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Wed, 26 Sep 2018 16:42:48 -0700 Subject: Add preliminary support for scraping orders from SRA endpoints (no pagination, only RR support for now) --- packages/pipeline/package.json | 1 + 1 file changed, 1 insertion(+) (limited to 'packages/pipeline/package.json') diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index 6670938f2..c01b7f448 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -39,6 +39,7 @@ "typescript": "3.0.1" }, "dependencies": { + "@0xproject/connect": "^2.0.4", "@0xproject/contract-wrappers": "^1.0.1", "@0xproject/order-utils": "^1.0.2", "@0xproject/subproviders": "^2.0.2", -- cgit v1.2.3 From 6c00dd6f3900e322e5b6d5fe09b35fdafc1bfb2f Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Tue, 16 Oct 2018 14:59:59 -0700 Subject: Rebase and update to use new modularized artifacts --- packages/pipeline/package.json | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) (limited to 'packages/pipeline/package.json') diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index c01b7f448..be23bfe2c 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -4,21 +4,17 @@ "private": true, "description": "Data pipeline for offline analysis", "scripts": { - "build": "yarn pre_build && tsc -b", - "pre_build": "run-s update_artifacts copy_artifacts", - "copy_artifacts": "copyfiles -u 2 './src/artifacts/**/*.json' ./lib/src/artifacts", - "update_artifacts": "for i in ${npm_package_config_contracts_v2}; do copyfiles -u 4 ../migrations/artifacts/2.0.0/$i.json src/artifacts; done;", + "build": "yarn tsc -b", "test": "yarn run_mocha", "rebuild_and_test": "run-s build test", "test:circleci": "yarn test:coverage", "run_mocha": "mocha --require source-map-support/register --require make-promises-safe lib/test/**/*_test.js --bail --exit", "test:coverage": "nyc npm run test --all && yarn coverage:report:lcov", "coverage:report:lcov": "nyc report --reporter=text-lcov > coverage/lcov.info", - "clean": "shx rm -rf lib src/artifacts", - "lint": "tslint --project . --exclude **/src/generated_contract_wrappers/**/*" + "clean": "shx rm -rf lib", + "lint": "tslint --project ." }, "config": { - "contracts_v2": "Exchange", "postpublish": { "assets": [] } @@ -39,6 +35,7 @@ "typescript": "3.0.1" }, "dependencies": { + "@0xproject/contract-artifacts": "^1.0.0", "@0xproject/connect": "^2.0.4", "@0xproject/contract-wrappers": "^1.0.1", "@0xproject/order-utils": "^1.0.2", -- cgit v1.2.3 From 91e7485ecc4e780ef454b642a52c3d93faa95af1 Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Wed, 17 Oct 2018 16:44:07 -0700 Subject: Update to use ContractWrappers + Infura instead of Etherscan --- packages/pipeline/package.json | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) (limited to 'packages/pipeline/package.json') diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index be23bfe2c..0071fab2c 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -1,5 +1,5 @@ { - "name": "@0xproject/pipeline", + "name": "@0x/pipeline", "version": "0.0.1", "private": true, "description": "Data pipeline for offline analysis", @@ -25,7 +25,8 @@ }, "license": "Apache-2.0", "devDependencies": { - "@0xproject/tslint-config": "^1.0.7", + "@types/ramda": "^0.25.38", + "@0x/tslint-config": "^1.0.9", "chai": "^4.1.2", "chai-as-promised": "^7.1.1", "chai-bignumber": "^2.0.2", @@ -35,14 +36,15 @@ "typescript": "3.0.1" }, "dependencies": { - "@0xproject/contract-artifacts": "^1.0.0", - "@0xproject/connect": "^2.0.4", - "@0xproject/contract-wrappers": "^1.0.1", - "@0xproject/order-utils": "^1.0.2", - "@0xproject/subproviders": "^2.0.2", - "@0xproject/types": "^1.0.1", - "@0xproject/utils": "^1.0.8", - "@types/ramda": "^0.25.38", + "@0x/dev-utils": "^1.0.13", + "@0x/contract-artifacts": "^1.0.1", + "@0x/connect": "^3.0.2", + "@0x/contract-wrappers": "^3.0.0", + "@0x/order-utils": "^2.0.0", + "@0x/subproviders": "^2.1.0", + "@0x/types": "^1.2.0", + "@0x/utils": "^2.0.3", + "@0x/web3-wrapper": "^3.1.0", "axios": "^0.18.0", "ethereum-types": "^1.0.6", "ramda": "^0.25.0", -- cgit v1.2.3 From 3a19faa5ff8a66395c5eb8d1400338cefaa21187 Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Thu, 18 Oct 2018 17:44:48 -0700 Subject: Update script to work with existing v1 pipeline data --- packages/pipeline/package.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'packages/pipeline/package.json') diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index 0071fab2c..47033bf33 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -25,8 +25,8 @@ }, "license": "Apache-2.0", "devDependencies": { - "@types/ramda": "^0.25.38", "@0x/tslint-config": "^1.0.9", + "@types/ramda": "^0.25.38", "chai": "^4.1.2", "chai-as-promised": "^7.1.1", "chai-bignumber": "^2.0.2", @@ -36,10 +36,10 @@ "typescript": "3.0.1" }, "dependencies": { - "@0x/dev-utils": "^1.0.13", - "@0x/contract-artifacts": "^1.0.1", "@0x/connect": "^3.0.2", + "@0x/contract-artifacts": "^1.0.1", "@0x/contract-wrappers": "^3.0.0", + "@0x/dev-utils": "^1.0.13", "@0x/order-utils": "^2.0.0", "@0x/subproviders": "^2.1.0", "@0x/types": "^1.2.0", @@ -47,6 +47,7 @@ "@0x/web3-wrapper": "^3.1.0", "axios": "^0.18.0", "ethereum-types": "^1.0.6", + "pg": "^7.5.0", "ramda": "^0.25.0", "reflect-metadata": "^0.1.12", "sqlite3": "^4.0.2", -- cgit v1.2.3 From 5e22a862b77d4947361342972c28a8552d13018e Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Wed, 24 Oct 2018 18:34:20 -0700 Subject: Implement fetching and parsing relayer info --- packages/pipeline/package.json | 1 + 1 file changed, 1 insertion(+) (limited to 'packages/pipeline/package.json') diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index 47033bf33..253da3f19 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -27,6 +27,7 @@ "devDependencies": { "@0x/tslint-config": "^1.0.9", "@types/ramda": "^0.25.38", + "@types/axios": "^0.14.0", "chai": "^4.1.2", "chai-as-promised": "^7.1.1", "chai-bignumber": "^2.0.2", -- cgit v1.2.3 From 21d4807206040e945aa4d4b224e20503873f8e50 Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Tue, 6 Nov 2018 10:40:07 -0800 Subject: Add build:ci to pipeline package.json --- packages/pipeline/package.json | 1 + 1 file changed, 1 insertion(+) (limited to 'packages/pipeline/package.json') diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index 253da3f19..3380a970a 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -5,6 +5,7 @@ "description": "Data pipeline for offline analysis", "scripts": { "build": "yarn tsc -b", + "build:ci": "yarn build", "test": "yarn run_mocha", "rebuild_and_test": "run-s build test", "test:circleci": "yarn test:coverage", -- cgit v1.2.3 From 329c68f610843ebded9ca31fc9cd6f3eed744a8e Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Mon, 12 Nov 2018 16:40:20 -0800 Subject: Configure TypeORM for migrations. Add new package.json scripts. --- packages/pipeline/package.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'packages/pipeline/package.json') diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index 3380a970a..367296e05 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -13,7 +13,10 @@ "test:coverage": "nyc npm run test --all && yarn coverage:report:lcov", "coverage:report:lcov": "nyc report --reporter=text-lcov > coverage/lcov.info", "clean": "shx rm -rf lib", - "lint": "tslint --project ." + "lint": "tslint --project .", + "migrate:run": "yarn typeorm migration:run --config ./lib/src/ormconfig", + "migrate:revert": "yarn typeorm migration:revert --config ./lib/src/ormconfig", + "migrate:create": "yarn typeorm migration:create --config ./lib/src/ormconfig --dir migrations" }, "config": { "postpublish": { -- cgit v1.2.3 From 688d277b30b287f66f0dbd49f2a23cab8b256219 Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Mon, 12 Nov 2018 17:36:33 -0800 Subject: Configure linter with --format stylish and fix linter errors --- packages/pipeline/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'packages/pipeline/package.json') diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index 367296e05..59cf24dc6 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -13,7 +13,7 @@ "test:coverage": "nyc npm run test --all && yarn coverage:report:lcov", "coverage:report:lcov": "nyc report --reporter=text-lcov > coverage/lcov.info", "clean": "shx rm -rf lib", - "lint": "tslint --project .", + "lint": "tslint --project . --format stylish --exclude ./migrations", "migrate:run": "yarn typeorm migration:run --config ./lib/src/ormconfig", "migrate:revert": "yarn typeorm migration:revert --config ./lib/src/ormconfig", "migrate:create": "yarn typeorm migration:create --config ./lib/src/ormconfig --dir migrations" -- cgit v1.2.3 From 96134003e1aa1903bdf051e39e7e244cc8684043 Mon Sep 17 00:00:00 2001 From: Jake Ellowitz Date: Tue, 13 Nov 2018 09:57:02 -0800 Subject: Pointing entities to raw schema Fix linter issues --- packages/pipeline/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'packages/pipeline/package.json') diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index 59cf24dc6..dcd19d150 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -13,7 +13,7 @@ "test:coverage": "nyc npm run test --all && yarn coverage:report:lcov", "coverage:report:lcov": "nyc report --reporter=text-lcov > coverage/lcov.info", "clean": "shx rm -rf lib", - "lint": "tslint --project . --format stylish --exclude ./migrations", + "lint": "tslint --project . --format stylish --exclude ./migrations/**/*", "migrate:run": "yarn typeorm migration:run --config ./lib/src/ormconfig", "migrate:revert": "yarn typeorm migration:revert --config ./lib/src/ormconfig", "migrate:create": "yarn typeorm migration:create --config ./lib/src/ormconfig --dir migrations" -- cgit v1.2.3 From 9986717671fe8e14c2168f7479bdaffe406bedc0 Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Mon, 19 Nov 2018 18:38:11 -0800 Subject: Add script for pulling missing block data --- packages/pipeline/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'packages/pipeline/package.json') diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index dcd19d150..a57fbf5bc 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -30,8 +30,8 @@ "license": "Apache-2.0", "devDependencies": { "@0x/tslint-config": "^1.0.9", - "@types/ramda": "^0.25.38", "@types/axios": "^0.14.0", + "@types/ramda": "^0.25.38", "chai": "^4.1.2", "chai-as-promised": "^7.1.1", "chai-bignumber": "^2.0.2", @@ -50,6 +50,7 @@ "@0x/types": "^1.2.0", "@0x/utils": "^2.0.3", "@0x/web3-wrapper": "^3.1.0", + "async-parallel": "^1.2.3", "axios": "^0.18.0", "ethereum-types": "^1.0.6", "pg": "^7.5.0", -- cgit v1.2.3 From 3d211c415b58a67f84332ff512bf9372cac5a3ac Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Wed, 28 Nov 2018 13:21:04 -0800 Subject: Introduce framework for running basic tests for entities (#1344) * Introduce framework for running basic tests for entities * Add pipeline tests to CircleCI config * Make pipeline tests more configurable and fix CircleCI config * Add coverage dir to pipeline package * Add basic tests for all exchange event entities * Add tests for remaining entities * Create separate test scripts in package.json and add new info to README * Update db_setup.ts to revert migrations even if you are using docker * Automatically pull the postgres image if needed * Add comment about why NumberToBigIntTransformer is needed --- packages/pipeline/package.json | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'packages/pipeline/package.json') diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index a57fbf5bc..9831517fb 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -7,10 +7,13 @@ "build": "yarn tsc -b", "build:ci": "yarn build", "test": "yarn run_mocha", - "rebuild_and_test": "run-s build test", + "rebuild_and_test": "run-s build test:all", + "test:db": "yarn run_mocha:db", + "test:all": "run-s test test:db", "test:circleci": "yarn test:coverage", - "run_mocha": "mocha --require source-map-support/register --require make-promises-safe lib/test/**/*_test.js --bail --exit", - "test:coverage": "nyc npm run test --all && yarn coverage:report:lcov", + "run_mocha": "mocha --require source-map-support/register --require make-promises-safe 'lib/test/!(entities)/**/*_test.js' --bail --exit", + "run_mocha:db": "mocha --require source-map-support/register --require make-promises-safe lib/test/db_global_hooks.js 'lib/test/entities/*_test.js' --bail --exit --timeout 60000", + "test:coverage": "nyc npm run test:all --all && yarn coverage:report:lcov", "coverage:report:lcov": "nyc report --reporter=text-lcov > coverage/lcov.info", "clean": "shx rm -rf lib", "lint": "tslint --project . --format stylish --exclude ./migrations/**/*", @@ -50,8 +53,10 @@ "@0x/types": "^1.2.0", "@0x/utils": "^2.0.3", "@0x/web3-wrapper": "^3.1.0", + "@types/dockerode": "^2.5.9", "async-parallel": "^1.2.3", "axios": "^0.18.0", + "dockerode": "^2.5.7", "ethereum-types": "^1.0.6", "pg": "^7.5.0", "ramda": "^0.25.0", -- cgit v1.2.3 From 8c21a700bae0c751f7f9ca47f9a47628a4478911 Mon Sep 17 00:00:00 2001 From: Xianny <8582774+xianny@users.noreply.github.com> Date: Tue, 4 Dec 2018 13:36:18 -0800 Subject: pull OHLCV records from Crypto Compare (#1349) * [WIP] pull OHLCV records from Crypto Compare * lint * refactor to pull logic out of script and into modules * add entity test for ohlcv_external entity * implement rate limit and chronological backfill for ohlcv * add unit tests; cleanup variable names * Fetch OHLCV pairs params from events table * better method names * fix outdated test * lint * Clean up after review * oops * fix failing test * better filtering of most recent records * fix bug when generating pairs * fix default earliest backfill date * fix bug with retrieving backfill time * prettier --- packages/pipeline/package.json | 3 +++ 1 file changed, 3 insertions(+) (limited to 'packages/pipeline/package.json') diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index 9831517fb..6af81c048 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -54,11 +54,14 @@ "@0x/utils": "^2.0.3", "@0x/web3-wrapper": "^3.1.0", "@types/dockerode": "^2.5.9", + "@types/p-limit": "^2.0.0", "async-parallel": "^1.2.3", "axios": "^0.18.0", "dockerode": "^2.5.7", "ethereum-types": "^1.0.6", + "p-limit": "^2.0.0", "pg": "^7.5.0", + "prettier": "^1.15.3", "ramda": "^0.25.0", "reflect-metadata": "^0.1.12", "sqlite3": "^4.0.2", -- cgit v1.2.3 From 00f86ca0f7871639d2b0be496f6f8c5e0d8d7ffe Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Tue, 4 Dec 2018 20:04:08 -0800 Subject: Address PR feedback --- packages/pipeline/package.json | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'packages/pipeline/package.json') diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index 6af81c048..0539618d4 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -1,6 +1,6 @@ { "name": "@0x/pipeline", - "version": "0.0.1", + "version": "1.0.0", "private": true, "description": "Data pipeline for offline analysis", "scripts": { @@ -21,11 +21,6 @@ "migrate:revert": "yarn typeorm migration:revert --config ./lib/src/ormconfig", "migrate:create": "yarn typeorm migration:create --config ./lib/src/ormconfig --dir migrations" }, - "config": { - "postpublish": { - "assets": [] - } - }, "repository": { "type": "git", "url": "https://github.com/0xProject/0x-monorepo" -- cgit v1.2.3