diff options
| author | fragosti <francesco.agosti93@gmail.com> | 2018-12-06 04:33:34 +0800 |
|---|---|---|
| committer | fragosti <francesco.agosti93@gmail.com> | 2018-12-06 04:33:34 +0800 |
| commit | f9e73d2a6f6b7c3126c0c10286c2f79c4fd2a7ac (patch) | |
| tree | e1b93ec089efa842372b885d51046c845728dd75 | |
| parent | 5c29b918df4ac8b0f7914e8da10fa1ae530ff4e8 (diff) | |
| parent | 08eb0b91b6d0f0dc90ae920a18ca5dd080bf235c (diff) | |
| download | dexon-sol-tools-f9e73d2a6f6b7c3126c0c10286c2f79c4fd2a7ac.tar dexon-sol-tools-f9e73d2a6f6b7c3126c0c10286c2f79c4fd2a7ac.tar.gz dexon-sol-tools-f9e73d2a6f6b7c3126c0c10286c2f79c4fd2a7ac.tar.bz2 dexon-sol-tools-f9e73d2a6f6b7c3126c0c10286c2f79c4fd2a7ac.tar.lz dexon-sol-tools-f9e73d2a6f6b7c3126c0c10286c2f79c4fd2a7ac.tar.xz dexon-sol-tools-f9e73d2a6f6b7c3126c0c10286c2f79c4fd2a7ac.tar.zst dexon-sol-tools-f9e73d2a6f6b7c3126c0c10286c2f79c4fd2a7ac.zip | |
Merge branch 'development' of https://github.com/0xProject/0x-monorepo into feature/website/instant-configurator
280 files changed, 6484 insertions, 607 deletions
diff --git a/.circleci/config.yml b/.circleci/config.yml index 0ab512f58..d11c7fcae 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,7 +40,10 @@ jobs: - restore_cache: keys: - repo-{{ .Environment.CIRCLE_SHA1 }} - - run: yarn wsrun test:circleci contracts + - run: yarn wsrun test:circleci @0x/contracts-multisig + - run: yarn wsrun test:circleci @0x/contracts-utils + - run: yarn wsrun test:circleci @0x/contracts-libs + - run: yarn wsrun test:circleci @0x/contracts-core test-contracts-geth: docker: - image: circleci/node:9 @@ -52,7 +55,10 @@ jobs: - repo-{{ .Environment.CIRCLE_SHA1 }} # HACK(albrow): we need to sleep 10 seconds to ensure the devnet is # initialized - - run: sleep 10 && TEST_PROVIDER=geth yarn wsrun test contracts + - run: sleep 10 && TEST_PROVIDER=geth yarn wsrun test @0x/contracts-multisig + - run: TEST_PROVIDER=geth yarn wsrun test @0x/contracts-utils + - run: TEST_PROVIDER=geth yarn wsrun test @0x/contracts-libs + - run: TEST_PROVIDER=geth yarn wsrun test @0x/contracts-core test-publish: resource_class: medium+ docker: @@ -73,6 +79,20 @@ jobs: keys: - repo-{{ .Environment.CIRCLE_SHA1 }} - run: yarn test:generate_docs:circleci + test-pipeline: + docker: + - image: circleci/node:9 + - image: postgres:11-alpine + working_directory: ~/repo + steps: + - restore_cache: + keys: + - repo-{{ .Environment.CIRCLE_SHA1 }} + - run: ZEROEX_DATA_PIPELINE_TEST_DB_URL='postgresql://postgres@localhost/postgres' yarn wsrun test:circleci @0x/pipeline + - save_cache: + key: coverage-pipeline-{{ .Environment.CIRCLE_SHA1 }} + paths: + - ~/repo/packages/pipeline/coverage/lcov.info test-rest: docker: - image: circleci/node:9 @@ -81,6 +101,7 @@ jobs: - restore_cache: keys: - repo-{{ .Environment.CIRCLE_SHA1 }} + - run: yarn wsrun test:circleci @0x/contracts-test-utils - run: yarn wsrun test:circleci @0x/abi-gen - run: yarn wsrun test:circleci @0x/assert - run: yarn wsrun test:circleci @0x/base-contract @@ -331,6 +352,9 @@ workflows: - test-contracts-geth: requires: - build + - test-pipeline: + requires: + - build - test-rest: requires: - build diff --git a/.gitignore b/.gitignore index 0e483dc0a..da7620a78 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,10 @@ pids *.seed *.pid.lock +# SQLite database files +*.db +*.sqlite + # Directory for instrumented libs generated by jscoverage/JSCover lib-cov @@ -80,12 +84,18 @@ packages/testnet-faucets/server/ # generated contract artifacts/ contracts/core/generated-artifacts/ +contracts/multisig/generated-artifacts/ +contracts/utils/generated-artifacts/ +contracts/libs/generated-artifacts/ packages/sol-cov/test/fixtures/artifacts/ packages/metacoin/artifacts/ # generated contract wrappers packages/abi-gen-wrappers/wrappers contracts/core/generated-wrappers/ +contracts/multisig/generated-wrappers/ +contracts/utils/generated-wrappers/ +contracts/libs/generated-wrappers/ packages/metacoin/src/contract_wrappers # solc-bin in sol-compiler diff --git a/.prettierignore b/.prettierignore index 9dbb83e27..43c8015fd 100644 --- a/.prettierignore +++ b/.prettierignore @@ -2,6 +2,12 @@ lib .nyc_output /contracts/core/generated-wrappers /contracts/core/generated-artifacts +/contracts/multisig/generated-wrappers +/contracts/multisig/generated-artifacts +/contracts/utils/generated-wrappers +/contracts/utils/generated-artifacts +/contracts/libs/generated-wrappers +/contracts/libs/generated-artifacts /packages/abi-gen-wrappers/src/generated-wrappers /packages/contract-artifacts/artifacts /python-packages/order_utils/src/zero_ex/contract_artifacts/artifacts diff --git a/CODEOWNERS b/CODEOWNERS index 346e42ac0..ca98ec19b 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -14,7 +14,7 @@ packages/website/ @BMillman19 @fragosti @fabioberger @steveklebanoff packages/abi-gen/ @LogvinovLeon packages/base-contract/ @LogvinovLeon packages/connect/ @fragosti -packages/contract_templates/ @LogvinovLeon +packages/abi-gen-templates/ @LogvinovLeon packages/contract-addresses/ @albrow packages/contract-artifacts/ @albrow packages/dev-utils/ @LogvinovLeon @fabioberger diff --git a/contracts/TESTING.md b/contracts/TESTING.md new file mode 100644 index 000000000..750b3c62c --- /dev/null +++ b/contracts/TESTING.md @@ -0,0 +1,48 @@ +# Contracts testing options + +## Revert stack traces + +If you want to see helpful stack traces (incl. line number, code snippet) for smart contract reverts, run the tests with: + +``` +yarn test:trace +``` + +**Note:** This currently slows down the test runs and is therefore not enabled by default. + +## Backing Ethereum node + +By default, our tests run against an in-process [Ganache](https://github.com/trufflesuite/ganache-core) instance. In order to run the tests against [Geth](https://github.com/ethereum/go-ethereum), first follow the instructions in the README for the devnet package to start the devnet Geth node. Then run: + +```bash +TEST_PROVIDER=geth yarn test +``` + +## Code coverage + +In order to see the Solidity code coverage output generated by `@0x/sol-cov`, run: + +``` +yarn test:coverage +``` + +## Gas profiler + +In order to profile the gas costs for a specific smart contract call/transaction, you can run the tests in `profiler` mode. + +**Note:** Traces emitted by ganache have incorrect gas costs so we recommend using Geth for profiling. + +``` +TEST_PROVIDER=geth yarn test:profiler +``` + +You'll see a warning that you need to explicitly enable and disable the profiler before and after the block of code you want to profile. + +```typescript +import { profiler } from './utils/profiler'; +profiler.start(); +// Some call to a smart contract +profiler.stop(); +``` + +Without explicitly starting and stopping the profiler, the profiler output will be too busy, and therefore unusable. diff --git a/contracts/core/README.md b/contracts/core/README.md index 97a2816ff..0004925c1 100644 --- a/contracts/core/README.md +++ b/contracts/core/README.md @@ -14,8 +14,6 @@ Contracts that make up and interact with version 2.0.0 of the protocol can be fo * This directory contains example implementations of contracts that interact with the protocol but are _not_ intended for use in production. Examples include [filter](https://github.com/0xProject/0x-protocol-specification/blob/master/v2/v2-specification.md#filter-contracts) contracts, a [Wallet](https://github.com/0xProject/0x-protocol-specification/blob/master/v2/v2-specification.md#wallet) contract, and a [Validator](https://github.com/0xProject/0x-protocol-specification/blob/master/v2/v2-specification.md#validator) contract, among others. * [tokens](./contracts/tokens) * This directory contains implementations of different tokens and token standards, including [wETH](https://weth.io/), ZRX, [ERC20](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-20.md), and [ERC721](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-721.md). -* [multisig](./contracts/multisig) - * This directory contains the [Gnosis MultiSigWallet](https://github.com/gnosis/MultiSigWallet) and a custom extension that adds a timelock to transactions within the MultiSigWallet. * [utils](./contracts/utils) * This directory contains libraries and utils that are shared across all of the other directories. * [test](./contracts/test) @@ -52,13 +50,13 @@ yarn install To build this package and all other monorepo packages that it depends on, run the following from the monorepo root directory: ```bash -PKG=contracts yarn build +PKG=@0x/contracts-core yarn build ``` Or continuously rebuild on change: ```bash -PKG=contracts yarn watch +PKG=@0x/contracts-core yarn watch ``` ### Clean @@ -81,49 +79,4 @@ yarn test #### Testing options -###### Revert stack traces - -If you want to see helpful stack traces (incl. line number, code snippet) for smart contract reverts, run the tests with: - -``` -yarn test:trace -``` - -**Note:** This currently slows down the test runs and is therefore not enabled by default. - -###### Backing Ethereum node - -By default, our tests run against an in-process [Ganache](https://github.com/trufflesuite/ganache-core) instance. In order to run the tests against [Geth](https://github.com/ethereum/go-ethereum), first follow the instructions in the README for the devnet package to start the devnet Geth node. Then run: - -```bash -TEST_PROVIDER=geth yarn test -``` - -###### Code coverage - -In order to see the Solidity code coverage output generated by `@0x/sol-cov`, run: - -``` -yarn test:coverage -``` - -###### Gas profiler - -In order to profile the gas costs for a specific smart contract call/transaction, you can run the tests in `profiler` mode. - -**Note:** Traces emitted by ganache have incorrect gas costs so we recommend using Geth for profiling. - -``` -TEST_PROVIDER=geth yarn test:profiler -``` - -You'll see a warning that you need to explicitly enable and disable the profiler before and after the block of code you want to profile. - -```typescript -import { profiler } from './utils/profiler'; -profiler.start(); -// Some call to a smart contract -profiler.stop(); -``` - -Without explicitly starting and stopping the profiler, the profiler output will be too busy, and therefore unusable. +Contracts testing options like coverage, profiling, revert traces or backing node choosing - are described [here](../TESTING.md). diff --git a/contracts/core/compiler.json b/contracts/core/compiler.json index a1a60c71d..7e527130a 100644 --- a/contracts/core/compiler.json +++ b/contracts/core/compiler.json @@ -40,15 +40,10 @@ "IWallet", "MixinAuthorizable", "MultiAssetProxy", - "MultiSigWallet", - "MultiSigWalletWithTimeLock", "OrderValidator", "ReentrantERC20Token", "TestAssetProxyOwner", "TestAssetProxyDispatcher", - "TestConstants", - "TestLibBytes", - "TestLibs", "TestExchangeInternals", "TestSignatureValidator", "TestStaticCallReceiver", diff --git a/contracts/core/contracts/examples/ExchangeWrapper/ExchangeWrapper.sol b/contracts/core/contracts/examples/ExchangeWrapper/ExchangeWrapper.sol index 2fa0e3c5e..ca5a64a26 100644 --- a/contracts/core/contracts/examples/ExchangeWrapper/ExchangeWrapper.sol +++ b/contracts/core/contracts/examples/ExchangeWrapper/ExchangeWrapper.sol @@ -20,7 +20,7 @@ pragma solidity 0.4.24; pragma experimental ABIEncoderV2; import "../../protocol/Exchange/interfaces/IExchange.sol"; -import "../../protocol/Exchange/libs/LibOrder.sol"; +import "@0x/contracts-libs/contracts/libs/LibOrder.sol"; contract ExchangeWrapper { diff --git a/contracts/core/contracts/examples/Wallet/Wallet.sol b/contracts/core/contracts/examples/Wallet/Wallet.sol index b75021a31..3738be841 100644 --- a/contracts/core/contracts/examples/Wallet/Wallet.sol +++ b/contracts/core/contracts/examples/Wallet/Wallet.sol @@ -19,7 +19,7 @@ pragma solidity 0.4.24; import "../../protocol/Exchange/interfaces/IWallet.sol"; -import "../../utils/LibBytes/LibBytes.sol"; +import "@0x/contracts-utils/contracts/utils/LibBytes/LibBytes.sol"; contract Wallet is diff --git a/contracts/core/contracts/examples/Whitelist/Whitelist.sol b/contracts/core/contracts/examples/Whitelist/Whitelist.sol index e4e25038c..cfcddddd3 100644 --- a/contracts/core/contracts/examples/Whitelist/Whitelist.sol +++ b/contracts/core/contracts/examples/Whitelist/Whitelist.sol @@ -20,8 +20,8 @@ pragma solidity 0.4.24; pragma experimental ABIEncoderV2; import "../../protocol/Exchange/interfaces/IExchange.sol"; -import "../../protocol/Exchange/libs/LibOrder.sol"; -import "../../utils/Ownable/Ownable.sol"; +import "@0x/contracts-libs/contracts/libs/LibOrder.sol"; +import "@0x/contracts-utils/contracts/utils/Ownable/Ownable.sol"; contract Whitelist is diff --git a/contracts/core/contracts/extensions/DutchAuction/DutchAuction.sol b/contracts/core/contracts/extensions/DutchAuction/DutchAuction.sol index abe8309cf..a40991ae7 100644 --- a/contracts/core/contracts/extensions/DutchAuction/DutchAuction.sol +++ b/contracts/core/contracts/extensions/DutchAuction/DutchAuction.sol @@ -20,10 +20,10 @@ pragma solidity 0.4.24; pragma experimental ABIEncoderV2; import "../../protocol/Exchange/interfaces/IExchange.sol"; -import "../../protocol/Exchange/libs/LibOrder.sol"; +import "@0x/contracts-libs/contracts/libs/LibOrder.sol"; import "../../tokens/ERC20Token/IERC20Token.sol"; -import "../../utils/LibBytes/LibBytes.sol"; -import "../../utils/SafeMath/SafeMath.sol"; +import "@0x/contracts-utils/contracts/utils/LibBytes/LibBytes.sol"; +import "@0x/contracts-utils/contracts/utils/SafeMath/SafeMath.sol"; contract DutchAuction is diff --git a/contracts/core/contracts/extensions/Forwarder/MixinAssets.sol b/contracts/core/contracts/extensions/Forwarder/MixinAssets.sol index 43efb5ff3..5f5f3456d 100644 --- a/contracts/core/contracts/extensions/Forwarder/MixinAssets.sol +++ b/contracts/core/contracts/extensions/Forwarder/MixinAssets.sol @@ -18,8 +18,8 @@ pragma solidity 0.4.24; -import "../../utils/LibBytes/LibBytes.sol"; -import "../../utils/Ownable/Ownable.sol"; +import "@0x/contracts-utils/contracts/utils/LibBytes/LibBytes.sol"; +import "@0x/contracts-utils/contracts/utils/Ownable/Ownable.sol"; import "../../tokens/ERC20Token/IERC20Token.sol"; import "../../tokens/ERC721Token/IERC721Token.sol"; import "./libs/LibConstants.sol"; diff --git a/contracts/core/contracts/extensions/Forwarder/MixinExchangeWrapper.sol b/contracts/core/contracts/extensions/Forwarder/MixinExchangeWrapper.sol index 4991c0ea5..210eb14c2 100644 --- a/contracts/core/contracts/extensions/Forwarder/MixinExchangeWrapper.sol +++ b/contracts/core/contracts/extensions/Forwarder/MixinExchangeWrapper.sol @@ -21,10 +21,10 @@ pragma experimental ABIEncoderV2; import "./libs/LibConstants.sol"; import "./mixins/MExchangeWrapper.sol"; -import "../../protocol/Exchange/libs/LibAbiEncoder.sol"; -import "../../protocol/Exchange/libs/LibOrder.sol"; -import "../../protocol/Exchange/libs/LibFillResults.sol"; -import "../../protocol/Exchange/libs/LibMath.sol"; +import "@0x/contracts-libs/contracts/libs/LibAbiEncoder.sol"; +import "@0x/contracts-libs/contracts/libs/LibOrder.sol"; +import "@0x/contracts-libs/contracts/libs/LibFillResults.sol"; +import "@0x/contracts-libs/contracts/libs/LibMath.sol"; contract MixinExchangeWrapper is diff --git a/contracts/core/contracts/extensions/Forwarder/MixinForwarderCore.sol b/contracts/core/contracts/extensions/Forwarder/MixinForwarderCore.sol index 54487f726..bab78d79b 100644 --- a/contracts/core/contracts/extensions/Forwarder/MixinForwarderCore.sol +++ b/contracts/core/contracts/extensions/Forwarder/MixinForwarderCore.sol @@ -24,10 +24,10 @@ import "./mixins/MWeth.sol"; import "./mixins/MAssets.sol"; import "./mixins/MExchangeWrapper.sol"; import "./interfaces/IForwarderCore.sol"; -import "../../utils/LibBytes/LibBytes.sol"; -import "../../protocol/Exchange/libs/LibOrder.sol"; -import "../../protocol/Exchange/libs/LibFillResults.sol"; -import "../../protocol/Exchange/libs/LibMath.sol"; +import "@0x/contracts-utils/contracts/utils/LibBytes/LibBytes.sol"; +import "@0x/contracts-libs/contracts/libs/LibOrder.sol"; +import "@0x/contracts-libs/contracts/libs/LibFillResults.sol"; +import "@0x/contracts-libs/contracts/libs/LibMath.sol"; contract MixinForwarderCore is diff --git a/contracts/core/contracts/extensions/Forwarder/MixinWeth.sol b/contracts/core/contracts/extensions/Forwarder/MixinWeth.sol index d2814a49b..2a281f3ae 100644 --- a/contracts/core/contracts/extensions/Forwarder/MixinWeth.sol +++ b/contracts/core/contracts/extensions/Forwarder/MixinWeth.sol @@ -18,7 +18,7 @@ pragma solidity 0.4.24; -import "../../protocol/Exchange/libs/LibMath.sol"; +import "@0x/contracts-libs/contracts/libs/LibMath.sol"; import "./libs/LibConstants.sol"; import "./mixins/MWeth.sol"; diff --git a/contracts/core/contracts/extensions/Forwarder/interfaces/IForwarderCore.sol b/contracts/core/contracts/extensions/Forwarder/interfaces/IForwarderCore.sol index 74c7da01d..eede20bb8 100644 --- a/contracts/core/contracts/extensions/Forwarder/interfaces/IForwarderCore.sol +++ b/contracts/core/contracts/extensions/Forwarder/interfaces/IForwarderCore.sol @@ -19,8 +19,8 @@ pragma solidity 0.4.24; pragma experimental ABIEncoderV2; -import "../../../protocol/Exchange/libs/LibOrder.sol"; -import "../../../protocol/Exchange/libs/LibFillResults.sol"; +import "@0x/contracts-libs/contracts/libs/LibOrder.sol"; +import "@0x/contracts-libs/contracts/libs/LibFillResults.sol"; contract IForwarderCore { diff --git a/contracts/core/contracts/extensions/Forwarder/libs/LibConstants.sol b/contracts/core/contracts/extensions/Forwarder/libs/LibConstants.sol index 704e42ce3..0f98ae595 100644 --- a/contracts/core/contracts/extensions/Forwarder/libs/LibConstants.sol +++ b/contracts/core/contracts/extensions/Forwarder/libs/LibConstants.sol @@ -18,7 +18,7 @@ pragma solidity 0.4.24; -import "../../../utils/LibBytes/LibBytes.sol"; +import "@0x/contracts-utils/contracts/utils/LibBytes/LibBytes.sol"; import "../../../protocol/Exchange/interfaces/IExchange.sol"; import "../../../tokens/EtherToken/IEtherToken.sol"; import "../../../tokens/ERC20Token/IERC20Token.sol"; diff --git a/contracts/core/contracts/extensions/Forwarder/mixins/MExchangeWrapper.sol b/contracts/core/contracts/extensions/Forwarder/mixins/MExchangeWrapper.sol index 13c26b03a..d9e71786a 100644 --- a/contracts/core/contracts/extensions/Forwarder/mixins/MExchangeWrapper.sol +++ b/contracts/core/contracts/extensions/Forwarder/mixins/MExchangeWrapper.sol @@ -19,8 +19,8 @@ pragma solidity 0.4.24; pragma experimental ABIEncoderV2; -import "../../../protocol/Exchange/libs/LibOrder.sol"; -import "../../../protocol/Exchange/libs/LibFillResults.sol"; +import "@0x/contracts-libs/contracts/libs/LibOrder.sol"; +import "@0x/contracts-libs/contracts/libs/LibFillResults.sol"; contract MExchangeWrapper { diff --git a/contracts/core/contracts/extensions/OrderValidator/OrderValidator.sol b/contracts/core/contracts/extensions/OrderValidator/OrderValidator.sol index 3385d35ef..9e9e63e9b 100644 --- a/contracts/core/contracts/extensions/OrderValidator/OrderValidator.sol +++ b/contracts/core/contracts/extensions/OrderValidator/OrderValidator.sol @@ -20,10 +20,10 @@ pragma solidity 0.4.24; pragma experimental ABIEncoderV2; import "../../protocol/Exchange/interfaces/IExchange.sol"; -import "../../protocol/Exchange/libs/LibOrder.sol"; +import "@0x/contracts-libs/contracts/libs/LibOrder.sol"; import "../../tokens/ERC20Token/IERC20Token.sol"; import "../../tokens/ERC721Token/IERC721Token.sol"; -import "../../utils/LibBytes/LibBytes.sol"; +import "@0x/contracts-utils/contracts/utils/LibBytes/LibBytes.sol"; contract OrderValidator { diff --git a/contracts/core/contracts/protocol/AssetProxy/MixinAuthorizable.sol b/contracts/core/contracts/protocol/AssetProxy/MixinAuthorizable.sol index fe9bbf848..08f9b94dc 100644 --- a/contracts/core/contracts/protocol/AssetProxy/MixinAuthorizable.sol +++ b/contracts/core/contracts/protocol/AssetProxy/MixinAuthorizable.sol @@ -18,7 +18,7 @@ pragma solidity 0.4.24; -import "../../utils/Ownable/Ownable.sol"; +import "@0x/contracts-utils/contracts/utils/Ownable/Ownable.sol"; import "./mixins/MAuthorizable.sol"; diff --git a/contracts/core/contracts/protocol/AssetProxy/interfaces/IAuthorizable.sol b/contracts/core/contracts/protocol/AssetProxy/interfaces/IAuthorizable.sol index ba1d4aa77..96ee05dee 100644 --- a/contracts/core/contracts/protocol/AssetProxy/interfaces/IAuthorizable.sol +++ b/contracts/core/contracts/protocol/AssetProxy/interfaces/IAuthorizable.sol @@ -18,7 +18,7 @@ pragma solidity 0.4.24; -import "../../../utils/Ownable/IOwnable.sol"; +import "@0x/contracts-utils/contracts/utils/Ownable/IOwnable.sol"; contract IAuthorizable is diff --git a/contracts/core/contracts/protocol/AssetProxyOwner/AssetProxyOwner.sol b/contracts/core/contracts/protocol/AssetProxyOwner/AssetProxyOwner.sol index edb788fab..bfc7b5a66 100644 --- a/contracts/core/contracts/protocol/AssetProxyOwner/AssetProxyOwner.sol +++ b/contracts/core/contracts/protocol/AssetProxyOwner/AssetProxyOwner.sol @@ -18,8 +18,8 @@ pragma solidity 0.4.24; -import "../../multisig/MultiSigWalletWithTimeLock.sol"; -import "../../utils/LibBytes/LibBytes.sol"; +import "@0x/contracts-multisig/contracts/multisig/MultiSigWalletWithTimeLock.sol"; +import "@0x/contracts-utils/contracts/utils/LibBytes/LibBytes.sol"; contract AssetProxyOwner is diff --git a/contracts/core/contracts/protocol/Exchange/Exchange.sol b/contracts/core/contracts/protocol/Exchange/Exchange.sol index ead36009f..65ca742ea 100644 --- a/contracts/core/contracts/protocol/Exchange/Exchange.sol +++ b/contracts/core/contracts/protocol/Exchange/Exchange.sol @@ -19,7 +19,7 @@ pragma solidity 0.4.24; pragma experimental ABIEncoderV2; -import "./libs/LibConstants.sol"; +import "@0x/contracts-libs/contracts/libs/LibConstants.sol"; import "./MixinExchangeCore.sol"; import "./MixinSignatureValidator.sol"; import "./MixinWrapperFunctions.sol"; diff --git a/contracts/core/contracts/protocol/Exchange/MixinAssetProxyDispatcher.sol b/contracts/core/contracts/protocol/Exchange/MixinAssetProxyDispatcher.sol index 87b09b6b3..02aeb4a13 100644 --- a/contracts/core/contracts/protocol/Exchange/MixinAssetProxyDispatcher.sol +++ b/contracts/core/contracts/protocol/Exchange/MixinAssetProxyDispatcher.sol @@ -18,7 +18,7 @@ pragma solidity 0.4.24; -import "../../utils/Ownable/Ownable.sol"; +import "@0x/contracts-utils/contracts/utils/Ownable/Ownable.sol"; import "./mixins/MAssetProxyDispatcher.sol"; import "../AssetProxy/interfaces/IAssetProxy.sol"; diff --git a/contracts/core/contracts/protocol/Exchange/MixinExchangeCore.sol b/contracts/core/contracts/protocol/Exchange/MixinExchangeCore.sol index 736dcd0b1..68d6a3897 100644 --- a/contracts/core/contracts/protocol/Exchange/MixinExchangeCore.sol +++ b/contracts/core/contracts/protocol/Exchange/MixinExchangeCore.sol @@ -19,11 +19,11 @@ pragma solidity 0.4.24; pragma experimental ABIEncoderV2; -import "../../utils/ReentrancyGuard/ReentrancyGuard.sol"; -import "./libs/LibConstants.sol"; -import "./libs/LibFillResults.sol"; -import "./libs/LibOrder.sol"; -import "./libs/LibMath.sol"; +import "@0x/contracts-utils/contracts/utils/ReentrancyGuard/ReentrancyGuard.sol"; +import "@0x/contracts-libs/contracts/libs/LibConstants.sol"; +import "@0x/contracts-libs/contracts/libs/LibFillResults.sol"; +import "@0x/contracts-libs/contracts/libs/LibOrder.sol"; +import "@0x/contracts-libs/contracts/libs/LibMath.sol"; import "./mixins/MExchangeCore.sol"; import "./mixins/MSignatureValidator.sol"; import "./mixins/MTransactions.sol"; diff --git a/contracts/core/contracts/protocol/Exchange/MixinMatchOrders.sol b/contracts/core/contracts/protocol/Exchange/MixinMatchOrders.sol index b4f6bdb26..fc6d73482 100644 --- a/contracts/core/contracts/protocol/Exchange/MixinMatchOrders.sol +++ b/contracts/core/contracts/protocol/Exchange/MixinMatchOrders.sol @@ -14,11 +14,11 @@ pragma solidity 0.4.24; pragma experimental ABIEncoderV2; -import "../../utils/ReentrancyGuard/ReentrancyGuard.sol"; -import "./libs/LibConstants.sol"; -import "./libs/LibMath.sol"; -import "./libs/LibOrder.sol"; -import "./libs/LibFillResults.sol"; +import "@0x/contracts-utils/contracts/utils/ReentrancyGuard/ReentrancyGuard.sol"; +import "@0x/contracts-libs/contracts/libs/LibConstants.sol"; +import "@0x/contracts-libs/contracts/libs/LibMath.sol"; +import "@0x/contracts-libs/contracts/libs/LibOrder.sol"; +import "@0x/contracts-libs/contracts/libs/LibFillResults.sol"; import "./mixins/MExchangeCore.sol"; import "./mixins/MMatchOrders.sol"; import "./mixins/MTransactions.sol"; diff --git a/contracts/core/contracts/protocol/Exchange/MixinSignatureValidator.sol b/contracts/core/contracts/protocol/Exchange/MixinSignatureValidator.sol index 176e28351..711535aa8 100644 --- a/contracts/core/contracts/protocol/Exchange/MixinSignatureValidator.sol +++ b/contracts/core/contracts/protocol/Exchange/MixinSignatureValidator.sol @@ -18,8 +18,8 @@ pragma solidity 0.4.24; -import "../../utils/LibBytes/LibBytes.sol"; -import "../../utils/ReentrancyGuard/ReentrancyGuard.sol"; +import "@0x/contracts-utils/contracts/utils/LibBytes/LibBytes.sol"; +import "@0x/contracts-utils/contracts/utils/ReentrancyGuard/ReentrancyGuard.sol"; import "./mixins/MSignatureValidator.sol"; import "./mixins/MTransactions.sol"; import "./interfaces/IWallet.sol"; @@ -239,18 +239,18 @@ contract MixinSignatureValidator is view returns (bool isValid) { - bytes memory calldata = abi.encodeWithSelector( + bytes memory callData = abi.encodeWithSelector( IWallet(walletAddress).isValidSignature.selector, hash, signature ); assembly { - let cdStart := add(calldata, 32) + let cdStart := add(callData, 32) let success := staticcall( gas, // forward all gas walletAddress, // address of Wallet contract cdStart, // pointer to start of input - mload(calldata), // length of input + mload(callData), // length of input cdStart, // write output over input 32 // output size is 32 bytes ) @@ -288,19 +288,19 @@ contract MixinSignatureValidator is view returns (bool isValid) { - bytes memory calldata = abi.encodeWithSelector( + bytes memory callData = abi.encodeWithSelector( IValidator(signerAddress).isValidSignature.selector, hash, signerAddress, signature ); assembly { - let cdStart := add(calldata, 32) + let cdStart := add(callData, 32) let success := staticcall( gas, // forward all gas validatorAddress, // address of Validator contract cdStart, // pointer to start of input - mload(calldata), // length of input + mload(callData), // length of input cdStart, // write output over input 32 // output size is 32 bytes ) diff --git a/contracts/core/contracts/protocol/Exchange/MixinTransactions.sol b/contracts/core/contracts/protocol/Exchange/MixinTransactions.sol index 3a76ca202..87c614382 100644 --- a/contracts/core/contracts/protocol/Exchange/MixinTransactions.sol +++ b/contracts/core/contracts/protocol/Exchange/MixinTransactions.sol @@ -17,10 +17,10 @@ */ pragma solidity 0.4.24; -import "./libs/LibExchangeErrors.sol"; +import "@0x/contracts-libs/contracts/libs/LibExchangeErrors.sol"; import "./mixins/MSignatureValidator.sol"; import "./mixins/MTransactions.sol"; -import "./libs/LibEIP712.sol"; +import "@0x/contracts-libs/contracts/libs/LibEIP712.sol"; contract MixinTransactions is diff --git a/contracts/core/contracts/protocol/Exchange/MixinWrapperFunctions.sol b/contracts/core/contracts/protocol/Exchange/MixinWrapperFunctions.sol index cddff0e5f..2d43432ff 100644 --- a/contracts/core/contracts/protocol/Exchange/MixinWrapperFunctions.sol +++ b/contracts/core/contracts/protocol/Exchange/MixinWrapperFunctions.sol @@ -19,11 +19,11 @@ pragma solidity 0.4.24; pragma experimental ABIEncoderV2; -import "../../utils/ReentrancyGuard/ReentrancyGuard.sol"; -import "./libs/LibMath.sol"; -import "./libs/LibOrder.sol"; -import "./libs/LibFillResults.sol"; -import "./libs/LibAbiEncoder.sol"; +import "@0x/contracts-utils/contracts/utils/ReentrancyGuard/ReentrancyGuard.sol"; +import "@0x/contracts-libs/contracts/libs/LibMath.sol"; +import "@0x/contracts-libs/contracts/libs/LibOrder.sol"; +import "@0x/contracts-libs/contracts/libs/LibFillResults.sol"; +import "@0x/contracts-libs/contracts/libs/LibAbiEncoder.sol"; import "./mixins/MExchangeCore.sol"; import "./mixins/MWrapperFunctions.sol"; diff --git a/contracts/core/contracts/protocol/Exchange/interfaces/IExchangeCore.sol b/contracts/core/contracts/protocol/Exchange/interfaces/IExchangeCore.sol index 9995e0385..0da73529c 100644 --- a/contracts/core/contracts/protocol/Exchange/interfaces/IExchangeCore.sol +++ b/contracts/core/contracts/protocol/Exchange/interfaces/IExchangeCore.sol @@ -19,8 +19,8 @@ pragma solidity 0.4.24; pragma experimental ABIEncoderV2; -import "../libs/LibOrder.sol"; -import "../libs/LibFillResults.sol"; +import "@0x/contracts-libs/contracts/libs/LibOrder.sol"; +import "@0x/contracts-libs/contracts/libs/LibFillResults.sol"; contract IExchangeCore { diff --git a/contracts/core/contracts/protocol/Exchange/interfaces/IMatchOrders.sol b/contracts/core/contracts/protocol/Exchange/interfaces/IMatchOrders.sol index 73447f3ae..b88e158c3 100644 --- a/contracts/core/contracts/protocol/Exchange/interfaces/IMatchOrders.sol +++ b/contracts/core/contracts/protocol/Exchange/interfaces/IMatchOrders.sol @@ -18,8 +18,8 @@ pragma solidity 0.4.24; pragma experimental ABIEncoderV2; -import "../libs/LibOrder.sol"; -import "../libs/LibFillResults.sol"; +import "@0x/contracts-libs/contracts/libs/LibOrder.sol"; +import "@0x/contracts-libs/contracts/libs/LibFillResults.sol"; contract IMatchOrders { diff --git a/contracts/core/contracts/protocol/Exchange/interfaces/IWrapperFunctions.sol b/contracts/core/contracts/protocol/Exchange/interfaces/IWrapperFunctions.sol index 56a533646..833bb7e88 100644 --- a/contracts/core/contracts/protocol/Exchange/interfaces/IWrapperFunctions.sol +++ b/contracts/core/contracts/protocol/Exchange/interfaces/IWrapperFunctions.sol @@ -19,8 +19,8 @@ pragma solidity 0.4.24; pragma experimental ABIEncoderV2; -import "../libs/LibOrder.sol"; -import "../libs/LibFillResults.sol"; +import "@0x/contracts-libs/contracts/libs/LibOrder.sol"; +import "@0x/contracts-libs/contracts/libs/LibFillResults.sol"; contract IWrapperFunctions { diff --git a/contracts/core/contracts/protocol/Exchange/mixins/MExchangeCore.sol b/contracts/core/contracts/protocol/Exchange/mixins/MExchangeCore.sol index 742499568..099bdcc33 100644 --- a/contracts/core/contracts/protocol/Exchange/mixins/MExchangeCore.sol +++ b/contracts/core/contracts/protocol/Exchange/mixins/MExchangeCore.sol @@ -19,8 +19,8 @@ pragma solidity 0.4.24; pragma experimental ABIEncoderV2; -import "../libs/LibOrder.sol"; -import "../libs/LibFillResults.sol"; +import "@0x/contracts-libs/contracts/libs/LibOrder.sol"; +import "@0x/contracts-libs/contracts/libs/LibFillResults.sol"; import "../interfaces/IExchangeCore.sol"; diff --git a/contracts/core/contracts/protocol/Exchange/mixins/MMatchOrders.sol b/contracts/core/contracts/protocol/Exchange/mixins/MMatchOrders.sol index 96fa34bc0..bb285de03 100644 --- a/contracts/core/contracts/protocol/Exchange/mixins/MMatchOrders.sol +++ b/contracts/core/contracts/protocol/Exchange/mixins/MMatchOrders.sol @@ -18,8 +18,8 @@ pragma solidity 0.4.24; pragma experimental ABIEncoderV2; -import "../libs/LibOrder.sol"; -import "../libs/LibFillResults.sol"; +import "@0x/contracts-libs/contracts/libs/LibOrder.sol"; +import "@0x/contracts-libs/contracts/libs/LibFillResults.sol"; import "../interfaces/IMatchOrders.sol"; diff --git a/contracts/core/contracts/protocol/Exchange/mixins/MWrapperFunctions.sol b/contracts/core/contracts/protocol/Exchange/mixins/MWrapperFunctions.sol index 4adfbde01..2d21bf057 100644 --- a/contracts/core/contracts/protocol/Exchange/mixins/MWrapperFunctions.sol +++ b/contracts/core/contracts/protocol/Exchange/mixins/MWrapperFunctions.sol @@ -19,8 +19,8 @@ pragma solidity 0.4.24; pragma experimental ABIEncoderV2; -import "../libs/LibOrder.sol"; -import "../libs/LibFillResults.sol"; +import "@0x/contracts-libs/contracts/libs/LibOrder.sol"; +import "@0x/contracts-libs/contracts/libs/LibFillResults.sol"; import "../interfaces/IWrapperFunctions.sol"; diff --git a/contracts/core/contracts/test/DummyERC20Token/DummyERC20Token.sol b/contracts/core/contracts/test/DummyERC20Token/DummyERC20Token.sol index 412c5d1ad..33028db0c 100644 --- a/contracts/core/contracts/test/DummyERC20Token/DummyERC20Token.sol +++ b/contracts/core/contracts/test/DummyERC20Token/DummyERC20Token.sol @@ -18,7 +18,7 @@ pragma solidity 0.4.24; -import "../../utils/Ownable/Ownable.sol"; +import "@0x/contracts-utils/contracts/utils/Ownable/Ownable.sol"; import "../../tokens/ERC20Token/MintableERC20Token.sol"; diff --git a/contracts/core/contracts/test/DummyERC721Token/DummyERC721Token.sol b/contracts/core/contracts/test/DummyERC721Token/DummyERC721Token.sol index ac9068d1d..4c978b2df 100644 --- a/contracts/core/contracts/test/DummyERC721Token/DummyERC721Token.sol +++ b/contracts/core/contracts/test/DummyERC721Token/DummyERC721Token.sol @@ -19,7 +19,7 @@ pragma solidity 0.4.24; import "../../tokens/ERC721Token/MintableERC721Token.sol"; -import "../../utils/Ownable/Ownable.sol"; +import "@0x/contracts-utils/contracts/utils/Ownable/Ownable.sol"; // solhint-disable no-empty-blocks diff --git a/contracts/core/contracts/test/ReentrantERC20Token/ReentrantERC20Token.sol b/contracts/core/contracts/test/ReentrantERC20Token/ReentrantERC20Token.sol index aa3bf9ab8..8e077e3e8 100644 --- a/contracts/core/contracts/test/ReentrantERC20Token/ReentrantERC20Token.sol +++ b/contracts/core/contracts/test/ReentrantERC20Token/ReentrantERC20Token.sol @@ -19,10 +19,10 @@ pragma solidity 0.4.24; pragma experimental ABIEncoderV2; -import "../../utils/LibBytes/LibBytes.sol"; +import "@0x/contracts-utils/contracts/utils/LibBytes/LibBytes.sol"; import "../../tokens/ERC20Token/ERC20Token.sol"; import "../../protocol/Exchange/interfaces/IExchange.sol"; -import "../../protocol/Exchange/libs/LibOrder.sol"; +import "@0x/contracts-libs/contracts/libs/LibOrder.sol"; // solhint-disable no-unused-vars diff --git a/contracts/core/contracts/tokens/ERC20Token/MintableERC20Token.sol b/contracts/core/contracts/tokens/ERC20Token/MintableERC20Token.sol index 9dc924422..58bccb5a1 100644 --- a/contracts/core/contracts/tokens/ERC20Token/MintableERC20Token.sol +++ b/contracts/core/contracts/tokens/ERC20Token/MintableERC20Token.sol @@ -18,7 +18,7 @@ pragma solidity 0.4.24; -import "../../utils/SafeMath/SafeMath.sol"; +import "@0x/contracts-utils/contracts/utils/SafeMath/SafeMath.sol"; import "./UnlimitedAllowanceERC20Token.sol"; diff --git a/contracts/core/contracts/tokens/ERC721Token/ERC721Token.sol b/contracts/core/contracts/tokens/ERC721Token/ERC721Token.sol index 530f080c0..600cee1ab 100644 --- a/contracts/core/contracts/tokens/ERC721Token/ERC721Token.sol +++ b/contracts/core/contracts/tokens/ERC721Token/ERC721Token.sol @@ -20,7 +20,7 @@ pragma solidity 0.4.24; import "./IERC721Token.sol"; import "./IERC721Receiver.sol"; -import "../../utils/SafeMath/SafeMath.sol"; +import "@0x/contracts-utils/contracts/utils/SafeMath/SafeMath.sol"; contract ERC721Token is diff --git a/contracts/core/package.json b/contracts/core/package.json index 457625791..43fa9370e 100644 --- a/contracts/core/package.json +++ b/contracts/core/package.json @@ -1,6 +1,6 @@ { "private": true, - "name": "contracts", + "name": "@0x/contracts-core", "version": "2.1.56", "engines": { "node": ">=6.12" @@ -33,20 +33,19 @@ "lint-contracts": "solhint contracts/**/**/**/**/*.sol" }, "config": { - "abis": - "generated-artifacts/@(AssetProxyOwner|DummyERC20Token|DummyERC721Receiver|DummyERC721Token|DummyMultipleReturnERC20Token|DummyNoReturnERC20Token|DutchAuction|ERC20Token|ERC20Proxy|ERC721Token|ERC721Proxy|Forwarder|Exchange|ExchangeWrapper|IAssetData|IAssetProxy|InvalidERC721Receiver|MixinAuthorizable|MultiAssetProxy|MultiSigWallet|MultiSigWalletWithTimeLock|OrderValidator|ReentrantERC20Token|TestAssetProxyOwner|TestAssetProxyDispatcher|TestConstants|TestExchangeInternals|TestLibBytes|TestLibs|TestSignatureValidator|TestStaticCallReceiver|Validator|Wallet|Whitelist|WETH9|ZRXToken).json" + "abis": "generated-artifacts/@(AssetProxyOwner|DummyERC20Token|DummyERC721Receiver|DummyERC721Token|DummyMultipleReturnERC20Token|DummyNoReturnERC20Token|DutchAuction|ERC20Token|ERC20Proxy|ERC721Token|ERC721Proxy|Forwarder|Exchange|ExchangeWrapper|IAssetData|IAssetProxy|InvalidERC721Receiver|MixinAuthorizable|MultiAssetProxy|OrderValidator|ReentrantERC20Token|TestAssetProxyOwner|TestAssetProxyDispatcher|TestConstants|TestExchangeInternals|TestLibBytes|TestSignatureValidator|TestStaticCallReceiver|Validator|Wallet|Whitelist|WETH9|ZRXToken).json" }, "repository": { "type": "git", "url": "https://github.com/0xProject/0x-monorepo.git" }, - "author": "Amir Bandeali", "license": "Apache-2.0", "bugs": { "url": "https://github.com/0xProject/0x-monorepo/issues" }, "homepage": "https://github.com/0xProject/0x-monorepo/contracts/core/README.md", "devDependencies": { + "@0x/contracts-test-utils": "^1.0.0", "@0x/abi-gen": "^1.0.17", "@0x/dev-utils": "^1.0.19", "@0x/sol-compiler": "^1.1.14", @@ -54,7 +53,6 @@ "@0x/subproviders": "^2.1.6", "@0x/tslint-config": "^1.0.10", "@types/bn.js": "^4.11.0", - "@types/ethereumjs-abi": "^0.6.0", "@types/lodash": "4.14.104", "@types/node": "*", "@types/yargs": "^10.0.0", @@ -63,6 +61,7 @@ "chai-bignumber": "^2.0.1", "dirty-chai": "^2.0.1", "make-promises-safe": "^1.1.0", + "ethereumjs-abi": "0.6.5", "mocha": "^4.1.0", "npm-run-all": "^4.1.2", "shx": "^0.2.2", @@ -75,6 +74,9 @@ "dependencies": { "@0x/base-contract": "^3.0.8", "@0x/order-utils": "^3.0.4", + "@0x/contracts-multisig": "^1.0.0", + "@0x/contracts-utils": "^1.0.0", + "@0x/contracts-libs": "^1.0.0", "@0x/types": "^1.3.0", "@0x/typescript-typings": "^3.0.4", "@0x/utils": "^2.0.6", @@ -82,10 +84,7 @@ "@types/js-combinatorics": "^0.5.29", "bn.js": "^4.11.8", "ethereum-types": "^1.1.2", - "ethereumjs-abi": "0.6.5", "ethereumjs-util": "^5.1.1", - "ethers": "~4.0.4", - "js-combinatorics": "^0.5.3", "lodash": "^4.17.5" }, "publishConfig": { diff --git a/contracts/core/src/artifacts/index.ts b/contracts/core/src/artifacts/index.ts index 8a8c5f4d4..d578c36fe 100644 --- a/contracts/core/src/artifacts/index.ts +++ b/contracts/core/src/artifacts/index.ts @@ -21,16 +21,11 @@ import * as IValidator from '../../generated-artifacts/IValidator.json'; import * as IWallet from '../../generated-artifacts/IWallet.json'; import * as MixinAuthorizable from '../../generated-artifacts/MixinAuthorizable.json'; import * as MultiAssetProxy from '../../generated-artifacts/MultiAssetProxy.json'; -import * as MultiSigWallet from '../../generated-artifacts/MultiSigWallet.json'; -import * as MultiSigWalletWithTimeLock from '../../generated-artifacts/MultiSigWalletWithTimeLock.json'; import * as OrderValidator from '../../generated-artifacts/OrderValidator.json'; import * as ReentrantERC20Token from '../../generated-artifacts/ReentrantERC20Token.json'; import * as TestAssetProxyDispatcher from '../../generated-artifacts/TestAssetProxyDispatcher.json'; import * as TestAssetProxyOwner from '../../generated-artifacts/TestAssetProxyOwner.json'; -import * as TestConstants from '../../generated-artifacts/TestConstants.json'; import * as TestExchangeInternals from '../../generated-artifacts/TestExchangeInternals.json'; -import * as TestLibBytes from '../../generated-artifacts/TestLibBytes.json'; -import * as TestLibs from '../../generated-artifacts/TestLibs.json'; import * as TestSignatureValidator from '../../generated-artifacts/TestSignatureValidator.json'; import * as TestStaticCallReceiver from '../../generated-artifacts/TestStaticCallReceiver.json'; import * as Validator from '../../generated-artifacts/Validator.json'; @@ -61,16 +56,11 @@ export const artifacts = { InvalidERC721Receiver: InvalidERC721Receiver as ContractArtifact, MixinAuthorizable: MixinAuthorizable as ContractArtifact, MultiAssetProxy: MultiAssetProxy as ContractArtifact, - MultiSigWallet: MultiSigWallet as ContractArtifact, - MultiSigWalletWithTimeLock: MultiSigWalletWithTimeLock as ContractArtifact, OrderValidator: OrderValidator as ContractArtifact, ReentrantERC20Token: ReentrantERC20Token as ContractArtifact, TestAssetProxyDispatcher: TestAssetProxyDispatcher as ContractArtifact, TestAssetProxyOwner: TestAssetProxyOwner as ContractArtifact, - TestConstants: TestConstants as ContractArtifact, TestExchangeInternals: TestExchangeInternals as ContractArtifact, - TestLibBytes: TestLibBytes as ContractArtifact, - TestLibs: TestLibs as ContractArtifact, TestSignatureValidator: TestSignatureValidator as ContractArtifact, TestStaticCallReceiver: TestStaticCallReceiver as ContractArtifact, Validator: Validator as ContractArtifact, diff --git a/contracts/core/src/wrappers/index.ts b/contracts/core/src/wrappers/index.ts index e9e3f4e79..ed9d8ef47 100644 --- a/contracts/core/src/wrappers/index.ts +++ b/contracts/core/src/wrappers/index.ts @@ -16,16 +16,11 @@ export * from '../../generated-wrappers/i_asset_data'; export * from '../../generated-wrappers/i_asset_proxy'; export * from '../../generated-wrappers/invalid_erc721_receiver'; export * from '../../generated-wrappers/mixin_authorizable'; -export * from '../../generated-wrappers/multi_sig_wallet'; -export * from '../../generated-wrappers/multi_sig_wallet_with_time_lock'; export * from '../../generated-wrappers/order_validator'; export * from '../../generated-wrappers/reentrant_erc20_token'; export * from '../../generated-wrappers/test_asset_proxy_dispatcher'; export * from '../../generated-wrappers/test_asset_proxy_owner'; -export * from '../../generated-wrappers/test_constants'; export * from '../../generated-wrappers/test_exchange_internals'; -export * from '../../generated-wrappers/test_lib_bytes'; -export * from '../../generated-wrappers/test_libs'; export * from '../../generated-wrappers/test_signature_validator'; export * from '../../generated-wrappers/test_static_call_receiver'; export * from '../../generated-wrappers/validator'; diff --git a/contracts/core/test/asset_proxy/authorizable.ts b/contracts/core/test/asset_proxy/authorizable.ts index e21af9b81..853d18be0 100644 --- a/contracts/core/test/asset_proxy/authorizable.ts +++ b/contracts/core/test/asset_proxy/authorizable.ts @@ -1,3 +1,11 @@ +import { + chaiSetup, + constants, + expectTransactionFailedAsync, + provider, + txDefaults, + web3Wrapper, +} from '@0x/contracts-test-utils'; import { BlockchainLifecycle } from '@0x/dev-utils'; import { RevertReason } from '@0x/types'; import { BigNumber } from '@0x/utils'; @@ -6,10 +14,6 @@ import * as _ from 'lodash'; import { MixinAuthorizableContract } from '../../generated-wrappers/mixin_authorizable'; import { artifacts } from '../../src/artifacts'; -import { expectTransactionFailedAsync } from '../utils/assertions'; -import { chaiSetup } from '../utils/chai_setup'; -import { constants } from '../utils/constants'; -import { provider, txDefaults, web3Wrapper } from '../utils/web3_wrapper'; chaiSetup.configure(); const expect = chai.expect; diff --git a/contracts/core/test/asset_proxy/proxies.ts b/contracts/core/test/asset_proxy/proxies.ts index 8fa1e602a..2527b0fbf 100644 --- a/contracts/core/test/asset_proxy/proxies.ts +++ b/contracts/core/test/asset_proxy/proxies.ts @@ -1,3 +1,13 @@ +import { + chaiSetup, + constants, + expectTransactionFailedAsync, + expectTransactionFailedWithoutReasonAsync, + LogDecoder, + provider, + txDefaults, + web3Wrapper, +} from '@0x/contracts-test-utils'; import { BlockchainLifecycle } from '@0x/dev-utils'; import { assetDataUtils } from '@0x/order-utils'; import { RevertReason } from '@0x/types'; @@ -16,13 +26,8 @@ import { IAssetDataContract } from '../../generated-wrappers/i_asset_data'; import { IAssetProxyContract } from '../../generated-wrappers/i_asset_proxy'; import { MultiAssetProxyContract } from '../../generated-wrappers/multi_asset_proxy'; import { artifacts } from '../../src/artifacts'; -import { expectTransactionFailedAsync, expectTransactionFailedWithoutReasonAsync } from '../utils/assertions'; -import { chaiSetup } from '../utils/chai_setup'; -import { constants } from '../utils/constants'; import { ERC20Wrapper } from '../utils/erc20_wrapper'; import { ERC721Wrapper } from '../utils/erc721_wrapper'; -import { LogDecoder } from '../utils/log_decoder'; -import { provider, txDefaults, web3Wrapper } from '../utils/web3_wrapper'; chaiSetup.configure(); const expect = chai.expect; @@ -553,7 +558,7 @@ describe('Asset Transfer Proxies', () => { erc721Receiver.address, amount, ); - const logDecoder = new LogDecoder(web3Wrapper); + const logDecoder = new LogDecoder(web3Wrapper, artifacts); const tx = await logDecoder.getTxWithDecodedLogsAsync( await web3Wrapper.sendTransactionAsync({ to: erc721Proxy.address, diff --git a/contracts/core/test/exchange/core.ts b/contracts/core/test/exchange/core.ts index 9159b0d8f..fd6b9ee6b 100644 --- a/contracts/core/test/exchange/core.ts +++ b/contracts/core/test/exchange/core.ts @@ -1,3 +1,16 @@ +import { + chaiSetup, + constants, + ERC20BalancesByOwner, + expectTransactionFailedAsync, + getLatestBlockTimestampAsync, + increaseTimeAndMineBlockAsync, + OrderFactory, + OrderStatus, + provider, + txDefaults, + web3Wrapper, +} from '@0x/contracts-test-utils'; import { BlockchainLifecycle } from '@0x/dev-utils'; import { assetDataUtils, orderHashUtils } from '@0x/order-utils'; import { RevertReason, SignatureType, SignedOrder } from '@0x/types'; @@ -19,16 +32,9 @@ import { MultiAssetProxyContract } from '../../generated-wrappers/multi_asset_pr import { ReentrantERC20TokenContract } from '../../generated-wrappers/reentrant_erc20_token'; import { TestStaticCallReceiverContract } from '../../generated-wrappers/test_static_call_receiver'; import { artifacts } from '../../src/artifacts'; -import { expectTransactionFailedAsync } from '../utils/assertions'; -import { getLatestBlockTimestampAsync, increaseTimeAndMineBlockAsync } from '../utils/block_timestamp'; -import { chaiSetup } from '../utils/chai_setup'; -import { constants } from '../utils/constants'; import { ERC20Wrapper } from '../utils/erc20_wrapper'; import { ERC721Wrapper } from '../utils/erc721_wrapper'; import { ExchangeWrapper } from '../utils/exchange_wrapper'; -import { OrderFactory } from '../utils/order_factory'; -import { ERC20BalancesByOwner, OrderStatus } from '../utils/types'; -import { provider, txDefaults, web3Wrapper } from '../utils/web3_wrapper'; chaiSetup.configure(); const expect = chai.expect; diff --git a/contracts/core/test/exchange/dispatcher.ts b/contracts/core/test/exchange/dispatcher.ts index 3d3aa42c2..9bc5cbcce 100644 --- a/contracts/core/test/exchange/dispatcher.ts +++ b/contracts/core/test/exchange/dispatcher.ts @@ -1,3 +1,12 @@ +import { + chaiSetup, + constants, + expectTransactionFailedAsync, + LogDecoder, + provider, + txDefaults, + web3Wrapper, +} from '@0x/contracts-test-utils'; import { BlockchainLifecycle } from '@0x/dev-utils'; import { assetDataUtils } from '@0x/order-utils'; import { AssetProxyId, RevertReason } from '@0x/types'; @@ -14,13 +23,8 @@ import { TestAssetProxyDispatcherContract, } from '../../generated-wrappers/test_asset_proxy_dispatcher'; import { artifacts } from '../../src/artifacts'; -import { expectTransactionFailedAsync } from '../utils/assertions'; -import { chaiSetup } from '../utils/chai_setup'; -import { constants } from '../utils/constants'; import { ERC20Wrapper } from '../utils/erc20_wrapper'; import { ERC721Wrapper } from '../utils/erc721_wrapper'; -import { LogDecoder } from '../utils/log_decoder'; -import { provider, txDefaults, web3Wrapper } from '../utils/web3_wrapper'; chaiSetup.configure(); const expect = chai.expect; @@ -145,7 +149,7 @@ describe('AssetProxyDispatcher', () => { }); it('should log an event with correct arguments when an asset proxy is registered', async () => { - const logDecoder = new LogDecoder(web3Wrapper); + const logDecoder = new LogDecoder(web3Wrapper, artifacts); const txReceipt = await logDecoder.getTxWithDecodedLogsAsync( await assetProxyDispatcher.registerAssetProxy.sendTransactionAsync(erc20Proxy.address, { from: owner }), ); diff --git a/contracts/core/test/exchange/fill_order.ts b/contracts/core/test/exchange/fill_order.ts index 37efaad2b..2bdbe4855 100644 --- a/contracts/core/test/exchange/fill_order.ts +++ b/contracts/core/test/exchange/fill_order.ts @@ -1,23 +1,25 @@ -import { BlockchainLifecycle } from '@0x/dev-utils'; -import * as _ from 'lodash'; - -import { chaiSetup } from '../utils/chai_setup'; -import { - FillOrderCombinatorialUtils, - fillOrderCombinatorialUtilsFactoryAsync, -} from '../utils/fill_order_combinatorial_utils'; import { AllowanceAmountScenario, AssetDataScenario, BalanceAmountScenario, + chaiSetup, ExpirationTimeSecondsScenario, FeeRecipientAddressScenario, FillScenario, OrderAssetAmountScenario, + provider, TakerAssetFillAmountScenario, TakerScenario, -} from '../utils/types'; -import { provider, txDefaults, web3Wrapper } from '../utils/web3_wrapper'; + txDefaults, + web3Wrapper, +} from '@0x/contracts-test-utils'; +import { BlockchainLifecycle } from '@0x/dev-utils'; +import * as _ from 'lodash'; + +import { + FillOrderCombinatorialUtils, + fillOrderCombinatorialUtilsFactoryAsync, +} from '../utils/fill_order_combinatorial_utils'; chaiSetup.configure(); const blockchainLifecycle = new BlockchainLifecycle(web3Wrapper); diff --git a/contracts/core/test/exchange/internal.ts b/contracts/core/test/exchange/internal.ts index 109be29c6..972f5efb6 100644 --- a/contracts/core/test/exchange/internal.ts +++ b/contracts/core/test/exchange/internal.ts @@ -1,3 +1,15 @@ +import { + bytes32Values, + chaiSetup, + constants, + FillResults, + getRevertReasonOrErrorMessageForSendTransactionAsync, + provider, + testCombinatoriallyWithReferenceFuncAsync, + txDefaults, + uint256Values, + web3Wrapper, +} from '@0x/contracts-test-utils'; import { BlockchainLifecycle } from '@0x/dev-utils'; import { Order, RevertReason, SignedOrder } from '@0x/types'; import { BigNumber } from '@0x/utils'; @@ -6,12 +18,6 @@ import * as _ from 'lodash'; import { TestExchangeInternalsContract } from '../../generated-wrappers/test_exchange_internals'; import { artifacts } from '../../src/artifacts'; -import { getRevertReasonOrErrorMessageForSendTransactionAsync } from '../utils/assertions'; -import { chaiSetup } from '../utils/chai_setup'; -import { bytes32Values, testCombinatoriallyWithReferenceFuncAsync, uint256Values } from '../utils/combinatorial_utils'; -import { constants } from '../utils/constants'; -import { FillResults } from '../utils/types'; -import { provider, txDefaults, web3Wrapper } from '../utils/web3_wrapper'; chaiSetup.configure(); const expect = chai.expect; diff --git a/contracts/core/test/exchange/match_orders.ts b/contracts/core/test/exchange/match_orders.ts index eea9992d9..0e841b359 100644 --- a/contracts/core/test/exchange/match_orders.ts +++ b/contracts/core/test/exchange/match_orders.ts @@ -1,3 +1,14 @@ +import { + chaiSetup, + constants, + ERC20BalancesByOwner, + ERC721TokenIdsByOwner, + expectTransactionFailedAsync, + OrderFactory, + provider, + txDefaults, + web3Wrapper, +} from '@0x/contracts-test-utils'; import { BlockchainLifecycle } from '@0x/dev-utils'; import { assetDataUtils } from '@0x/order-utils'; import { RevertReason } from '@0x/types'; @@ -14,16 +25,10 @@ import { ExchangeContract } from '../../generated-wrappers/exchange'; import { ReentrantERC20TokenContract } from '../../generated-wrappers/reentrant_erc20_token'; import { TestExchangeInternalsContract } from '../../generated-wrappers/test_exchange_internals'; import { artifacts } from '../../src/artifacts'; -import { expectTransactionFailedAsync } from '../utils/assertions'; -import { chaiSetup } from '../utils/chai_setup'; -import { constants } from '../utils/constants'; import { ERC20Wrapper } from '../utils/erc20_wrapper'; import { ERC721Wrapper } from '../utils/erc721_wrapper'; import { ExchangeWrapper } from '../utils/exchange_wrapper'; import { MatchOrderTester } from '../utils/match_order_tester'; -import { OrderFactory } from '../utils/order_factory'; -import { ERC20BalancesByOwner, ERC721TokenIdsByOwner } from '../utils/types'; -import { provider, txDefaults, web3Wrapper } from '../utils/web3_wrapper'; const blockchainLifecycle = new BlockchainLifecycle(web3Wrapper); chaiSetup.configure(); diff --git a/contracts/core/test/exchange/signature_validator.ts b/contracts/core/test/exchange/signature_validator.ts index 756c72766..b84a488a1 100644 --- a/contracts/core/test/exchange/signature_validator.ts +++ b/contracts/core/test/exchange/signature_validator.ts @@ -1,3 +1,14 @@ +import { + addressUtils, + chaiSetup, + constants, + expectContractCallFailedAsync, + LogDecoder, + OrderFactory, + provider, + txDefaults, + web3Wrapper, +} from '@0x/contracts-test-utils'; import { BlockchainLifecycle } from '@0x/dev-utils'; import { assetDataUtils, orderHashUtils, signatureUtils } from '@0x/order-utils'; import { RevertReason, SignatureType, SignedOrder } from '@0x/types'; @@ -13,13 +24,6 @@ import { TestStaticCallReceiverContract } from '../../generated-wrappers/test_st import { ValidatorContract } from '../../generated-wrappers/validator'; import { WalletContract } from '../../generated-wrappers/wallet'; import { artifacts } from '../../src/artifacts'; -import { addressUtils } from '../utils/address_utils'; -import { expectContractCallFailedAsync } from '../utils/assertions'; -import { chaiSetup } from '../utils/chai_setup'; -import { constants } from '../utils/constants'; -import { LogDecoder } from '../utils/log_decoder'; -import { OrderFactory } from '../utils/order_factory'; -import { provider, txDefaults, web3Wrapper } from '../utils/web3_wrapper'; chaiSetup.configure(); const expect = chai.expect; @@ -73,7 +77,7 @@ describe('MixinSignatureValidator', () => { provider, txDefaults, ); - signatureValidatorLogDecoder = new LogDecoder(web3Wrapper); + signatureValidatorLogDecoder = new LogDecoder(web3Wrapper, artifacts); await web3Wrapper.awaitTransactionSuccessAsync( await signatureValidator.setSignatureValidatorApproval.sendTransactionAsync(testValidator.address, true, { from: signerAddress, diff --git a/contracts/core/test/exchange/transactions.ts b/contracts/core/test/exchange/transactions.ts index 1b5eef295..c4086d9be 100644 --- a/contracts/core/test/exchange/transactions.ts +++ b/contracts/core/test/exchange/transactions.ts @@ -1,3 +1,16 @@ +import { + chaiSetup, + constants, + ERC20BalancesByOwner, + expectTransactionFailedAsync, + OrderFactory, + orderUtils, + provider, + SignedTransaction, + TransactionFactory, + txDefaults, + web3Wrapper, +} from '@0x/contracts-test-utils'; import { BlockchainLifecycle } from '@0x/dev-utils'; import { assetDataUtils, generatePseudoRandomSalt } from '@0x/order-utils'; import { OrderWithoutExchangeAddress, RevertReason, SignedOrder } from '@0x/types'; @@ -11,16 +24,8 @@ import { ExchangeContract } from '../../generated-wrappers/exchange'; import { ExchangeWrapperContract } from '../../generated-wrappers/exchange_wrapper'; import { WhitelistContract } from '../../generated-wrappers/whitelist'; import { artifacts } from '../../src/artifacts'; -import { expectTransactionFailedAsync } from '../utils/assertions'; -import { chaiSetup } from '../utils/chai_setup'; -import { constants } from '../utils/constants'; import { ERC20Wrapper } from '../utils/erc20_wrapper'; import { ExchangeWrapper } from '../utils/exchange_wrapper'; -import { OrderFactory } from '../utils/order_factory'; -import { orderUtils } from '../utils/order_utils'; -import { TransactionFactory } from '../utils/transaction_factory'; -import { ERC20BalancesByOwner, SignedTransaction } from '../utils/types'; -import { provider, txDefaults, web3Wrapper } from '../utils/web3_wrapper'; chaiSetup.configure(); const expect = chai.expect; diff --git a/contracts/core/test/exchange/wrapper.ts b/contracts/core/test/exchange/wrapper.ts index 6b660aac5..17cb7a3bb 100644 --- a/contracts/core/test/exchange/wrapper.ts +++ b/contracts/core/test/exchange/wrapper.ts @@ -1,3 +1,16 @@ +import { + chaiSetup, + constants, + ERC20BalancesByOwner, + expectTransactionFailedAsync, + getLatestBlockTimestampAsync, + increaseTimeAndMineBlockAsync, + OrderFactory, + OrderStatus, + provider, + txDefaults, + web3Wrapper, +} from '@0x/contracts-test-utils'; import { BlockchainLifecycle } from '@0x/dev-utils'; import { assetDataUtils, orderHashUtils } from '@0x/order-utils'; import { RevertReason, SignedOrder } from '@0x/types'; @@ -13,16 +26,9 @@ import { ERC721ProxyContract } from '../../generated-wrappers/erc721_proxy'; import { ExchangeContract } from '../../generated-wrappers/exchange'; import { ReentrantERC20TokenContract } from '../../generated-wrappers/reentrant_erc20_token'; import { artifacts } from '../../src/artifacts'; -import { expectTransactionFailedAsync } from '../utils/assertions'; -import { getLatestBlockTimestampAsync, increaseTimeAndMineBlockAsync } from '../utils/block_timestamp'; -import { chaiSetup } from '../utils/chai_setup'; -import { constants } from '../utils/constants'; import { ERC20Wrapper } from '../utils/erc20_wrapper'; import { ERC721Wrapper } from '../utils/erc721_wrapper'; import { ExchangeWrapper } from '../utils/exchange_wrapper'; -import { OrderFactory } from '../utils/order_factory'; -import { ERC20BalancesByOwner, OrderStatus } from '../utils/types'; -import { provider, txDefaults, web3Wrapper } from '../utils/web3_wrapper'; chaiSetup.configure(); const expect = chai.expect; diff --git a/contracts/core/test/extensions/dutch_auction.ts b/contracts/core/test/extensions/dutch_auction.ts index c133d8c60..54e6092d7 100644 --- a/contracts/core/test/extensions/dutch_auction.ts +++ b/contracts/core/test/extensions/dutch_auction.ts @@ -1,3 +1,15 @@ +import { + chaiSetup, + constants, + ContractName, + ERC20BalancesByOwner, + expectTransactionFailedAsync, + getLatestBlockTimestampAsync, + OrderFactory, + provider, + txDefaults, + web3Wrapper, +} from '@0x/contracts-test-utils'; import { BlockchainLifecycle } from '@0x/dev-utils'; import { assetDataUtils, generatePseudoRandomSalt } from '@0x/order-utils'; import { RevertReason, SignedOrder } from '@0x/types'; @@ -14,16 +26,9 @@ import { DutchAuctionContract } from '../../generated-wrappers/dutch_auction'; import { ExchangeContract } from '../../generated-wrappers/exchange'; import { WETH9Contract } from '../../generated-wrappers/weth9'; import { artifacts } from '../../src/artifacts'; -import { expectTransactionFailedAsync } from '../utils/assertions'; -import { getLatestBlockTimestampAsync } from '../utils/block_timestamp'; -import { chaiSetup } from '../utils/chai_setup'; -import { constants } from '../utils/constants'; import { ERC20Wrapper } from '../utils/erc20_wrapper'; import { ERC721Wrapper } from '../utils/erc721_wrapper'; import { ExchangeWrapper } from '../utils/exchange_wrapper'; -import { OrderFactory } from '../utils/order_factory'; -import { ContractName, ERC20BalancesByOwner } from '../utils/types'; -import { provider, txDefaults, web3Wrapper } from '../utils/web3_wrapper'; chaiSetup.configure(); const expect = chai.expect; @@ -57,16 +62,6 @@ describe(ContractName.DutchAuction, () => { let erc721MakerAssetIds: BigNumber[]; const tenMinutesInSeconds = 10 * 60; - async function increaseTimeAsync(): Promise<void> { - const timestampBefore = await getLatestBlockTimestampAsync(); - await web3Wrapper.increaseTimeAsync(5); - const timestampAfter = await getLatestBlockTimestampAsync(); - // HACK send some transactions when a time increase isn't supported - if (timestampAfter === timestampBefore) { - await web3Wrapper.sendTransactionAsync({ to: makerAddress, from: makerAddress, value: new BigNumber(1) }); - } - } - function extendMakerAssetData(makerAssetData: string, beginTimeSeconds: BigNumber, beginAmount: BigNumber): string { return ethUtil.bufferToHex( Buffer.concat([ @@ -271,34 +266,6 @@ describe(ContractName.DutchAuction, () => { erc20Balances[takerAddress][wethContract.address].minus(beforeAuctionDetails.currentAmount), ); }); - it('should have valid getAuctionDetails at some block in the future', async () => { - let auctionDetails = await dutchAuctionContract.getAuctionDetails.callAsync(sellOrder); - const beforeAmount = auctionDetails.currentAmount; - await increaseTimeAsync(); - auctionDetails = await dutchAuctionContract.getAuctionDetails.callAsync(sellOrder); - const currentAmount = auctionDetails.currentAmount; - expect(beforeAmount).to.be.bignumber.greaterThan(currentAmount); - - buyOrder = await buyerOrderFactory.newSignedOrderAsync({ - makerAssetAmount: currentAmount, - }); - const txHash = await dutchAuctionContract.matchOrders.sendTransactionAsync( - buyOrder, - sellOrder, - buyOrder.signature, - sellOrder.signature, - { - from: takerAddress, - // HACK geth seems to miscalculate the gas required intermittently - gas: 400000, - }, - ); - await web3Wrapper.awaitTransactionSuccessAsync(txHash); - const newBalances = await erc20Wrapper.getBalancesAsync(); - expect(newBalances[makerAddress][wethContract.address]).to.be.bignumber.equal( - erc20Balances[makerAddress][wethContract.address].plus(currentAmount), - ); - }); it('maker fees on sellOrder are paid to the fee receipient', async () => { sellOrder = await sellerOrderFactory.newSignedOrderAsync({ makerFee: new BigNumber(1), @@ -370,7 +337,6 @@ describe(ContractName.DutchAuction, () => { ); }); it('cannot be filled for less than the current price', async () => { - await increaseTimeAsync(); buyOrder = await buyerOrderFactory.newSignedOrderAsync({ makerAssetAmount: sellOrder.takerAssetAmount, }); diff --git a/contracts/core/test/extensions/forwarder.ts b/contracts/core/test/extensions/forwarder.ts index c006be0fe..44ad4d6ff 100644 --- a/contracts/core/test/extensions/forwarder.ts +++ b/contracts/core/test/extensions/forwarder.ts @@ -1,3 +1,16 @@ +import { + chaiSetup, + constants, + ContractName, + ERC20BalancesByOwner, + expectContractCreationFailedAsync, + expectTransactionFailedAsync, + OrderFactory, + provider, + sendTransactionResult, + txDefaults, + web3Wrapper, +} from '@0x/contracts-test-utils'; import { BlockchainLifecycle } from '@0x/dev-utils'; import { assetDataUtils } from '@0x/order-utils'; import { RevertReason, SignedOrder } from '@0x/types'; @@ -12,20 +25,10 @@ import { ExchangeContract } from '../../generated-wrappers/exchange'; import { ForwarderContract } from '../../generated-wrappers/forwarder'; import { WETH9Contract } from '../../generated-wrappers/weth9'; import { artifacts } from '../../src/artifacts'; -import { - expectContractCreationFailedAsync, - expectTransactionFailedAsync, - sendTransactionResult, -} from '../utils/assertions'; -import { chaiSetup } from '../utils/chai_setup'; -import { constants } from '../utils/constants'; import { ERC20Wrapper } from '../utils/erc20_wrapper'; import { ERC721Wrapper } from '../utils/erc721_wrapper'; import { ExchangeWrapper } from '../utils/exchange_wrapper'; import { ForwarderWrapper } from '../utils/forwarder_wrapper'; -import { OrderFactory } from '../utils/order_factory'; -import { ContractName, ERC20BalancesByOwner } from '../utils/types'; -import { provider, txDefaults, web3Wrapper } from '../utils/web3_wrapper'; chaiSetup.configure(); const expect = chai.expect; diff --git a/contracts/core/test/extensions/order_validator.ts b/contracts/core/test/extensions/order_validator.ts index 37d7c4c5a..3dbe76f6e 100644 --- a/contracts/core/test/extensions/order_validator.ts +++ b/contracts/core/test/extensions/order_validator.ts @@ -1,3 +1,12 @@ +import { + chaiSetup, + constants, + OrderFactory, + OrderStatus, + provider, + txDefaults, + web3Wrapper, +} from '@0x/contracts-test-utils'; import { BlockchainLifecycle } from '@0x/dev-utils'; import { assetDataUtils, orderHashUtils } from '@0x/order-utils'; import { SignedOrder } from '@0x/types'; @@ -12,14 +21,9 @@ import { ERC721ProxyContract } from '../../generated-wrappers/erc721_proxy'; import { ExchangeContract } from '../../generated-wrappers/exchange'; import { OrderValidatorContract } from '../../generated-wrappers/order_validator'; import { artifacts } from '../../src/artifacts'; -import { chaiSetup } from '../utils/chai_setup'; -import { constants } from '../utils/constants'; import { ERC20Wrapper } from '../utils/erc20_wrapper'; import { ERC721Wrapper } from '../utils/erc721_wrapper'; import { ExchangeWrapper } from '../utils/exchange_wrapper'; -import { OrderFactory } from '../utils/order_factory'; -import { OrderStatus } from '../utils/types'; -import { provider, txDefaults, web3Wrapper } from '../utils/web3_wrapper'; chaiSetup.configure(); const expect = chai.expect; diff --git a/contracts/core/test/global_hooks.ts b/contracts/core/test/global_hooks.ts index 2e9ac9e21..f8ace376a 100644 --- a/contracts/core/test/global_hooks.ts +++ b/contracts/core/test/global_hooks.ts @@ -1,8 +1,9 @@ import { env, EnvVars } from '@0x/dev-utils'; -import { coverage } from './utils/coverage'; -import { profiler } from './utils/profiler'; - +import { coverage, profiler, provider } from '@0x/contracts-test-utils'; +before('start web3 provider', () => { + provider.start(); +}); after('generate coverage report', async () => { if (env.parseBoolean(EnvVars.SolidityCoverage)) { const coverageSubprovider = coverage.getCoverageSubproviderSingleton(); @@ -12,4 +13,5 @@ after('generate coverage report', async () => { const profilerSubprovider = profiler.getProfilerSubproviderSingleton(); await profilerSubprovider.writeProfilerOutputAsync(); } + provider.stop(); }); diff --git a/contracts/core/test/multisig/asset_proxy_owner.ts b/contracts/core/test/multisig/asset_proxy_owner.ts index 087152316..daebfb7fb 100644 --- a/contracts/core/test/multisig/asset_proxy_owner.ts +++ b/contracts/core/test/multisig/asset_proxy_owner.ts @@ -1,3 +1,16 @@ +import { + chaiSetup, + constants, + expectContractCallFailedAsync, + expectContractCreationFailedAsync, + expectTransactionFailedAsync, + expectTransactionFailedWithoutReasonAsync, + increaseTimeAndMineBlockAsync, + provider, + sendTransactionResult, + txDefaults, + web3Wrapper, +} from '@0x/contracts-test-utils'; import { BlockchainLifecycle } from '@0x/dev-utils'; import { RevertReason } from '@0x/types'; import { BigNumber } from '@0x/utils'; @@ -14,18 +27,7 @@ import { import { MixinAuthorizableContract } from '../../generated-wrappers/mixin_authorizable'; import { TestAssetProxyOwnerContract } from '../../generated-wrappers/test_asset_proxy_owner'; import { artifacts } from '../../src/artifacts'; -import { - expectContractCallFailedAsync, - expectContractCreationFailedAsync, - expectTransactionFailedAsync, - expectTransactionFailedWithoutReasonAsync, - sendTransactionResult, -} from '../utils/assertions'; -import { increaseTimeAndMineBlockAsync } from '../utils/block_timestamp'; -import { chaiSetup } from '../utils/chai_setup'; -import { constants } from '../utils/constants'; -import { MultiSigWrapper } from '../utils/multi_sig_wrapper'; -import { provider, txDefaults, web3Wrapper } from '../utils/web3_wrapper'; +import { AssetProxyOwnerWrapper } from '../utils/asset_proxy_owner_wrapper'; chaiSetup.configure(); const expect = chai.expect; @@ -41,7 +43,7 @@ describe('AssetProxyOwner', () => { let erc20Proxy: MixinAuthorizableContract; let erc721Proxy: MixinAuthorizableContract; let testAssetProxyOwner: TestAssetProxyOwnerContract; - let multiSigWrapper: MultiSigWrapper; + let assetProxyOwnerWrapper: AssetProxyOwnerWrapper; before(async () => { await blockchainLifecycle.startAsync(); @@ -75,7 +77,7 @@ describe('AssetProxyOwner', () => { REQUIRED_APPROVALS, SECONDS_TIME_LOCKED, ); - multiSigWrapper = new MultiSigWrapper(testAssetProxyOwner, provider); + assetProxyOwnerWrapper = new AssetProxyOwnerWrapper(testAssetProxyOwner, provider); await web3Wrapper.awaitTransactionSuccessAsync( await erc20Proxy.transferOwnership.sendTransactionAsync(testAssetProxyOwner.address, { from: initialOwner, @@ -172,7 +174,7 @@ describe('AssetProxyOwner', () => { addressToRegister, isRegistered, ); - const submitTxRes = await multiSigWrapper.submitTransactionAsync( + const submitTxRes = await assetProxyOwnerWrapper.submitTransactionAsync( testAssetProxyOwner.address, registerAssetProxyData, owners[0], @@ -181,10 +183,10 @@ describe('AssetProxyOwner', () => { const log = submitTxRes.logs[0] as LogWithDecodedArgs<AssetProxyOwnerSubmissionEventArgs>; const txId = log.args.transactionId; - await multiSigWrapper.confirmTransactionAsync(txId, owners[1]); + await assetProxyOwnerWrapper.confirmTransactionAsync(txId, owners[1]); await increaseTimeAndMineBlockAsync(SECONDS_TIME_LOCKED.toNumber()); - const executeTxRes = await multiSigWrapper.executeTransactionAsync(txId, owners[0]); + const executeTxRes = await assetProxyOwnerWrapper.executeTransactionAsync(txId, owners[0]); const registerLog = executeTxRes.logs[0] as LogWithDecodedArgs< AssetProxyOwnerAssetProxyRegistrationEventArgs >; @@ -204,7 +206,7 @@ describe('AssetProxyOwner', () => { addressToRegister, isRegistered, ); - const submitTxRes = await multiSigWrapper.submitTransactionAsync( + const submitTxRes = await assetProxyOwnerWrapper.submitTransactionAsync( testAssetProxyOwner.address, registerAssetProxyData, owners[0], @@ -212,10 +214,10 @@ describe('AssetProxyOwner', () => { const log = submitTxRes.logs[0] as LogWithDecodedArgs<AssetProxyOwnerSubmissionEventArgs>; const txId = log.args.transactionId; - await multiSigWrapper.confirmTransactionAsync(txId, owners[1]); + await assetProxyOwnerWrapper.confirmTransactionAsync(txId, owners[1]); await increaseTimeAndMineBlockAsync(SECONDS_TIME_LOCKED.toNumber()); - const executeTxRes = await multiSigWrapper.executeTransactionAsync(txId, owners[0]); + const executeTxRes = await assetProxyOwnerWrapper.executeTransactionAsync(txId, owners[0]); const failureLog = executeTxRes.logs[0] as LogWithDecodedArgs<AssetProxyOwnerExecutionFailureEventArgs>; expect(failureLog.args.transactionId).to.be.bignumber.equal(txId); @@ -237,7 +239,7 @@ describe('AssetProxyOwner', () => { addressToRegister, isRegistered, ); - const registerAssetProxySubmitRes = await multiSigWrapper.submitTransactionAsync( + const registerAssetProxySubmitRes = await assetProxyOwnerWrapper.submitTransactionAsync( testAssetProxyOwner.address, registerAssetProxyData, owners[0], @@ -247,12 +249,12 @@ describe('AssetProxyOwner', () => { >; const addAuthorizedAddressData = erc20Proxy.addAuthorizedAddress.getABIEncodedTransactionData(authorized); - const erc20AddAuthorizedAddressSubmitRes = await multiSigWrapper.submitTransactionAsync( + const erc20AddAuthorizedAddressSubmitRes = await assetProxyOwnerWrapper.submitTransactionAsync( erc20Proxy.address, addAuthorizedAddressData, owners[0], ); - const erc721AddAuthorizedAddressSubmitRes = await multiSigWrapper.submitTransactionAsync( + const erc721AddAuthorizedAddressSubmitRes = await assetProxyOwnerWrapper.submitTransactionAsync( erc721Proxy.address, addAuthorizedAddressData, owners[0], @@ -267,15 +269,15 @@ describe('AssetProxyOwner', () => { const erc20AddAuthorizedAddressTxId = erc20AddAuthorizedAddressSubmitLog.args.transactionId; const erc721AddAuthorizedAddressTxId = erc721AddAuthorizedAddressSubmitLog.args.transactionId; - await multiSigWrapper.confirmTransactionAsync(registerAssetProxyTxId, owners[1]); - await multiSigWrapper.confirmTransactionAsync(erc20AddAuthorizedAddressTxId, owners[1]); - await multiSigWrapper.confirmTransactionAsync(erc721AddAuthorizedAddressTxId, owners[1]); + await assetProxyOwnerWrapper.confirmTransactionAsync(registerAssetProxyTxId, owners[1]); + await assetProxyOwnerWrapper.confirmTransactionAsync(erc20AddAuthorizedAddressTxId, owners[1]); + await assetProxyOwnerWrapper.confirmTransactionAsync(erc721AddAuthorizedAddressTxId, owners[1]); await increaseTimeAndMineBlockAsync(SECONDS_TIME_LOCKED.toNumber()); - await multiSigWrapper.executeTransactionAsync(registerAssetProxyTxId, owners[0]); - await multiSigWrapper.executeTransactionAsync(erc20AddAuthorizedAddressTxId, owners[0], { + await assetProxyOwnerWrapper.executeTransactionAsync(registerAssetProxyTxId, owners[0]); + await assetProxyOwnerWrapper.executeTransactionAsync(erc20AddAuthorizedAddressTxId, owners[0], { gas: constants.MAX_EXECUTE_TRANSACTION_GAS, }); - await multiSigWrapper.executeTransactionAsync(erc721AddAuthorizedAddressTxId, owners[0], { + await assetProxyOwnerWrapper.executeTransactionAsync(erc721AddAuthorizedAddressTxId, owners[0], { gas: constants.MAX_EXECUTE_TRANSACTION_GAS, }); }); @@ -285,7 +287,7 @@ describe('AssetProxyOwner', () => { const notRemoveAuthorizedAddressData = erc20Proxy.addAuthorizedAddress.getABIEncodedTransactionData( authorized, ); - const submitTxRes = await multiSigWrapper.submitTransactionAsync( + const submitTxRes = await assetProxyOwnerWrapper.submitTransactionAsync( erc20Proxy.address, notRemoveAuthorizedAddressData, owners[0], @@ -303,7 +305,7 @@ describe('AssetProxyOwner', () => { authorized, erc20Index, ); - const submitTxRes = await multiSigWrapper.submitTransactionAsync( + const submitTxRes = await assetProxyOwnerWrapper.submitTransactionAsync( erc20Proxy.address, removeAuthorizedAddressAtIndexData, owners[0], @@ -321,7 +323,7 @@ describe('AssetProxyOwner', () => { authorized, erc721Index, ); - const submitTxRes = await multiSigWrapper.submitTransactionAsync( + const submitTxRes = await assetProxyOwnerWrapper.submitTransactionAsync( erc721Proxy.address, removeAuthorizedAddressAtIndexData, owners[0], @@ -341,7 +343,7 @@ describe('AssetProxyOwner', () => { authorized, erc20Index, ); - const res = await multiSigWrapper.submitTransactionAsync( + const res = await assetProxyOwnerWrapper.submitTransactionAsync( erc20Proxy.address, removeAuthorizedAddressAtIndexData, owners[0], @@ -362,7 +364,7 @@ describe('AssetProxyOwner', () => { authorized, erc721Index, ); - const res = await multiSigWrapper.submitTransactionAsync( + const res = await assetProxyOwnerWrapper.submitTransactionAsync( erc721Proxy.address, removeAuthorizedAddressAtIndexData, owners[0], @@ -370,7 +372,7 @@ describe('AssetProxyOwner', () => { const log = res.logs[0] as LogWithDecodedArgs<AssetProxyOwnerSubmissionEventArgs>; const txId = log.args.transactionId; - await multiSigWrapper.confirmTransactionAsync(txId, owners[1]); + await assetProxyOwnerWrapper.confirmTransactionAsync(txId, owners[1]); return expectTransactionFailedAsync( testAssetProxyOwner.executeRemoveAuthorizedAddressAtIndex.sendTransactionAsync(txId, { @@ -385,7 +387,7 @@ describe('AssetProxyOwner', () => { const addAuthorizedAddressData = erc20Proxy.addAuthorizedAddress.getABIEncodedTransactionData( newAuthorized, ); - const res = await multiSigWrapper.submitTransactionAsync( + const res = await assetProxyOwnerWrapper.submitTransactionAsync( erc20Proxy.address, addAuthorizedAddressData, owners[0], @@ -393,7 +395,7 @@ describe('AssetProxyOwner', () => { const log = res.logs[0] as LogWithDecodedArgs<AssetProxyOwnerSubmissionEventArgs>; const txId = log.args.transactionId; - await multiSigWrapper.confirmTransactionAsync(txId, owners[1]); + await assetProxyOwnerWrapper.confirmTransactionAsync(txId, owners[1]); return expectTransactionFailedAsync( testAssetProxyOwner.executeRemoveAuthorizedAddressAtIndex.sendTransactionAsync(txId, { @@ -411,7 +413,7 @@ describe('AssetProxyOwner', () => { authorized, erc20Index, ); - const submitRes = await multiSigWrapper.submitTransactionAsync( + const submitRes = await assetProxyOwnerWrapper.submitTransactionAsync( erc20Proxy.address, removeAuthorizedAddressAtIndexData, owners[0], @@ -419,9 +421,12 @@ describe('AssetProxyOwner', () => { const submitLog = submitRes.logs[0] as LogWithDecodedArgs<AssetProxyOwnerSubmissionEventArgs>; const txId = submitLog.args.transactionId; - await multiSigWrapper.confirmTransactionAsync(txId, owners[1]); + await assetProxyOwnerWrapper.confirmTransactionAsync(txId, owners[1]); - const execRes = await multiSigWrapper.executeRemoveAuthorizedAddressAtIndexAsync(txId, owners[0]); + const execRes = await assetProxyOwnerWrapper.executeRemoveAuthorizedAddressAtIndexAsync( + txId, + owners[0], + ); const execLog = execRes.logs[1] as LogWithDecodedArgs<AssetProxyOwnerExecutionEventArgs>; expect(execLog.args.transactionId).to.be.bignumber.equal(txId); @@ -441,7 +446,7 @@ describe('AssetProxyOwner', () => { authorized, erc20Index, ); - const submitRes = await multiSigWrapper.submitTransactionAsync( + const submitRes = await assetProxyOwnerWrapper.submitTransactionAsync( erc20Proxy.address, removeAuthorizedAddressAtIndexData, owners[0], @@ -449,9 +454,9 @@ describe('AssetProxyOwner', () => { const submitLog = submitRes.logs[0] as LogWithDecodedArgs<AssetProxyOwnerSubmissionEventArgs>; const txId = submitLog.args.transactionId; - await multiSigWrapper.confirmTransactionAsync(txId, owners[1]); + await assetProxyOwnerWrapper.confirmTransactionAsync(txId, owners[1]); - const execRes = await multiSigWrapper.executeRemoveAuthorizedAddressAtIndexAsync(txId, notOwner); + const execRes = await assetProxyOwnerWrapper.executeRemoveAuthorizedAddressAtIndexAsync(txId, notOwner); const execLog = execRes.logs[1] as LogWithDecodedArgs<AssetProxyOwnerExecutionEventArgs>; expect(execLog.args.transactionId).to.be.bignumber.equal(txId); @@ -468,7 +473,7 @@ describe('AssetProxyOwner', () => { authorized, erc20Index, ); - const submitRes = await multiSigWrapper.submitTransactionAsync( + const submitRes = await assetProxyOwnerWrapper.submitTransactionAsync( erc20Proxy.address, removeAuthorizedAddressAtIndexData, owners[0], @@ -476,9 +481,12 @@ describe('AssetProxyOwner', () => { const submitLog = submitRes.logs[0] as LogWithDecodedArgs<AssetProxyOwnerSubmissionEventArgs>; const txId = submitLog.args.transactionId; - await multiSigWrapper.confirmTransactionAsync(txId, owners[1]); + await assetProxyOwnerWrapper.confirmTransactionAsync(txId, owners[1]); - const execRes = await multiSigWrapper.executeRemoveAuthorizedAddressAtIndexAsync(txId, owners[0]); + const execRes = await assetProxyOwnerWrapper.executeRemoveAuthorizedAddressAtIndexAsync( + txId, + owners[0], + ); const execLog = execRes.logs[1] as LogWithDecodedArgs<AssetProxyOwnerExecutionEventArgs>; expect(execLog.args.transactionId).to.be.bignumber.equal(txId); @@ -495,4 +503,4 @@ describe('AssetProxyOwner', () => { }); }); }); -// tslint:enable:no-unnecessary-type-assertion +// tslint:disable-line max-file-line-count diff --git a/contracts/core/test/tokens/erc721_token.ts b/contracts/core/test/tokens/erc721_token.ts index 72407748f..3b0a5f001 100644 --- a/contracts/core/test/tokens/erc721_token.ts +++ b/contracts/core/test/tokens/erc721_token.ts @@ -1,3 +1,13 @@ +import { + chaiSetup, + constants, + expectTransactionFailedAsync, + expectTransactionFailedWithoutReasonAsync, + LogDecoder, + provider, + txDefaults, + web3Wrapper, +} from '@0x/contracts-test-utils'; import { BlockchainLifecycle } from '@0x/dev-utils'; import { RevertReason } from '@0x/types'; import { BigNumber } from '@0x/utils'; @@ -14,11 +24,6 @@ import { } from '../../generated-wrappers/dummy_erc721_token'; import { InvalidERC721ReceiverContract } from '../../generated-wrappers/invalid_erc721_receiver'; import { artifacts } from '../../src/artifacts'; -import { expectTransactionFailedAsync, expectTransactionFailedWithoutReasonAsync } from '../utils/assertions'; -import { chaiSetup } from '../utils/chai_setup'; -import { constants } from '../utils/constants'; -import { LogDecoder } from '../utils/log_decoder'; -import { provider, txDefaults, web3Wrapper } from '../utils/web3_wrapper'; chaiSetup.configure(); const expect = chai.expect; @@ -53,7 +58,7 @@ describe('ERC721Token', () => { provider, txDefaults, ); - logDecoder = new LogDecoder(web3Wrapper); + logDecoder = new LogDecoder(web3Wrapper, artifacts); await web3Wrapper.awaitTransactionSuccessAsync( await token.mint.sendTransactionAsync(owner, tokenId, { from: owner }), constants.AWAIT_TRANSACTION_MINED_MS, diff --git a/contracts/core/test/tokens/unlimited_allowance_token.ts b/contracts/core/test/tokens/unlimited_allowance_token.ts index ea5a50522..c3e4072c5 100644 --- a/contracts/core/test/tokens/unlimited_allowance_token.ts +++ b/contracts/core/test/tokens/unlimited_allowance_token.ts @@ -1,3 +1,11 @@ +import { + chaiSetup, + constants, + expectContractCallFailedAsync, + provider, + txDefaults, + web3Wrapper, +} from '@0x/contracts-test-utils'; import { BlockchainLifecycle } from '@0x/dev-utils'; import { RevertReason } from '@0x/types'; import { BigNumber } from '@0x/utils'; @@ -5,10 +13,6 @@ import * as chai from 'chai'; import { DummyERC20TokenContract } from '../../generated-wrappers/dummy_erc20_token'; import { artifacts } from '../../src/artifacts'; -import { expectContractCallFailedAsync } from '../utils/assertions'; -import { chaiSetup } from '../utils/chai_setup'; -import { constants } from '../utils/constants'; -import { provider, txDefaults, web3Wrapper } from '../utils/web3_wrapper'; chaiSetup.configure(); const expect = chai.expect; diff --git a/contracts/core/test/tokens/weth9.ts b/contracts/core/test/tokens/weth9.ts index 9a31dc3f2..225481904 100644 --- a/contracts/core/test/tokens/weth9.ts +++ b/contracts/core/test/tokens/weth9.ts @@ -1,3 +1,12 @@ +import { + chaiSetup, + constants, + expectInsufficientFundsAsync, + expectTransactionFailedWithoutReasonAsync, + provider, + txDefaults, + web3Wrapper, +} from '@0x/contracts-test-utils'; import { BlockchainLifecycle } from '@0x/dev-utils'; import { BigNumber } from '@0x/utils'; import { Web3Wrapper } from '@0x/web3-wrapper'; @@ -5,10 +14,6 @@ import * as chai from 'chai'; import { WETH9Contract } from '../../generated-wrappers/weth9'; import { artifacts } from '../../src/artifacts'; -import { expectInsufficientFundsAsync, expectTransactionFailedWithoutReasonAsync } from '../utils/assertions'; -import { chaiSetup } from '../utils/chai_setup'; -import { constants } from '../utils/constants'; -import { provider, txDefaults, web3Wrapper } from '../utils/web3_wrapper'; chaiSetup.configure(); const expect = chai.expect; diff --git a/contracts/core/test/tokens/zrx_token.ts b/contracts/core/test/tokens/zrx_token.ts index cab62c205..6bc5e164c 100644 --- a/contracts/core/test/tokens/zrx_token.ts +++ b/contracts/core/test/tokens/zrx_token.ts @@ -1,3 +1,4 @@ +import { chaiSetup, constants, provider, txDefaults, web3Wrapper } from '@0x/contracts-test-utils'; import { BlockchainLifecycle } from '@0x/dev-utils'; import { BigNumber } from '@0x/utils'; import { Web3Wrapper } from '@0x/web3-wrapper'; @@ -5,9 +6,6 @@ import * as chai from 'chai'; import { ZRXTokenContract } from '../../generated-wrappers/zrx_token'; import { artifacts } from '../../src/artifacts'; -import { chaiSetup } from '../utils/chai_setup'; -import { constants } from '../utils/constants'; -import { provider, txDefaults, web3Wrapper } from '../utils/web3_wrapper'; chaiSetup.configure(); const expect = chai.expect; diff --git a/contracts/core/test/utils/asset_proxy_owner_wrapper.ts b/contracts/core/test/utils/asset_proxy_owner_wrapper.ts new file mode 100644 index 000000000..d5aaaf519 --- /dev/null +++ b/contracts/core/test/utils/asset_proxy_owner_wrapper.ts @@ -0,0 +1,69 @@ +import { LogDecoder } from '@0x/contracts-test-utils'; +import { BigNumber } from '@0x/utils'; +import { Web3Wrapper } from '@0x/web3-wrapper'; +import { Provider, TransactionReceiptWithDecodedLogs } from 'ethereum-types'; +import * as _ from 'lodash'; + +import { AssetProxyOwnerContract } from '../../generated-wrappers/asset_proxy_owner'; +import { artifacts } from '../../src/artifacts'; + +export class AssetProxyOwnerWrapper { + private readonly _assetProxyOwner: AssetProxyOwnerContract; + private readonly _web3Wrapper: Web3Wrapper; + private readonly _logDecoder: LogDecoder; + constructor(assetproxyOwnerContract: AssetProxyOwnerContract, provider: Provider) { + this._assetProxyOwner = assetproxyOwnerContract; + this._web3Wrapper = new Web3Wrapper(provider); + this._logDecoder = new LogDecoder(this._web3Wrapper, artifacts); + } + public async submitTransactionAsync( + destination: string, + data: string, + from: string, + opts: { value?: BigNumber } = {}, + ): Promise<TransactionReceiptWithDecodedLogs> { + const value = _.isUndefined(opts.value) ? new BigNumber(0) : opts.value; + const txHash = await this._assetProxyOwner.submitTransaction.sendTransactionAsync(destination, value, data, { + from, + }); + const tx = await this._logDecoder.getTxWithDecodedLogsAsync(txHash); + return tx; + } + public async confirmTransactionAsync(txId: BigNumber, from: string): Promise<TransactionReceiptWithDecodedLogs> { + const txHash = await this._assetProxyOwner.confirmTransaction.sendTransactionAsync(txId, { from }); + const tx = await this._logDecoder.getTxWithDecodedLogsAsync(txHash); + return tx; + } + public async revokeConfirmationAsync(txId: BigNumber, from: string): Promise<TransactionReceiptWithDecodedLogs> { + const txHash = await this._assetProxyOwner.revokeConfirmation.sendTransactionAsync(txId, { from }); + const tx = await this._logDecoder.getTxWithDecodedLogsAsync(txHash); + return tx; + } + public async executeTransactionAsync( + txId: BigNumber, + from: string, + opts: { gas?: number } = {}, + ): Promise<TransactionReceiptWithDecodedLogs> { + const txHash = await this._assetProxyOwner.executeTransaction.sendTransactionAsync(txId, { + from, + gas: opts.gas, + }); + const tx = await this._logDecoder.getTxWithDecodedLogsAsync(txHash); + return tx; + } + public async executeRemoveAuthorizedAddressAtIndexAsync( + txId: BigNumber, + from: string, + ): Promise<TransactionReceiptWithDecodedLogs> { + // tslint:disable-next-line:no-unnecessary-type-assertion + const txHash = await (this + ._assetProxyOwner as AssetProxyOwnerContract).executeRemoveAuthorizedAddressAtIndex.sendTransactionAsync( + txId, + { + from, + }, + ); + const tx = await this._logDecoder.getTxWithDecodedLogsAsync(txHash); + return tx; + } +} diff --git a/contracts/core/test/utils/asset_wrapper.ts b/contracts/core/test/utils/asset_wrapper.ts index 4e7696066..e4090ad74 100644 --- a/contracts/core/test/utils/asset_wrapper.ts +++ b/contracts/core/test/utils/asset_wrapper.ts @@ -1,10 +1,9 @@ +import { AbstractAssetWrapper, constants } from '@0x/contracts-test-utils'; import { assetDataUtils } from '@0x/order-utils'; import { AssetProxyId } from '@0x/types'; import { BigNumber, errorUtils } from '@0x/utils'; import * as _ from 'lodash'; -import { AbstractAssetWrapper } from './abstract_asset_wrapper'; -import { constants } from './constants'; import { ERC20Wrapper } from './erc20_wrapper'; import { ERC721Wrapper } from './erc721_wrapper'; diff --git a/contracts/core/test/utils/erc20_wrapper.ts b/contracts/core/test/utils/erc20_wrapper.ts index c281a2abf..d6210646c 100644 --- a/contracts/core/test/utils/erc20_wrapper.ts +++ b/contracts/core/test/utils/erc20_wrapper.ts @@ -1,3 +1,4 @@ +import { constants, ERC20BalancesByOwner, txDefaults } from '@0x/contracts-test-utils'; import { assetDataUtils } from '@0x/order-utils'; import { BigNumber } from '@0x/utils'; import { Web3Wrapper } from '@0x/web3-wrapper'; @@ -8,10 +9,6 @@ import { DummyERC20TokenContract } from '../../generated-wrappers/dummy_erc20_to import { ERC20ProxyContract } from '../../generated-wrappers/erc20_proxy'; import { artifacts } from '../../src/artifacts'; -import { constants } from './constants'; -import { ERC20BalancesByOwner } from './types'; -import { txDefaults } from './web3_wrapper'; - export class ERC20Wrapper { private readonly _tokenOwnerAddresses: string[]; private readonly _contractOwnerAddress: string; diff --git a/contracts/core/test/utils/erc721_wrapper.ts b/contracts/core/test/utils/erc721_wrapper.ts index e9da553d0..b5ae34e60 100644 --- a/contracts/core/test/utils/erc721_wrapper.ts +++ b/contracts/core/test/utils/erc721_wrapper.ts @@ -1,3 +1,4 @@ +import { constants, ERC721TokenIdsByOwner, txDefaults } from '@0x/contracts-test-utils'; import { generatePseudoRandomSalt } from '@0x/order-utils'; import { BigNumber } from '@0x/utils'; import { Web3Wrapper } from '@0x/web3-wrapper'; @@ -8,10 +9,6 @@ import { DummyERC721TokenContract } from '../../generated-wrappers/dummy_erc721_ import { ERC721ProxyContract } from '../../generated-wrappers/erc721_proxy'; import { artifacts } from '../../src/artifacts'; -import { constants } from './constants'; -import { ERC721TokenIdsByOwner } from './types'; -import { txDefaults } from './web3_wrapper'; - export class ERC721Wrapper { private readonly _tokenOwnerAddresses: string[]; private readonly _contractOwnerAddress: string; diff --git a/contracts/core/test/utils/exchange_wrapper.ts b/contracts/core/test/utils/exchange_wrapper.ts index c28989d3f..2a24b880f 100644 --- a/contracts/core/test/utils/exchange_wrapper.ts +++ b/contracts/core/test/utils/exchange_wrapper.ts @@ -1,14 +1,18 @@ +import { + FillResults, + formatters, + LogDecoder, + OrderInfo, + orderUtils, + SignedTransaction, +} from '@0x/contracts-test-utils'; import { SignedOrder } from '@0x/types'; import { BigNumber } from '@0x/utils'; import { Web3Wrapper } from '@0x/web3-wrapper'; import { Provider, TransactionReceiptWithDecodedLogs } from 'ethereum-types'; import { ExchangeContract } from '../../generated-wrappers/exchange'; - -import { formatters } from './formatters'; -import { LogDecoder } from './log_decoder'; -import { orderUtils } from './order_utils'; -import { FillResults, OrderInfo, SignedTransaction } from './types'; +import { artifacts } from '../../src/artifacts'; export class ExchangeWrapper { private readonly _exchange: ExchangeContract; @@ -17,7 +21,7 @@ export class ExchangeWrapper { constructor(exchangeContract: ExchangeContract, provider: Provider) { this._exchange = exchangeContract; this._web3Wrapper = new Web3Wrapper(provider); - this._logDecoder = new LogDecoder(this._web3Wrapper); + this._logDecoder = new LogDecoder(this._web3Wrapper, artifacts); } public async fillOrderAsync( signedOrder: SignedOrder, diff --git a/contracts/core/test/utils/fill_order_combinatorial_utils.ts b/contracts/core/test/utils/fill_order_combinatorial_utils.ts index 8046771f9..5d0ea07a8 100644 --- a/contracts/core/test/utils/fill_order_combinatorial_utils.ts +++ b/contracts/core/test/utils/fill_order_combinatorial_utils.ts @@ -1,3 +1,21 @@ +import { artifacts as libsArtifacts, TestLibsContract } from '@0x/contracts-libs'; +import { + AllowanceAmountScenario, + AssetDataScenario, + BalanceAmountScenario, + chaiSetup, + constants, + expectTransactionFailedAsync, + ExpirationTimeSecondsScenario, + FeeRecipientAddressScenario, + FillScenario, + OrderAssetAmountScenario, + orderUtils, + signingUtils, + TakerAssetFillAmountScenario, + TakerScenario, + TraderStateScenario, +} from '@0x/contracts-test-utils'; import { assetDataUtils, BalanceAndProxyAllowanceLazyStore, @@ -15,33 +33,15 @@ import * as _ from 'lodash'; import 'make-promises-safe'; import { ExchangeContract, ExchangeFillEventArgs } from '../../generated-wrappers/exchange'; -import { TestLibsContract } from '../../generated-wrappers/test_libs'; import { artifacts } from '../../src/artifacts'; -import { expectTransactionFailedAsync } from './assertions'; import { AssetWrapper } from './asset_wrapper'; -import { chaiSetup } from './chai_setup'; -import { constants } from './constants'; import { ERC20Wrapper } from './erc20_wrapper'; import { ERC721Wrapper } from './erc721_wrapper'; import { ExchangeWrapper } from './exchange_wrapper'; import { OrderFactoryFromScenario } from './order_factory_from_scenario'; -import { orderUtils } from './order_utils'; -import { signingUtils } from './signing_utils'; import { SimpleAssetBalanceAndProxyAllowanceFetcher } from './simple_asset_balance_and_proxy_allowance_fetcher'; import { SimpleOrderFilledCancelledFetcher } from './simple_order_filled_cancelled_fetcher'; -import { - AllowanceAmountScenario, - AssetDataScenario, - BalanceAmountScenario, - ExpirationTimeSecondsScenario, - FeeRecipientAddressScenario, - FillScenario, - OrderAssetAmountScenario, - TakerAssetFillAmountScenario, - TakerScenario, - TraderStateScenario, -} from './types'; chaiSetup.configure(); const expect = chai.expect; @@ -131,7 +131,11 @@ export async function fillOrderCombinatorialUtilsFactoryAsync( exchangeContract.address, ); - const testLibsContract = await TestLibsContract.deployFrom0xArtifactAsync(artifacts.TestLibs, provider, txDefaults); + const testLibsContract = await TestLibsContract.deployFrom0xArtifactAsync( + libsArtifacts.TestLibs, + provider, + txDefaults, + ); const fillOrderCombinatorialUtils = new FillOrderCombinatorialUtils( orderFactory, diff --git a/contracts/core/test/utils/forwarder_wrapper.ts b/contracts/core/test/utils/forwarder_wrapper.ts index a0bfcfe1d..9c5560381 100644 --- a/contracts/core/test/utils/forwarder_wrapper.ts +++ b/contracts/core/test/utils/forwarder_wrapper.ts @@ -1,3 +1,4 @@ +import { constants, formatters, LogDecoder, MarketSellOrders } from '@0x/contracts-test-utils'; import { SignedOrder } from '@0x/types'; import { BigNumber } from '@0x/utils'; import { Web3Wrapper } from '@0x/web3-wrapper'; @@ -5,11 +6,7 @@ import { Provider, TransactionReceiptWithDecodedLogs, TxDataPayable } from 'ethe import * as _ from 'lodash'; import { ForwarderContract } from '../../generated-wrappers/forwarder'; - -import { constants } from './constants'; -import { formatters } from './formatters'; -import { LogDecoder } from './log_decoder'; -import { MarketSellOrders } from './types'; +import { artifacts } from '../../src/artifacts'; export class ForwarderWrapper { private readonly _web3Wrapper: Web3Wrapper; @@ -61,7 +58,7 @@ export class ForwarderWrapper { constructor(contractInstance: ForwarderContract, provider: Provider) { this._forwarderContract = contractInstance; this._web3Wrapper = new Web3Wrapper(provider); - this._logDecoder = new LogDecoder(this._web3Wrapper); + this._logDecoder = new LogDecoder(this._web3Wrapper, artifacts); } public async marketSellOrdersWithEthAsync( orders: SignedOrder[], diff --git a/contracts/core/test/utils/match_order_tester.ts b/contracts/core/test/utils/match_order_tester.ts index 6c2c84959..8f574704e 100644 --- a/contracts/core/test/utils/match_order_tester.ts +++ b/contracts/core/test/utils/match_order_tester.ts @@ -1,3 +1,12 @@ +import { + chaiSetup, + ERC20BalancesByOwner, + ERC721TokenIdsByOwner, + OrderInfo, + OrderStatus, + TransferAmountsByMatchOrders as TransferAmounts, + TransferAmountsLoggedByMatchOrders as LoggedTransferAmounts, +} from '@0x/contracts-test-utils'; import { assetDataUtils, orderHashUtils } from '@0x/order-utils'; import { AssetProxyId, SignedOrder } from '@0x/types'; import { BigNumber } from '@0x/utils'; @@ -6,18 +15,9 @@ import * as _ from 'lodash'; import { TransactionReceiptWithDecodedLogs } from '../../../../node_modules/ethereum-types'; -import { chaiSetup } from './chai_setup'; import { ERC20Wrapper } from './erc20_wrapper'; import { ERC721Wrapper } from './erc721_wrapper'; import { ExchangeWrapper } from './exchange_wrapper'; -import { - ERC20BalancesByOwner, - ERC721TokenIdsByOwner, - OrderInfo, - OrderStatus, - TransferAmountsByMatchOrders as TransferAmounts, - TransferAmountsLoggedByMatchOrders as LoggedTransferAmounts, -} from './types'; chaiSetup.configure(); const expect = chai.expect; @@ -270,18 +270,14 @@ export class MatchOrderTester { const leftExpectedStatus = expectedTransferAmounts.amountBoughtByLeftMaker.equals(maxAmountBoughtByLeftMaker) ? OrderStatus.FULLY_FILLED : OrderStatus.FILLABLE; - expect(leftOrderInfo.orderStatus as OrderStatus, 'Checking exchange status for left order').to.be.equal( - leftExpectedStatus, - ); + expect(leftOrderInfo.orderStatus, 'Checking exchange status for left order').to.be.equal(leftExpectedStatus); // Assert right order status const maxAmountBoughtByRightMaker = signedOrderRight.takerAssetAmount.minus(initialRightOrderFilledAmount); const rightOrderInfo: OrderInfo = await this._exchangeWrapper.getOrderInfoAsync(signedOrderRight); const rightExpectedStatus = expectedTransferAmounts.amountBoughtByRightMaker.equals(maxAmountBoughtByRightMaker) ? OrderStatus.FULLY_FILLED : OrderStatus.FILLABLE; - expect(rightOrderInfo.orderStatus as OrderStatus, 'Checking exchange status for right order').to.be.equal( - rightExpectedStatus, - ); + expect(rightOrderInfo.orderStatus, 'Checking exchange status for right order').to.be.equal(rightExpectedStatus); } /// @dev Asserts account balances after matching orders. /// @param signedOrderLeft First matched order. diff --git a/contracts/core/test/utils/order_factory_from_scenario.ts b/contracts/core/test/utils/order_factory_from_scenario.ts index 60c8606c4..1cc962020 100644 --- a/contracts/core/test/utils/order_factory_from_scenario.ts +++ b/contracts/core/test/utils/order_factory_from_scenario.ts @@ -1,19 +1,18 @@ -import { assetDataUtils, generatePseudoRandomSalt } from '@0x/order-utils'; -import { Order } from '@0x/types'; -import { BigNumber, errorUtils } from '@0x/utils'; - -import { DummyERC721TokenContract } from '../../generated-wrappers/dummy_erc721_token'; - -import { constants } from './constants'; import { AssetDataScenario, + constants, ERC721TokenIdsByOwner, ExpirationTimeSecondsScenario, FeeRecipientAddressScenario, OrderAssetAmountScenario, OrderScenario, TakerScenario, -} from './types'; +} from '@0x/contracts-test-utils'; +import { assetDataUtils, generatePseudoRandomSalt } from '@0x/order-utils'; +import { Order } from '@0x/types'; +import { BigNumber, errorUtils } from '@0x/utils'; + +import { DummyERC721TokenContract } from '../../generated-wrappers/dummy_erc721_token'; const TEN_UNITS_EIGHTEEN_DECIMALS = new BigNumber(10_000_000_000_000_000_000); const FIVE_UNITS_EIGHTEEN_DECIMALS = new BigNumber(5_000_000_000_000_000_000); diff --git a/contracts/core/tsconfig.json b/contracts/core/tsconfig.json index 23b069110..f2f3c4e97 100644 --- a/contracts/core/tsconfig.json +++ b/contracts/core/tsconfig.json @@ -28,16 +28,11 @@ "./generated-artifacts/InvalidERC721Receiver.json", "./generated-artifacts/MixinAuthorizable.json", "./generated-artifacts/MultiAssetProxy.json", - "./generated-artifacts/MultiSigWallet.json", - "./generated-artifacts/MultiSigWalletWithTimeLock.json", "./generated-artifacts/OrderValidator.json", "./generated-artifacts/ReentrantERC20Token.json", "./generated-artifacts/TestAssetProxyDispatcher.json", "./generated-artifacts/TestAssetProxyOwner.json", - "./generated-artifacts/TestConstants.json", "./generated-artifacts/TestExchangeInternals.json", - "./generated-artifacts/TestLibBytes.json", - "./generated-artifacts/TestLibs.json", "./generated-artifacts/TestSignatureValidator.json", "./generated-artifacts/TestStaticCallReceiver.json", "./generated-artifacts/Validator.json", diff --git a/contracts/libs/.solhint.json b/contracts/libs/.solhint.json new file mode 100644 index 000000000..076afe9f3 --- /dev/null +++ b/contracts/libs/.solhint.json @@ -0,0 +1,20 @@ +{ + "extends": "default", + "rules": { + "avoid-low-level-calls": false, + "avoid-tx-origin": "warn", + "bracket-align": false, + "code-complexity": false, + "const-name-snakecase": "error", + "expression-indent": "error", + "function-max-lines": false, + "func-order": "error", + "indent": ["error", 4], + "max-line-length": ["warn", 160], + "no-inline-assembly": false, + "quotes": ["error", "double"], + "separate-by-one-line-in-contract": "error", + "space-after-comma": "error", + "statement-indent": "error" + } +} diff --git a/contracts/libs/README.md b/contracts/libs/README.md new file mode 100644 index 000000000..66eedf6be --- /dev/null +++ b/contracts/libs/README.md @@ -0,0 +1,70 @@ +## Contracts libs + +Smart contracts libs used in the 0x protocol. + +## Usage + +Contracts can be found in the [contracts](./contracts) directory. The contents of this directory are broken down into the following subdirectories: + +* [libs](./contracts/protocol) + * This directory contains the libs. +* [test](./contracts/test) + * This directory contains mocks and other contracts that are used solely for testing contracts within the other directories. + +## Contributing + +We strongly recommend that the community help us make improvements and determine the future direction of the protocol. To report bugs within this package, please create an issue in this repository. + +For proposals regarding the 0x protocol's smart contract architecture, message format, or additional functionality, go to the [0x Improvement Proposals (ZEIPs)](https://github.com/0xProject/ZEIPs) repository and follow the contribution guidelines provided therein. + +Please read our [contribution guidelines](../../CONTRIBUTING.md) before getting started. + +### Install Dependencies + +If you don't have yarn workspaces enabled (Yarn < v1.0) - enable them: + +```bash +yarn config set workspaces-experimental true +``` + +Then install dependencies + +```bash +yarn install +``` + +### Build + +To build this package and all other monorepo packages that it depends on, run the following from the monorepo root directory: + +```bash +PKG=@0x/contracts-libs yarn build +``` + +Or continuously rebuild on change: + +```bash +PKG=@0x/contracts-libs yarn watch +``` + +### Clean + +```bash +yarn clean +``` + +### Lint + +```bash +yarn lint +``` + +### Run Tests + +```bash +yarn test +``` + +#### Testing options + +Contracts testing options like coverage, profiling, revert traces or backing node choosing - are described [here](../TESTING.md). diff --git a/contracts/libs/compiler.json b/contracts/libs/compiler.json new file mode 100644 index 000000000..cf7c52a73 --- /dev/null +++ b/contracts/libs/compiler.json @@ -0,0 +1,22 @@ +{ + "artifactsDir": "./generated-artifacts", + "contractsDir": "./contracts", + "compilerSettings": { + "optimizer": { + "enabled": true, + "runs": 1000000 + }, + "outputSelection": { + "*": { + "*": [ + "abi", + "evm.bytecode.object", + "evm.bytecode.sourceMap", + "evm.deployedBytecode.object", + "evm.deployedBytecode.sourceMap" + ] + } + } + }, + "contracts": ["TestLibs", "LibOrder", "LibMath", "LibFillResults", "LibAbiEncoder", "LibEIP712"] +} diff --git a/contracts/core/contracts/protocol/Exchange/libs/LibAbiEncoder.sol b/contracts/libs/contracts/libs/LibAbiEncoder.sol index 4aad37709..4aad37709 100644 --- a/contracts/core/contracts/protocol/Exchange/libs/LibAbiEncoder.sol +++ b/contracts/libs/contracts/libs/LibAbiEncoder.sol diff --git a/contracts/core/contracts/protocol/AssetProxy/libs/LibAssetProxyErrors.sol b/contracts/libs/contracts/libs/LibAssetProxyErrors.sol index 1d9a70cc1..1d9a70cc1 100644 --- a/contracts/core/contracts/protocol/AssetProxy/libs/LibAssetProxyErrors.sol +++ b/contracts/libs/contracts/libs/LibAssetProxyErrors.sol diff --git a/contracts/core/contracts/protocol/Exchange/libs/LibConstants.sol b/contracts/libs/contracts/libs/LibConstants.sol index 8d2732cd3..8d2732cd3 100644 --- a/contracts/core/contracts/protocol/Exchange/libs/LibConstants.sol +++ b/contracts/libs/contracts/libs/LibConstants.sol diff --git a/contracts/core/contracts/protocol/Exchange/libs/LibEIP712.sol b/contracts/libs/contracts/libs/LibEIP712.sol index 203edc1fd..203edc1fd 100644 --- a/contracts/core/contracts/protocol/Exchange/libs/LibEIP712.sol +++ b/contracts/libs/contracts/libs/LibEIP712.sol diff --git a/contracts/core/contracts/protocol/Exchange/libs/LibExchangeErrors.sol b/contracts/libs/contracts/libs/LibExchangeErrors.sol index a0f75bc06..a0f75bc06 100644 --- a/contracts/core/contracts/protocol/Exchange/libs/LibExchangeErrors.sol +++ b/contracts/libs/contracts/libs/LibExchangeErrors.sol diff --git a/contracts/core/contracts/protocol/Exchange/libs/LibFillResults.sol b/contracts/libs/contracts/libs/LibFillResults.sol index 659ae9a69..fbd9950bf 100644 --- a/contracts/core/contracts/protocol/Exchange/libs/LibFillResults.sol +++ b/contracts/libs/contracts/libs/LibFillResults.sol @@ -18,7 +18,7 @@ pragma solidity 0.4.24; -import "../../../utils/SafeMath/SafeMath.sol"; +import "@0x/contracts-utils/contracts/utils/SafeMath/SafeMath.sol"; contract LibFillResults is diff --git a/contracts/core/contracts/protocol/Exchange/libs/LibMath.sol b/contracts/libs/contracts/libs/LibMath.sol index c0b85ea10..b24876a9c 100644 --- a/contracts/core/contracts/protocol/Exchange/libs/LibMath.sol +++ b/contracts/libs/contracts/libs/LibMath.sol @@ -18,7 +18,7 @@ pragma solidity 0.4.24; -import "../../../utils/SafeMath/SafeMath.sol"; +import "@0x/contracts-utils/contracts/utils/SafeMath/SafeMath.sol"; contract LibMath is diff --git a/contracts/core/contracts/protocol/Exchange/libs/LibOrder.sol b/contracts/libs/contracts/libs/LibOrder.sol index 0fe7c2161..0fe7c2161 100644 --- a/contracts/core/contracts/protocol/Exchange/libs/LibOrder.sol +++ b/contracts/libs/contracts/libs/LibOrder.sol diff --git a/contracts/core/contracts/test/TestLibs/TestLibs.sol b/contracts/libs/contracts/test/TestLibs/TestLibs.sol index a10f981fc..bd5f9f9da 100644 --- a/contracts/core/contracts/test/TestLibs/TestLibs.sol +++ b/contracts/libs/contracts/test/TestLibs/TestLibs.sol @@ -19,10 +19,10 @@ pragma solidity 0.4.24; pragma experimental ABIEncoderV2; -import "../../protocol/Exchange/libs/LibMath.sol"; -import "../../protocol/Exchange/libs/LibOrder.sol"; -import "../../protocol/Exchange/libs/LibFillResults.sol"; -import "../../protocol/Exchange/libs/LibAbiEncoder.sol"; +import "../../libs/LibMath.sol"; +import "../../libs/LibOrder.sol"; +import "../../libs/LibFillResults.sol"; +import "../../libs/LibAbiEncoder.sol"; contract TestLibs is diff --git a/contracts/libs/package.json b/contracts/libs/package.json new file mode 100644 index 000000000..74288be76 --- /dev/null +++ b/contracts/libs/package.json @@ -0,0 +1,92 @@ +{ + "private": true, + "name": "@0x/contracts-libs", + "version": "1.0.0", + "engines": { + "node": ">=6.12" + }, + "description": "Smart contract libs of 0x protocol", + "main": "lib/src/index.js", + "directories": { + "test": "test" + }, + "scripts": { + "build": "yarn pre_build && tsc -b", + "build:ci": "yarn build", + "pre_build": "run-s compile generate_contract_wrappers", + "test": "yarn run_mocha", + "rebuild_and_test": "run-s build test", + "test:coverage": "SOLIDITY_COVERAGE=true run-s build run_mocha coverage:report:text coverage:report:lcov", + "test:profiler": "SOLIDITY_PROFILER=true run-s build run_mocha profiler:report:html", + "test:trace": "SOLIDITY_REVERT_TRACE=true run-s build run_mocha", + "run_mocha": + "mocha --require source-map-support/register --require make-promises-safe 'lib/test/**/*.js' --timeout 100000 --bail --exit", + "compile": "sol-compiler --contracts-dir contracts", + "clean": "shx rm -rf lib generated-artifacts generated-wrappers", + "generate_contract_wrappers": "abi-gen --abis ${npm_package_config_abis} --template ../../node_modules/@0x/abi-gen-templates/contract.handlebars --partials '../../node_modules/@0x/abi-gen-templates/partials/**/*.handlebars' --output generated-wrappers --backend ethers", + "lint": "tslint --format stylish --project . --exclude ./generated-wrappers/**/* --exclude ./generated-artifacts/**/* --exclude **/lib/**/* && yarn lint-contracts", + "coverage:report:text": "istanbul report text", + "coverage:report:html": "istanbul report html && open coverage/index.html", + "profiler:report:html": "istanbul report html && open coverage/index.html", + "coverage:report:lcov": "istanbul report lcov", + "test:circleci": "yarn test", + "lint-contracts": "solhint contracts/**/**/**/**/*.sol" + }, + "config": { + "abis": "generated-artifacts/@(LibMath|LibOrder|LibFillResults|LibAbiEncoder|TestLibs|LibEIP712).json" + }, + "repository": { + "type": "git", + "url": "https://github.com/0xProject/0x-monorepo.git" + }, + "license": "Apache-2.0", + "bugs": { + "url": "https://github.com/0xProject/0x-monorepo/issues" + }, + "homepage": "https://github.com/0xProject/0x-monorepo/contracts/libs/README.md", + "devDependencies": { + "@0x/contracts-test-utils": "^1.0.0", + "@0x/abi-gen": "^1.0.17", + "@0x/dev-utils": "^1.0.19", + "@0x/sol-compiler": "^1.1.14", + "@0x/sol-cov": "^2.1.14", + "@0x/subproviders": "^2.1.6", + "@0x/tslint-config": "^1.0.10", + "@types/bn.js": "^4.11.0", + "@types/lodash": "4.14.104", + "@types/node": "*", + "@types/yargs": "^10.0.0", + "chai": "^4.0.1", + "chai-as-promised": "^7.1.0", + "chai-bignumber": "^2.0.1", + "dirty-chai": "^2.0.1", + "make-promises-safe": "^1.1.0", + "ethereumjs-abi": "0.6.5", + "mocha": "^4.1.0", + "npm-run-all": "^4.1.2", + "shx": "^0.2.2", + "solc": "^0.4.24", + "solhint": "^1.2.1", + "tslint": "5.11.0", + "typescript": "3.0.1", + "yargs": "^10.0.3" + }, + "dependencies": { + "@0x/base-contract": "^3.0.8", + "@0x/order-utils": "^3.0.4", + "@0x/contracts-multisig": "^1.0.0", + "@0x/contracts-utils": "^1.0.0", + "@0x/types": "^1.3.0", + "@0x/typescript-typings": "^3.0.4", + "@0x/utils": "^2.0.6", + "@0x/web3-wrapper": "^3.1.6", + "@types/js-combinatorics": "^0.5.29", + "bn.js": "^4.11.8", + "ethereum-types": "^1.1.2", + "ethereumjs-util": "^5.1.1", + "lodash": "^4.17.5" + }, + "publishConfig": { + "access": "public" + } +} diff --git a/contracts/libs/src/artifacts/index.ts b/contracts/libs/src/artifacts/index.ts new file mode 100644 index 000000000..3955bbe2b --- /dev/null +++ b/contracts/libs/src/artifacts/index.ts @@ -0,0 +1,17 @@ +import { ContractArtifact } from 'ethereum-types'; + +import * as LibAbiEncoder from '../../generated-artifacts/LibAbiEncoder.json'; +import * as LibEIP721 from '../../generated-artifacts/LibEIP712.json'; +import * as LibFillResults from '../../generated-artifacts/LibFillResults.json'; +import * as LibMath from '../../generated-artifacts/LibMath.json'; +import * as LibOrder from '../../generated-artifacts/LibOrder.json'; +import * as TestLibs from '../../generated-artifacts/TestLibs.json'; + +export const artifacts = { + TestLibs: TestLibs as ContractArtifact, + LibAbiEncoder: LibAbiEncoder as ContractArtifact, + LibFillResults: LibFillResults as ContractArtifact, + LibMath: LibMath as ContractArtifact, + LibOrder: LibOrder as ContractArtifact, + LibEIP721: LibEIP721 as ContractArtifact, +}; diff --git a/contracts/libs/src/index.ts b/contracts/libs/src/index.ts new file mode 100644 index 000000000..d55f08ea2 --- /dev/null +++ b/contracts/libs/src/index.ts @@ -0,0 +1,2 @@ +export * from './artifacts'; +export * from './wrappers'; diff --git a/contracts/libs/src/wrappers/index.ts b/contracts/libs/src/wrappers/index.ts new file mode 100644 index 000000000..baaae6e34 --- /dev/null +++ b/contracts/libs/src/wrappers/index.ts @@ -0,0 +1,6 @@ +export * from '../../generated-wrappers/test_libs'; +export * from '../../generated-wrappers/lib_abi_encoder'; +export * from '../../generated-wrappers/lib_fill_results'; +export * from '../../generated-wrappers/lib_math'; +export * from '../../generated-wrappers/lib_order'; +export * from '../../generated-wrappers/lib_e_i_p712'; diff --git a/contracts/core/test/exchange/libs.ts b/contracts/libs/test/exchange/libs.ts index 503ef0e0f..44ff6a844 100644 --- a/contracts/core/test/exchange/libs.ts +++ b/contracts/libs/test/exchange/libs.ts @@ -1,17 +1,20 @@ +import { + addressUtils, + chaiSetup, + constants, + OrderFactory, + provider, + txDefaults, + web3Wrapper, +} from '@0x/contracts-test-utils'; import { BlockchainLifecycle } from '@0x/dev-utils'; import { assetDataUtils, orderHashUtils } from '@0x/order-utils'; import { SignedOrder } from '@0x/types'; import { BigNumber } from '@0x/utils'; import * as chai from 'chai'; -import { TestConstantsContract } from '../../generated-wrappers/test_constants'; import { TestLibsContract } from '../../generated-wrappers/test_libs'; import { artifacts } from '../../src/artifacts'; -import { addressUtils } from '../utils/address_utils'; -import { chaiSetup } from '../utils/chai_setup'; -import { constants } from '../utils/constants'; -import { OrderFactory } from '../utils/order_factory'; -import { provider, txDefaults, web3Wrapper } from '../utils/web3_wrapper'; chaiSetup.configure(); const expect = chai.expect; @@ -22,7 +25,6 @@ describe('Exchange libs', () => { let signedOrder: SignedOrder; let orderFactory: OrderFactory; let libs: TestLibsContract; - let testConstants: TestConstantsContract; before(async () => { await blockchainLifecycle.startAsync(); @@ -34,11 +36,6 @@ describe('Exchange libs', () => { const accounts = await web3Wrapper.getAvailableAddressesAsync(); const makerAddress = accounts[0]; libs = await TestLibsContract.deployFrom0xArtifactAsync(artifacts.TestLibs, provider, txDefaults); - testConstants = await TestConstantsContract.deployFrom0xArtifactAsync( - artifacts.TestConstants, - provider, - txDefaults, - ); const defaultOrderParams = { ...constants.STATIC_ORDER_PARAMS, @@ -58,15 +55,6 @@ describe('Exchange libs', () => { afterEach(async () => { await blockchainLifecycle.revertAsync(); }); - - describe('LibConstants', () => { - describe('ZRX_ASSET_DATA', () => { - it('should have the correct ZRX_ASSET_DATA', async () => { - const isValid = await testConstants.assertValidZrxAssetData.callAsync(); - expect(isValid).to.be.equal(true); - }); - }); - }); // Note(albrow): These tests are designed to be supplemental to the // combinatorial tests in test/exchange/internal. They test specific edge // cases that are not covered by the combinatorial tests. diff --git a/contracts/libs/test/global_hooks.ts b/contracts/libs/test/global_hooks.ts new file mode 100644 index 000000000..f8ace376a --- /dev/null +++ b/contracts/libs/test/global_hooks.ts @@ -0,0 +1,17 @@ +import { env, EnvVars } from '@0x/dev-utils'; + +import { coverage, profiler, provider } from '@0x/contracts-test-utils'; +before('start web3 provider', () => { + provider.start(); +}); +after('generate coverage report', async () => { + if (env.parseBoolean(EnvVars.SolidityCoverage)) { + const coverageSubprovider = coverage.getCoverageSubproviderSingleton(); + await coverageSubprovider.writeCoverageAsync(); + } + if (env.parseBoolean(EnvVars.SolidityProfiler)) { + const profilerSubprovider = profiler.getProfilerSubproviderSingleton(); + await profilerSubprovider.writeProfilerOutputAsync(); + } + provider.stop(); +}); diff --git a/contracts/libs/tsconfig.json b/contracts/libs/tsconfig.json new file mode 100644 index 000000000..27ca35085 --- /dev/null +++ b/contracts/libs/tsconfig.json @@ -0,0 +1,18 @@ +{ + "extends": "../../tsconfig", + "compilerOptions": { + "outDir": "lib", + "rootDir": ".", + "resolveJsonModule": true + }, + "include": ["./src/**/*", "./test/**/*", "./generated-wrappers/**/*"], + "files": [ + "./generated-artifacts/TestLibs.json", + "./generated-artifacts/LibOrder.json", + "./generated-artifacts/LibFillResults.json", + "./generated-artifacts/LibAbiEncoder.json", + "./generated-artifacts/LibEIP712.json", + "./generated-artifacts/LibMath.json" + ], + "exclude": ["./deploy/solc/solc_bin"] +} diff --git a/contracts/libs/tslint.json b/contracts/libs/tslint.json new file mode 100644 index 000000000..1bb3ac2a2 --- /dev/null +++ b/contracts/libs/tslint.json @@ -0,0 +1,6 @@ +{ + "extends": ["@0x/tslint-config"], + "rules": { + "custom-no-magic-numbers": false + } +} diff --git a/contracts/multisig/.solhint.json b/contracts/multisig/.solhint.json new file mode 100644 index 000000000..076afe9f3 --- /dev/null +++ b/contracts/multisig/.solhint.json @@ -0,0 +1,20 @@ +{ + "extends": "default", + "rules": { + "avoid-low-level-calls": false, + "avoid-tx-origin": "warn", + "bracket-align": false, + "code-complexity": false, + "const-name-snakecase": "error", + "expression-indent": "error", + "function-max-lines": false, + "func-order": "error", + "indent": ["error", 4], + "max-line-length": ["warn", 160], + "no-inline-assembly": false, + "quotes": ["error", "double"], + "separate-by-one-line-in-contract": "error", + "space-after-comma": "error", + "statement-indent": "error" + } +} diff --git a/contracts/multisig/CHANGELOG.json b/contracts/multisig/CHANGELOG.json new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/contracts/multisig/CHANGELOG.json @@ -0,0 +1 @@ +[] diff --git a/contracts/multisig/README.md b/contracts/multisig/README.md new file mode 100644 index 000000000..93db63b5b --- /dev/null +++ b/contracts/multisig/README.md @@ -0,0 +1,70 @@ +## MultisSig Contracts + +MultiSig smart contracts + +## Usage + +Contracts can be found in the [contracts](./contracts) directory. The contents of this directory are broken down into the following subdirectories: + +* [multisig](./contracts/multisig) + * This directory contains the [Gnosis MultiSigWallet](https://github.com/gnosis/MultiSigWallet) and a custom extension that adds a timelock to transactions within the MultiSigWallet. +* [test](./contracts/test) + * This directory contains mocks and other contracts that are used solely for testing contracts within the other directories. + +## Contributing + +We strongly recommend that the community help us make improvements and determine the future direction of the protocol. To report bugs within this package, please create an issue in this repository. + +For proposals regarding the 0x protocol's smart contract architecture, message format, or additional functionality, go to the [0x Improvement Proposals (ZEIPs)](https://github.com/0xProject/ZEIPs) repository and follow the contribution guidelines provided therein. + +Please read our [contribution guidelines](../../CONTRIBUTING.md) before getting started. + +### Install Dependencies + +If you don't have yarn workspaces enabled (Yarn < v1.0) - enable them: + +```bash +yarn config set workspaces-experimental true +``` + +Then install dependencies + +```bash +yarn install +``` + +### Build + +To build this package and all other monorepo packages that it depends on, run the following from the monorepo root directory: + +```bash +PKG=@0x/contracts-multisig yarn build +``` + +Or continuously rebuild on change: + +```bash +PKG=@0x/contracts-multisig yarn watch +``` + +### Clean + +```bash +yarn clean +``` + +### Lint + +```bash +yarn lint +``` + +### Run Tests + +```bash +yarn test +``` + +#### Testing options + +Contracts testing options like coverage, profiling, revert traces or backing node choosing - are described [here](../TESTING.md). diff --git a/contracts/multisig/compiler.json b/contracts/multisig/compiler.json new file mode 100644 index 000000000..5a1f689e2 --- /dev/null +++ b/contracts/multisig/compiler.json @@ -0,0 +1,22 @@ +{ + "artifactsDir": "./generated-artifacts", + "contractsDir": "./contracts", + "compilerSettings": { + "optimizer": { + "enabled": true, + "runs": 1000000 + }, + "outputSelection": { + "*": { + "*": [ + "abi", + "evm.bytecode.object", + "evm.bytecode.sourceMap", + "evm.deployedBytecode.object", + "evm.deployedBytecode.sourceMap" + ] + } + } + }, + "contracts": ["MultiSigWallet", "MultiSigWalletWithTimeLock", "TestRejectEther"] +} diff --git a/contracts/core/contracts/multisig/MultiSigWallet.sol b/contracts/multisig/contracts/multisig/MultiSigWallet.sol index 516e7391c..516e7391c 100644 --- a/contracts/core/contracts/multisig/MultiSigWallet.sol +++ b/contracts/multisig/contracts/multisig/MultiSigWallet.sol diff --git a/contracts/core/contracts/multisig/MultiSigWalletWithTimeLock.sol b/contracts/multisig/contracts/multisig/MultiSigWalletWithTimeLock.sol index 9513d3b30..9513d3b30 100644 --- a/contracts/core/contracts/multisig/MultiSigWalletWithTimeLock.sol +++ b/contracts/multisig/contracts/multisig/MultiSigWalletWithTimeLock.sol diff --git a/contracts/multisig/contracts/test/TestRejectEther/TestRejectEther.sol b/contracts/multisig/contracts/test/TestRejectEther/TestRejectEther.sol new file mode 100644 index 000000000..e523f591d --- /dev/null +++ b/contracts/multisig/contracts/test/TestRejectEther/TestRejectEther.sol @@ -0,0 +1,23 @@ +/* + + Copyright 2018 ZeroEx Intl. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/ + +pragma solidity 0.4.24; + + +// solhint-disable no-empty-blocks +contract TestRejectEther {} diff --git a/contracts/multisig/package.json b/contracts/multisig/package.json new file mode 100644 index 000000000..37d064fef --- /dev/null +++ b/contracts/multisig/package.json @@ -0,0 +1,86 @@ +{ + "private": true, + "name": "@0x/contracts-multisig", + "version": "1.0.0", + "engines": { + "node": ">=6.12" + }, + "description": "Multisig contracts used by 0x protocol", + "main": "lib/src/index.js", + "directories": { + "test": "test" + }, + "scripts": { + "build": "yarn pre_build && tsc -b", + "build:ci": "yarn build", + "pre_build": "run-s compile generate_contract_wrappers", + "test": "yarn run_mocha", + "rebuild_and_test": "run-s build test", + "test:coverage": "SOLIDITY_COVERAGE=true run-s build run_mocha coverage:report:text coverage:report:lcov", + "test:profiler": "SOLIDITY_PROFILER=true run-s build run_mocha profiler:report:html", + "test:trace": "SOLIDITY_REVERT_TRACE=true run-s build run_mocha", + "run_mocha": "mocha --require source-map-support/register --require make-promises-safe 'lib/test/**/*.js' --timeout 100000 --bail --exit", + "compile": "sol-compiler --contracts-dir contracts", + "clean": "shx rm -rf lib generated-artifacts generated-wrappers", + "generate_contract_wrappers": "abi-gen --abis ${npm_package_config_abis} --template ../../packages/abi-gen-templates/contract.handlebars --partials '../../packages/abi-gen-templates/partials/**/*.handlebars' --output generated-wrappers --backend ethers", + "lint": "tslint --format stylish --project . --exclude ./generated-wrappers/**/* --exclude ./generated-artifacts/**/* --exclude **/lib/**/* && yarn lint-contracts", + "coverage:report:text": "istanbul report text", + "coverage:report:html": "istanbul report html && open coverage/index.html", + "profiler:report:html": "istanbul report html && open coverage/index.html", + "coverage:report:lcov": "istanbul report lcov", + "test:circleci": "yarn test", + "lint-contracts": "solhint contracts/**/**/**/**/*.sol" + }, + "config": { + "abis": "generated-artifacts/@(MultiSigWallet|MultiSigWalletWithTimeLock|TestRejectEther).json" + }, + "repository": { + "type": "git", + "url": "https://github.com/0xProject/0x-monorepo.git" + }, + "license": "Apache-2.0", + "bugs": { + "url": "https://github.com/0xProject/0x-monorepo/issues" + }, + "homepage": "https://github.com/0xProject/0x-monorepo/contracts/core/README.md", + "devDependencies": { + "@0x/contracts-test-utils": "^1.0.0", + "@0x/abi-gen": "^1.0.17", + "@0x/dev-utils": "^1.0.18", + "@0x/sol-compiler": "^1.1.13", + "@0x/sol-cov": "^2.1.13", + "@0x/subproviders": "^2.1.5", + "@0x/tslint-config": "^1.0.10", + "@types/bn.js": "^4.11.0", + "@types/ethereumjs-abi": "^0.6.0", + "@types/lodash": "4.14.104", + "@types/node": "*", + "@types/yargs": "^10.0.0", + "chai": "^4.0.1", + "chai-as-promised": "^7.1.0", + "chai-bignumber": "^2.0.1", + "dirty-chai": "^2.0.1", + "make-promises-safe": "^1.1.0", + "mocha": "^4.1.0", + "npm-run-all": "^4.1.2", + "shx": "^0.2.2", + "solc": "^0.4.24", + "solhint": "^1.2.1", + "tslint": "5.11.0", + "typescript": "3.0.1", + "yargs": "^10.0.3" + }, + "dependencies": { + "@0x/base-contract": "^3.0.7", + "@0x/order-utils": "^3.0.3", + "@0x/types": "^1.3.0", + "@0x/typescript-typings": "^3.0.4", + "@0x/utils": "^2.0.6", + "@0x/web3-wrapper": "^3.1.5", + "ethereum-types": "^1.1.2", + "lodash": "^4.17.5" + }, + "publishConfig": { + "access": "public" + } +} diff --git a/contracts/multisig/src/artifacts/index.ts b/contracts/multisig/src/artifacts/index.ts new file mode 100644 index 000000000..7cf47be01 --- /dev/null +++ b/contracts/multisig/src/artifacts/index.ts @@ -0,0 +1,11 @@ +import { ContractArtifact } from 'ethereum-types'; + +import * as MultiSigWallet from '../../generated-artifacts/MultiSigWallet.json'; +import * as MultiSigWalletWithTimeLock from '../../generated-artifacts/MultiSigWalletWithTimeLock.json'; +import * as TestRejectEther from '../../generated-artifacts/TestRejectEther.json'; + +export const artifacts = { + TestRejectEther: TestRejectEther as ContractArtifact, + MultiSigWallet: MultiSigWallet as ContractArtifact, + MultiSigWalletWithTimeLock: MultiSigWalletWithTimeLock as ContractArtifact, +}; diff --git a/contracts/multisig/src/wrappers/index.ts b/contracts/multisig/src/wrappers/index.ts new file mode 100644 index 000000000..69abd62f2 --- /dev/null +++ b/contracts/multisig/src/wrappers/index.ts @@ -0,0 +1,2 @@ +export * from '../../generated-wrappers/multi_sig_wallet'; +export * from '../../generated-wrappers/multi_sig_wallet_with_time_lock'; diff --git a/contracts/multisig/test/global_hooks.ts b/contracts/multisig/test/global_hooks.ts new file mode 100644 index 000000000..68eb4f8d5 --- /dev/null +++ b/contracts/multisig/test/global_hooks.ts @@ -0,0 +1,19 @@ +import { env, EnvVars } from '@0x/dev-utils'; + +import { coverage, profiler, provider } from '@0x/contracts-test-utils'; + +before('start web3 provider engine', () => { + provider.start(); +}); + +after('generate coverage report', async () => { + if (env.parseBoolean(EnvVars.SolidityCoverage)) { + const coverageSubprovider = coverage.getCoverageSubproviderSingleton(); + await coverageSubprovider.writeCoverageAsync(); + } + if (env.parseBoolean(EnvVars.SolidityProfiler)) { + const profilerSubprovider = profiler.getProfilerSubproviderSingleton(); + await profilerSubprovider.writeProfilerOutputAsync(); + } + provider.stop(); +}); diff --git a/contracts/core/test/multisig/multi_sig_with_time_lock.ts b/contracts/multisig/test/multi_sig_with_time_lock.ts index 1c0cb0515..31c215505 100644 --- a/contracts/core/test/multisig/multi_sig_with_time_lock.ts +++ b/contracts/multisig/test/multi_sig_with_time_lock.ts @@ -1,3 +1,13 @@ +import { + chaiSetup, + constants, + expectTransactionFailedAsync, + expectTransactionFailedWithoutReasonAsync, + increaseTimeAndMineBlockAsync, + provider, + txDefaults, + web3Wrapper, +} from '@0x/contracts-test-utils'; import { BlockchainLifecycle } from '@0x/dev-utils'; import { RevertReason } from '@0x/types'; import { BigNumber } from '@0x/utils'; @@ -5,7 +15,6 @@ import * as chai from 'chai'; import { LogWithDecodedArgs } from 'ethereum-types'; import * as _ from 'lodash'; -import { DummyERC20TokenContract } from '../../generated-wrappers/dummy_erc20_token'; import { MultiSigWalletWithTimeLockConfirmationEventArgs, MultiSigWalletWithTimeLockConfirmationTimeSetEventArgs, @@ -13,14 +22,11 @@ import { MultiSigWalletWithTimeLockExecutionEventArgs, MultiSigWalletWithTimeLockExecutionFailureEventArgs, MultiSigWalletWithTimeLockSubmissionEventArgs, -} from '../../generated-wrappers/multi_sig_wallet_with_time_lock'; -import { artifacts } from '../../src/artifacts'; -import { expectTransactionFailedAsync, expectTransactionFailedWithoutReasonAsync } from '../utils/assertions'; -import { increaseTimeAndMineBlockAsync } from '../utils/block_timestamp'; -import { chaiSetup } from '../utils/chai_setup'; -import { constants } from '../utils/constants'; -import { MultiSigWrapper } from '../utils/multi_sig_wrapper'; -import { provider, txDefaults, web3Wrapper } from '../utils/web3_wrapper'; +} from '../generated-wrappers/multi_sig_wallet_with_time_lock'; +import { TestRejectEtherContract } from '../generated-wrappers/test_reject_ether'; +import { artifacts } from '../src/artifacts'; + +import { MultiSigWrapper } from './utils/multi_sig_wrapper'; chaiSetup.configure(); const expect = chai.expect; @@ -189,14 +195,10 @@ describe('MultiSigWalletWithTimeLock', () => { await expectTransactionFailedWithoutReasonAsync(multiSigWrapper.executeTransactionAsync(txId, owners[1])); }); it("should log an ExecutionFailure event and not update the transaction's execution state if unsuccessful", async () => { - const contractWithoutFallback = await DummyERC20TokenContract.deployFrom0xArtifactAsync( - artifacts.DummyERC20Token, + const contractWithoutFallback = await TestRejectEtherContract.deployFrom0xArtifactAsync( + artifacts.TestRejectEther, provider, txDefaults, - constants.DUMMY_TOKEN_NAME, - constants.DUMMY_TOKEN_SYMBOL, - constants.DUMMY_TOKEN_DECIMALS, - constants.DUMMY_TOKEN_TOTAL_SUPPLY, ); const data = constants.NULL_BYTES; const value = new BigNumber(10); diff --git a/contracts/core/test/utils/multi_sig_wrapper.ts b/contracts/multisig/test/utils/multi_sig_wrapper.ts index 74fd3b4d6..086143613 100644 --- a/contracts/core/test/utils/multi_sig_wrapper.ts +++ b/contracts/multisig/test/utils/multi_sig_wrapper.ts @@ -1,12 +1,11 @@ +import { LogDecoder } from '@0x/contracts-test-utils'; import { BigNumber } from '@0x/utils'; import { Web3Wrapper } from '@0x/web3-wrapper'; import { Provider, TransactionReceiptWithDecodedLogs } from 'ethereum-types'; import * as _ from 'lodash'; -import { AssetProxyOwnerContract } from '../../generated-wrappers/asset_proxy_owner'; import { MultiSigWalletContract } from '../../generated-wrappers/multi_sig_wallet'; - -import { LogDecoder } from './log_decoder'; +import { artifacts } from '../../src/artifacts'; export class MultiSigWrapper { private readonly _multiSig: MultiSigWalletContract; @@ -15,7 +14,7 @@ export class MultiSigWrapper { constructor(multiSigContract: MultiSigWalletContract, provider: Provider) { this._multiSig = multiSigContract; this._web3Wrapper = new Web3Wrapper(provider); - this._logDecoder = new LogDecoder(this._web3Wrapper); + this._logDecoder = new LogDecoder(this._web3Wrapper, artifacts); } public async submitTransactionAsync( destination: string, @@ -52,16 +51,4 @@ export class MultiSigWrapper { const tx = await this._logDecoder.getTxWithDecodedLogsAsync(txHash); return tx; } - public async executeRemoveAuthorizedAddressAtIndexAsync( - txId: BigNumber, - from: string, - ): Promise<TransactionReceiptWithDecodedLogs> { - // tslint:disable-next-line:no-unnecessary-type-assertion - const txHash = await (this - ._multiSig as AssetProxyOwnerContract).executeRemoveAuthorizedAddressAtIndex.sendTransactionAsync(txId, { - from, - }); - const tx = await this._logDecoder.getTxWithDecodedLogsAsync(txHash); - return tx; - } } diff --git a/contracts/multisig/tsconfig.json b/contracts/multisig/tsconfig.json new file mode 100644 index 000000000..6f381620e --- /dev/null +++ b/contracts/multisig/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../../tsconfig", + "compilerOptions": { + "outDir": "lib", + "rootDir": ".", + "resolveJsonModule": true + }, + "include": ["./src/**/*", "./test/**/*", "./generated-wrappers/**/*"], + "files": [ + "./generated-artifacts/MultiSigWallet.json", + "./generated-artifacts/MultiSigWalletWithTimeLock.json", + "./generated-artifacts/TestRejectEther.json" + ], + "exclude": ["./deploy/solc/solc_bin"] +} diff --git a/contracts/multisig/tslint.json b/contracts/multisig/tslint.json new file mode 100644 index 000000000..1bb3ac2a2 --- /dev/null +++ b/contracts/multisig/tslint.json @@ -0,0 +1,6 @@ +{ + "extends": ["@0x/tslint-config"], + "rules": { + "custom-no-magic-numbers": false + } +} diff --git a/contracts/test-utils/README.md b/contracts/test-utils/README.md new file mode 100644 index 000000000..73fd93f45 --- /dev/null +++ b/contracts/test-utils/README.md @@ -0,0 +1,73 @@ +## Contracts test utils + +This package contains test utilities used by other smart contracts packages. + +## Usage + +```typescript +import { + chaiSetup, + constants, + expectContractCallFailedAsync, + expectContractCreationFailedAsync, + expectTransactionFailedAsync, + expectTransactionFailedWithoutReasonAsync, + increaseTimeAndMineBlockAsync, + provider, + sendTransactionResult, + txDefaults, + web3Wrapper, +} from '@0x/contracts-test-utils'; +``` + +## Contributing + +We strongly recommend that the community help us make improvements and determine the future direction of the protocol. To report bugs within this package, please create an issue in this repository. + +Please read our [contribution guidelines](../../CONTRIBUTING.md) before getting started. + +### Install Dependencies + +If you don't have yarn workspaces enabled (Yarn < v1.0) - enable them: + +```bash +yarn config set workspaces-experimental true +``` + +Then install dependencies + +```bash +yarn install +``` + +### Build + +To build this package and all other monorepo packages that it depends on, run the following from the monorepo root directory: + +```bash +PKG=@0x/contracts-test-utils yarn build +``` + +Or continuously rebuild on change: + +```bash +PKG=@0x/contracts-test-utils yarn watch +``` + +### Clean + +```bash +yarn clean +``` + +### Lint + +```bash +yarn lint +``` + +### Run Tests + +```bash +yarn test +``` diff --git a/contracts/test-utils/package.json b/contracts/test-utils/package.json new file mode 100644 index 000000000..513cfdc10 --- /dev/null +++ b/contracts/test-utils/package.json @@ -0,0 +1,75 @@ +{ + "name": "@0x/contracts-test-utils", + "version": "1.0.0", + "engines": { + "node": ">=6.12" + }, + "description": "Test utils for 0x contracts", + "main": "lib/src/index.js", + "directories": { + "test": "test" + }, + "scripts": { + "build": "tsc -b", + "build:ci": "yarn build", + "test": "yarn run_mocha", + "test:coverage": "run-s build run_mocha coverage:report:text coverage:report:lcov", + "run_mocha": "mocha --require source-map-support/register --require make-promises-safe 'lib/test/**/*.js' --timeout 100000 --bail --exit", + "clean": "shx rm -rf lib", + "lint": "tslint --format stylish --project tsconfig.lint.json", + "coverage:report:text": "istanbul report text", + "coverage:report:html": "istanbul report html && open coverage/index.html", + "profiler:report:html": "istanbul report html && open coverage/index.html", + "coverage:report:lcov": "istanbul report lcov", + "test:circleci": "yarn test" + }, + "repository": { + "type": "git", + "url": "https://github.com/0xProject/0x-monorepo.git" + }, + "license": "Apache-2.0", + "bugs": { + "url": "https://github.com/0xProject/0x-monorepo/issues" + }, + "homepage": "https://github.com/0xProject/0x-monorepo/contracts/test-utils/README.md", + "devDependencies": { + "mocha": "^4.1.0", + "npm-run-all": "^4.1.2", + "shx": "^0.2.2", + "tslint": "5.11.0", + "typescript": "3.0.1" + }, + "dependencies": { + "@0x/abi-gen": "^1.0.17", + "@0x/dev-utils": "^1.0.18", + "@0x/sol-compiler": "^1.1.13", + "@0x/subproviders": "^2.1.5", + "@0x/tslint-config": "^1.0.10", + "@types/bn.js": "^4.11.0", + "@types/ethereumjs-abi": "^0.6.0", + "@types/lodash": "4.14.104", + "@types/node": "*", + "chai": "^4.0.1", + "chai-bignumber": "^2.0.1", + "dirty-chai": "^2.0.1", + "make-promises-safe": "^1.1.0", + "@0x/order-utils": "^3.0.3", + "@0x/types": "^1.3.0", + "@0x/typescript-typings": "^3.0.4", + "@0x/utils": "^2.0.6", + "@0x/sol-cov": "^2.1.13", + "@0x/web3-wrapper": "^3.1.5", + "@types/js-combinatorics": "^0.5.29", + "chai-as-promised": "^7.1.0", + "bn.js": "^4.11.8", + "ethereum-types": "^1.1.2", + "ethereumjs-abi": "0.6.5", + "ethereumjs-util": "^5.1.1", + "ethers": "~4.0.4", + "js-combinatorics": "^0.5.3", + "lodash": "^4.17.5" + }, + "publishConfig": { + "access": "public" + } +} diff --git a/contracts/core/test/utils/abstract_asset_wrapper.ts b/contracts/test-utils/src/abstract_asset_wrapper.ts index 4b56a8502..4b56a8502 100644 --- a/contracts/core/test/utils/abstract_asset_wrapper.ts +++ b/contracts/test-utils/src/abstract_asset_wrapper.ts diff --git a/contracts/core/test/utils/address_utils.ts b/contracts/test-utils/src/address_utils.ts index 634da0c16..634da0c16 100644 --- a/contracts/core/test/utils/address_utils.ts +++ b/contracts/test-utils/src/address_utils.ts diff --git a/contracts/core/test/utils/assertions.ts b/contracts/test-utils/src/assertions.ts index 5b1cedfcc..5b1cedfcc 100644 --- a/contracts/core/test/utils/assertions.ts +++ b/contracts/test-utils/src/assertions.ts diff --git a/contracts/core/test/utils/block_timestamp.ts b/contracts/test-utils/src/block_timestamp.ts index 66c13eed1..66c13eed1 100644 --- a/contracts/core/test/utils/block_timestamp.ts +++ b/contracts/test-utils/src/block_timestamp.ts diff --git a/contracts/core/test/utils/chai_setup.ts b/contracts/test-utils/src/chai_setup.ts index 1a8733093..1a8733093 100644 --- a/contracts/core/test/utils/chai_setup.ts +++ b/contracts/test-utils/src/chai_setup.ts diff --git a/contracts/core/test/utils/combinatorial_utils.ts b/contracts/test-utils/src/combinatorial_utils.ts index bb1b55b4d..bb1b55b4d 100644 --- a/contracts/core/test/utils/combinatorial_utils.ts +++ b/contracts/test-utils/src/combinatorial_utils.ts diff --git a/contracts/core/test/utils/constants.ts b/contracts/test-utils/src/constants.ts index d2c3ab512..d2c3ab512 100644 --- a/contracts/core/test/utils/constants.ts +++ b/contracts/test-utils/src/constants.ts diff --git a/contracts/core/test/utils/coverage.ts b/contracts/test-utils/src/coverage.ts index 5becfa1b6..5becfa1b6 100644 --- a/contracts/core/test/utils/coverage.ts +++ b/contracts/test-utils/src/coverage.ts diff --git a/contracts/core/test/utils/formatters.ts b/contracts/test-utils/src/formatters.ts index 813eb45db..813eb45db 100644 --- a/contracts/core/test/utils/formatters.ts +++ b/contracts/test-utils/src/formatters.ts diff --git a/contracts/test-utils/src/global_hooks.ts b/contracts/test-utils/src/global_hooks.ts new file mode 100644 index 000000000..307dd0777 --- /dev/null +++ b/contracts/test-utils/src/global_hooks.ts @@ -0,0 +1,15 @@ +import { env, EnvVars } from '@0x/dev-utils'; + +import { coverage } from './coverage'; +import { profiler } from './profiler'; + +after('generate coverage report', async () => { + if (env.parseBoolean(EnvVars.SolidityCoverage)) { + const coverageSubprovider = coverage.getCoverageSubproviderSingleton(); + await coverageSubprovider.writeCoverageAsync(); + } + if (env.parseBoolean(EnvVars.SolidityProfiler)) { + const profilerSubprovider = profiler.getProfilerSubproviderSingleton(); + await profilerSubprovider.writeProfilerOutputAsync(); + } +}); diff --git a/contracts/test-utils/src/index.ts b/contracts/test-utils/src/index.ts new file mode 100644 index 000000000..7880de0bf --- /dev/null +++ b/contracts/test-utils/src/index.ts @@ -0,0 +1,55 @@ +export { AbstractAssetWrapper } from './abstract_asset_wrapper'; +export { chaiSetup } from './chai_setup'; +export { constants } from './constants'; +export { + expectContractCallFailedAsync, + expectContractCallFailedWithoutReasonAsync, + expectContractCreationFailedAsync, + expectContractCreationFailedWithoutReasonAsync, + expectInsufficientFundsAsync, + expectTransactionFailedAsync, + sendTransactionResult, + expectTransactionFailedWithoutReasonAsync, + getInvalidOpcodeErrorMessageForCallAsync, + getRevertReasonOrErrorMessageForSendTransactionAsync, +} from './assertions'; +export { getLatestBlockTimestampAsync, increaseTimeAndMineBlockAsync } from './block_timestamp'; +export { provider, txDefaults, web3Wrapper } from './web3_wrapper'; +export { LogDecoder } from './log_decoder'; +export { formatters } from './formatters'; +export { signingUtils } from './signing_utils'; +export { orderUtils } from './order_utils'; +export { typeEncodingUtils } from './type_encoding_utils'; +export { profiler } from './profiler'; +export { coverage } from './coverage'; +export { addressUtils } from './address_utils'; +export { OrderFactory } from './order_factory'; +export { bytes32Values, testCombinatoriallyWithReferenceFuncAsync, uint256Values } from './combinatorial_utils'; +export { TransactionFactory } from './transaction_factory'; +export { testWithReferenceFuncAsync } from './test_with_reference'; +export { + MarketBuyOrders, + MarketSellOrders, + ERC721TokenIdsByOwner, + SignedTransaction, + OrderStatus, + AllowanceAmountScenario, + AssetDataScenario, + BalanceAmountScenario, + ContractName, + ExpirationTimeSecondsScenario, + TransferAmountsLoggedByMatchOrders, + TransferAmountsByMatchOrders, + OrderScenario, + TraderStateScenario, + TransactionDataParams, + Token, + FillScenario, + FeeRecipientAddressScenario, + OrderAssetAmountScenario, + TakerAssetFillAmountScenario, + TakerScenario, + OrderInfo, + ERC20BalancesByOwner, + FillResults, +} from './types'; diff --git a/contracts/core/test/utils/log_decoder.ts b/contracts/test-utils/src/log_decoder.ts index 05b0a9204..54666ea5f 100644 --- a/contracts/core/test/utils/log_decoder.ts +++ b/contracts/test-utils/src/log_decoder.ts @@ -11,8 +11,6 @@ import { } from 'ethereum-types'; import * as _ from 'lodash'; -import { artifacts } from '../../src/artifacts'; - import { constants } from './constants'; export class LogDecoder { @@ -27,7 +25,7 @@ export class LogDecoder { } } } - constructor(web3Wrapper: Web3Wrapper) { + constructor(web3Wrapper: Web3Wrapper, artifacts: { [contractName: string]: ContractArtifact }) { this._web3Wrapper = web3Wrapper; const abiArrays: AbiDefinition[][] = []; _.forEach(artifacts, (artifact: ContractArtifact) => { diff --git a/contracts/core/test/utils/order_factory.ts b/contracts/test-utils/src/order_factory.ts index 2449d1a8a..2449d1a8a 100644 --- a/contracts/core/test/utils/order_factory.ts +++ b/contracts/test-utils/src/order_factory.ts diff --git a/contracts/core/test/utils/order_utils.ts b/contracts/test-utils/src/order_utils.ts index 4f7a34011..4f7a34011 100644 --- a/contracts/core/test/utils/order_utils.ts +++ b/contracts/test-utils/src/order_utils.ts diff --git a/contracts/core/test/utils/profiler.ts b/contracts/test-utils/src/profiler.ts index 2c7c1d66c..2c7c1d66c 100644 --- a/contracts/core/test/utils/profiler.ts +++ b/contracts/test-utils/src/profiler.ts diff --git a/contracts/core/test/utils/revert_trace.ts b/contracts/test-utils/src/revert_trace.ts index 3f74fd28b..3f74fd28b 100644 --- a/contracts/core/test/utils/revert_trace.ts +++ b/contracts/test-utils/src/revert_trace.ts diff --git a/contracts/core/test/utils/signing_utils.ts b/contracts/test-utils/src/signing_utils.ts index 21f864bfa..21f864bfa 100644 --- a/contracts/core/test/utils/signing_utils.ts +++ b/contracts/test-utils/src/signing_utils.ts diff --git a/contracts/core/test/utils/test_with_reference.ts b/contracts/test-utils/src/test_with_reference.ts index b80be4a6c..b80be4a6c 100644 --- a/contracts/core/test/utils/test_with_reference.ts +++ b/contracts/test-utils/src/test_with_reference.ts diff --git a/contracts/core/test/utils/transaction_factory.ts b/contracts/test-utils/src/transaction_factory.ts index dbab3ade4..dbab3ade4 100644 --- a/contracts/core/test/utils/transaction_factory.ts +++ b/contracts/test-utils/src/transaction_factory.ts diff --git a/contracts/core/test/utils/type_encoding_utils.ts b/contracts/test-utils/src/type_encoding_utils.ts index bfd9c9ef5..bfd9c9ef5 100644 --- a/contracts/core/test/utils/type_encoding_utils.ts +++ b/contracts/test-utils/src/type_encoding_utils.ts diff --git a/contracts/core/test/utils/types.ts b/contracts/test-utils/src/types.ts index d738fcd4e..d738fcd4e 100644 --- a/contracts/core/test/utils/types.ts +++ b/contracts/test-utils/src/types.ts diff --git a/contracts/core/test/utils/web3_wrapper.ts b/contracts/test-utils/src/web3_wrapper.ts index f7b1a732a..cb33476f3 100644 --- a/contracts/core/test/utils/web3_wrapper.ts +++ b/contracts/test-utils/src/web3_wrapper.ts @@ -48,6 +48,7 @@ const ganacheConfigs = { const providerConfigs = testProvider === ProviderType.Ganache ? ganacheConfigs : gethConfigs; export const provider: Web3ProviderEngine = web3Factory.getRpcProvider(providerConfigs); +provider.stop(); const isCoverageEnabled = env.parseBoolean(EnvVars.SolidityCoverage); const isProfilerEnabled = env.parseBoolean(EnvVars.SolidityProfiler); const isRevertTraceEnabled = env.parseBoolean(EnvVars.SolidityRevertTrace); diff --git a/contracts/core/test/utils_test/test_with_reference.ts b/contracts/test-utils/test/test_with_reference.ts index 8d633cd1f..1c1211003 100644 --- a/contracts/core/test/utils_test/test_with_reference.ts +++ b/contracts/test-utils/test/test_with_reference.ts @@ -1,7 +1,7 @@ import * as chai from 'chai'; -import { chaiSetup } from '../utils/chai_setup'; -import { testWithReferenceFuncAsync } from '../utils/test_with_reference'; +import { chaiSetup } from '../src/chai_setup'; +import { testWithReferenceFuncAsync } from '../src/test_with_reference'; chaiSetup.configure(); const expect = chai.expect; diff --git a/contracts/test-utils/tsconfig.json b/contracts/test-utils/tsconfig.json new file mode 100644 index 000000000..e35816553 --- /dev/null +++ b/contracts/test-utils/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig", + "compilerOptions": { + "outDir": "lib" + }, + "include": ["./src/**/*", "./test/**/*"] +} diff --git a/contracts/test-utils/tsconfig.lint.json b/contracts/test-utils/tsconfig.lint.json new file mode 100644 index 000000000..b557e706a --- /dev/null +++ b/contracts/test-utils/tsconfig.lint.json @@ -0,0 +1,7 @@ +{ + // This file is a workaround that issue: https://github.com/palantir/tslint/issues/4148#issuecomment-419872702 + "extends": "./tsconfig", + "compilerOptions": { + "composite": false + } +} diff --git a/contracts/test-utils/tslint.json b/contracts/test-utils/tslint.json new file mode 100644 index 000000000..1bb3ac2a2 --- /dev/null +++ b/contracts/test-utils/tslint.json @@ -0,0 +1,6 @@ +{ + "extends": ["@0x/tslint-config"], + "rules": { + "custom-no-magic-numbers": false + } +} diff --git a/contracts/utils/.solhint.json b/contracts/utils/.solhint.json new file mode 100644 index 000000000..076afe9f3 --- /dev/null +++ b/contracts/utils/.solhint.json @@ -0,0 +1,20 @@ +{ + "extends": "default", + "rules": { + "avoid-low-level-calls": false, + "avoid-tx-origin": "warn", + "bracket-align": false, + "code-complexity": false, + "const-name-snakecase": "error", + "expression-indent": "error", + "function-max-lines": false, + "func-order": "error", + "indent": ["error", 4], + "max-line-length": ["warn", 160], + "no-inline-assembly": false, + "quotes": ["error", "double"], + "separate-by-one-line-in-contract": "error", + "space-after-comma": "error", + "statement-indent": "error" + } +} diff --git a/contracts/utils/README.md b/contracts/utils/README.md new file mode 100644 index 000000000..e7c7b49ff --- /dev/null +++ b/contracts/utils/README.md @@ -0,0 +1,70 @@ +## Contracts utils + +Smart contracts utils used in the 0x protocol. + +## Usage + +Contracts can be found in the [contracts](./contracts) directory. The contents of this directory are broken down into the following subdirectories: + +* [utils](./contracts/utils) + * This directory contains libraries and utils. +* [test](./contracts/test) + * This directory contains mocks and other contracts that are used solely for testing contracts within the other directories. + +## Contributing + +We strongly recommend that the community help us make improvements and determine the future direction of the protocol. To report bugs within this package, please create an issue in this repository. + +For proposals regarding the 0x protocol's smart contract architecture, message format, or additional functionality, go to the [0x Improvement Proposals (ZEIPs)](https://github.com/0xProject/ZEIPs) repository and follow the contribution guidelines provided therein. + +Please read our [contribution guidelines](../../CONTRIBUTING.md) before getting started. + +### Install Dependencies + +If you don't have yarn workspaces enabled (Yarn < v1.0) - enable them: + +```bash +yarn config set workspaces-experimental true +``` + +Then install dependencies + +```bash +yarn install +``` + +### Build + +To build this package and all other monorepo packages that it depends on, run the following from the monorepo root directory: + +```bash +PKG=@0x/contracts-utils yarn build +``` + +Or continuously rebuild on change: + +```bash +PKG=@0x/contracts-utils yarn watch +``` + +### Clean + +```bash +yarn clean +``` + +### Lint + +```bash +yarn lint +``` + +### Run Tests + +```bash +yarn test +``` + +#### Testing options + +Contracts testing options like coverage, profiling, revert traces or backing node choosing - are described [here](../TESTING.md). diff --git a/contracts/utils/compiler.json b/contracts/utils/compiler.json new file mode 100644 index 000000000..1524c1eaa --- /dev/null +++ b/contracts/utils/compiler.json @@ -0,0 +1,22 @@ +{ + "artifactsDir": "./generated-artifacts", + "contractsDir": "./contracts", + "compilerSettings": { + "optimizer": { + "enabled": true, + "runs": 1000000 + }, + "outputSelection": { + "*": { + "*": [ + "abi", + "evm.bytecode.object", + "evm.bytecode.sourceMap", + "evm.deployedBytecode.object", + "evm.deployedBytecode.sourceMap" + ] + } + } + }, + "contracts": ["TestConstants", "TestLibBytes", "LibBytes", "Ownable", "IOwnable", "ReentrancyGuard", "SafeMath"] +} diff --git a/contracts/core/contracts/test/TestConstants/TestConstants.sol b/contracts/utils/contracts/test/TestConstants/TestConstants.sol index 1275d007b..3c852173b 100644 --- a/contracts/core/contracts/test/TestConstants/TestConstants.sol +++ b/contracts/utils/contracts/test/TestConstants/TestConstants.sol @@ -18,7 +18,7 @@ pragma solidity 0.4.24; -import "../../utils/LibBytes/LibBytes.sol"; +import "@0x/contracts-utils/contracts/utils/LibBytes/LibBytes.sol"; // solhint-disable max-line-length diff --git a/contracts/core/contracts/test/TestLibBytes/TestLibBytes.sol b/contracts/utils/contracts/test/TestLibBytes/TestLibBytes.sol index 00d861e61..444a3e717 100644 --- a/contracts/core/contracts/test/TestLibBytes/TestLibBytes.sol +++ b/contracts/utils/contracts/test/TestLibBytes/TestLibBytes.sol @@ -18,7 +18,7 @@ pragma solidity 0.4.24; -import "../../utils/LibBytes/LibBytes.sol"; +import "@0x/contracts-utils/contracts/utils/LibBytes/LibBytes.sol"; contract TestLibBytes { diff --git a/contracts/core/contracts/utils/LibBytes/LibBytes.sol b/contracts/utils/contracts/utils/LibBytes/LibBytes.sol index 369f588ad..369f588ad 100644 --- a/contracts/core/contracts/utils/LibBytes/LibBytes.sol +++ b/contracts/utils/contracts/utils/LibBytes/LibBytes.sol diff --git a/contracts/core/contracts/utils/Ownable/IOwnable.sol b/contracts/utils/contracts/utils/Ownable/IOwnable.sol index 5deb13497..5deb13497 100644 --- a/contracts/core/contracts/utils/Ownable/IOwnable.sol +++ b/contracts/utils/contracts/utils/Ownable/IOwnable.sol diff --git a/contracts/core/contracts/utils/Ownable/Ownable.sol b/contracts/utils/contracts/utils/Ownable/Ownable.sol index 0c830be68..0c830be68 100644 --- a/contracts/core/contracts/utils/Ownable/Ownable.sol +++ b/contracts/utils/contracts/utils/Ownable/Ownable.sol diff --git a/contracts/core/contracts/utils/ReentrancyGuard/ReentrancyGuard.sol b/contracts/utils/contracts/utils/ReentrancyGuard/ReentrancyGuard.sol index 9f98a7a16..9f98a7a16 100644 --- a/contracts/core/contracts/utils/ReentrancyGuard/ReentrancyGuard.sol +++ b/contracts/utils/contracts/utils/ReentrancyGuard/ReentrancyGuard.sol diff --git a/contracts/core/contracts/utils/SafeMath/SafeMath.sol b/contracts/utils/contracts/utils/SafeMath/SafeMath.sol index 2855edb9d..2855edb9d 100644 --- a/contracts/core/contracts/utils/SafeMath/SafeMath.sol +++ b/contracts/utils/contracts/utils/SafeMath/SafeMath.sol diff --git a/contracts/utils/package.json b/contracts/utils/package.json new file mode 100644 index 000000000..c0bc8bfcf --- /dev/null +++ b/contracts/utils/package.json @@ -0,0 +1,90 @@ +{ + "private": true, + "name": "@0x/contracts-utils", + "version": "1.0.0", + "engines": { + "node": ">=6.12" + }, + "description": "Smart contract utils of 0x protocol", + "main": "lib/src/index.js", + "directories": { + "test": "test" + }, + "scripts": { + "build": "yarn pre_build && tsc -b", + "build:ci": "yarn build", + "pre_build": "run-s compile generate_contract_wrappers", + "test": "yarn run_mocha", + "rebuild_and_test": "run-s build test", + "test:coverage": "SOLIDITY_COVERAGE=true run-s build run_mocha coverage:report:text coverage:report:lcov", + "test:profiler": "SOLIDITY_PROFILER=true run-s build run_mocha profiler:report:html", + "test:trace": "SOLIDITY_REVERT_TRACE=true run-s build run_mocha", + "run_mocha": + "mocha --require source-map-support/register --require make-promises-safe 'lib/test/**/*.js' --timeout 100000 --bail --exit", + "compile": "sol-compiler --contracts-dir contracts", + "clean": "shx rm -rf lib generated-artifacts generated-wrappers", + "generate_contract_wrappers": "abi-gen --abis ${npm_package_config_abis} --template ../../node_modules/@0x/abi-gen-templates/contract.handlebars --partials '../../node_modules/@0x/abi-gen-templates/partials/**/*.handlebars' --output generated-wrappers --backend ethers", + "lint": "tslint --format stylish --project . --exclude ./generated-wrappers/**/* --exclude ./generated-artifacts/**/* --exclude **/lib/**/* && yarn lint-contracts", + "coverage:report:text": "istanbul report text", + "coverage:report:html": "istanbul report html && open coverage/index.html", + "profiler:report:html": "istanbul report html && open coverage/index.html", + "coverage:report:lcov": "istanbul report lcov", + "test:circleci": "yarn test", + "lint-contracts": "solhint contracts/**/**/**/**/*.sol" + }, + "config": { + "abis": "generated-artifacts/@(IOwnable|Ownable|LibBytes|ReentrancyGuard|SafeMath|TestConstants|TestLibBytes).json" + }, + "repository": { + "type": "git", + "url": "https://github.com/0xProject/0x-monorepo.git" + }, + "license": "Apache-2.0", + "bugs": { + "url": "https://github.com/0xProject/0x-monorepo/issues" + }, + "homepage": "https://github.com/0xProject/0x-monorepo/contracts/utils/README.md", + "devDependencies": { + "@0x/contracts-test-utils": "^1.0.0", + "@0x/abi-gen": "^1.0.17", + "@0x/dev-utils": "^1.0.19", + "@0x/sol-compiler": "^1.1.14", + "@0x/sol-cov": "^2.1.14", + "@0x/subproviders": "^2.1.6", + "@0x/tslint-config": "^1.0.10", + "@types/bn.js": "^4.11.0", + "@types/lodash": "4.14.104", + "@types/node": "*", + "@types/yargs": "^10.0.0", + "bn.js": "^4.11.8", + "chai": "^4.0.1", + "chai-as-promised": "^7.1.0", + "chai-bignumber": "^2.0.1", + "dirty-chai": "^2.0.1", + "make-promises-safe": "^1.1.0", + "ethereumjs-abi": "0.6.5", + "mocha": "^4.1.0", + "npm-run-all": "^4.1.2", + "shx": "^0.2.2", + "solc": "^0.4.24", + "solhint": "^1.2.1", + "tslint": "5.11.0", + "typescript": "3.0.1", + "yargs": "^10.0.3" + }, + "dependencies": { + "@0x/base-contract": "^3.0.8", + "@0x/order-utils": "^3.0.4", + "@0x/contracts-multisig": "^1.0.0", + "@0x/types": "^1.3.0", + "@0x/typescript-typings": "^3.0.4", + "@0x/utils": "^2.0.6", + "@0x/web3-wrapper": "^3.1.6", + "ethereum-types": "^1.1.2", + "ethereumjs-util": "^5.1.1", + "lodash": "^4.17.5" + }, + "publishConfig": { + "access": "public" + } +} diff --git a/contracts/utils/src/artifacts/index.ts b/contracts/utils/src/artifacts/index.ts new file mode 100644 index 000000000..a5c2b215c --- /dev/null +++ b/contracts/utils/src/artifacts/index.ts @@ -0,0 +1,19 @@ +import { ContractArtifact } from 'ethereum-types'; + +import * as IOwnable from '../../generated-artifacts/IOwnable.json'; +import * as LibBytes from '../../generated-artifacts/LibBytes.json'; +import * as Ownable from '../../generated-artifacts/Ownable.json'; +import * as ReentrancyGuard from '../../generated-artifacts/ReentrancyGuard.json'; +import * as SafeMath from '../../generated-artifacts/SafeMath.json'; +import * as TestConstants from '../../generated-artifacts/TestConstants.json'; +import * as TestLibBytes from '../../generated-artifacts/TestLibBytes.json'; + +export const artifacts = { + TestConstants: TestConstants as ContractArtifact, + TestLibBytes: TestLibBytes as ContractArtifact, + IOwnable: IOwnable as ContractArtifact, + LibBytes: LibBytes as ContractArtifact, + Ownable: Ownable as ContractArtifact, + SafeMath: SafeMath as ContractArtifact, + ReentrancyGuard: ReentrancyGuard as ContractArtifact, +}; diff --git a/contracts/utils/src/index.ts b/contracts/utils/src/index.ts new file mode 100644 index 000000000..d55f08ea2 --- /dev/null +++ b/contracts/utils/src/index.ts @@ -0,0 +1,2 @@ +export * from './artifacts'; +export * from './wrappers'; diff --git a/contracts/utils/src/wrappers/index.ts b/contracts/utils/src/wrappers/index.ts new file mode 100644 index 000000000..823b7fa4b --- /dev/null +++ b/contracts/utils/src/wrappers/index.ts @@ -0,0 +1,2 @@ +export * from '../../generated-wrappers/test_constants'; +export * from '../../generated-wrappers/test_lib_bytes'; diff --git a/contracts/utils/test/global_hooks.ts b/contracts/utils/test/global_hooks.ts new file mode 100644 index 000000000..f8ace376a --- /dev/null +++ b/contracts/utils/test/global_hooks.ts @@ -0,0 +1,17 @@ +import { env, EnvVars } from '@0x/dev-utils'; + +import { coverage, profiler, provider } from '@0x/contracts-test-utils'; +before('start web3 provider', () => { + provider.start(); +}); +after('generate coverage report', async () => { + if (env.parseBoolean(EnvVars.SolidityCoverage)) { + const coverageSubprovider = coverage.getCoverageSubproviderSingleton(); + await coverageSubprovider.writeCoverageAsync(); + } + if (env.parseBoolean(EnvVars.SolidityProfiler)) { + const profilerSubprovider = profiler.getProfilerSubproviderSingleton(); + await profilerSubprovider.writeProfilerOutputAsync(); + } + provider.stop(); +}); diff --git a/contracts/core/test/libraries/lib_bytes.ts b/contracts/utils/test/lib_bytes.ts index b1a389f00..985a98943 100644 --- a/contracts/core/test/libraries/lib_bytes.ts +++ b/contracts/utils/test/lib_bytes.ts @@ -1,3 +1,12 @@ +import { + chaiSetup, + constants, + expectContractCallFailedAsync, + provider, + txDefaults, + typeEncodingUtils, + web3Wrapper, +} from '@0x/contracts-test-utils'; import { BlockchainLifecycle } from '@0x/dev-utils'; import { generatePseudoRandomSalt } from '@0x/order-utils'; import { RevertReason } from '@0x/types'; @@ -7,13 +16,8 @@ import * as chai from 'chai'; import ethUtil = require('ethereumjs-util'); import * as _ from 'lodash'; -import { TestLibBytesContract } from '../../generated-wrappers/test_lib_bytes'; -import { artifacts } from '../../src/artifacts'; -import { expectContractCallFailedAsync } from '../utils/assertions'; -import { chaiSetup } from '../utils/chai_setup'; -import { constants } from '../utils/constants'; -import { typeEncodingUtils } from '../utils/type_encoding_utils'; -import { provider, txDefaults, web3Wrapper } from '../utils/web3_wrapper'; +import { TestLibBytesContract } from '../generated-wrappers/test_lib_bytes'; +import { artifacts } from '../src'; chaiSetup.configure(); const expect = chai.expect; diff --git a/contracts/utils/test/libs.ts b/contracts/utils/test/libs.ts new file mode 100644 index 000000000..81596b2e4 --- /dev/null +++ b/contracts/utils/test/libs.ts @@ -0,0 +1,34 @@ +import { chaiSetup, provider, txDefaults, web3Wrapper } from '@0x/contracts-test-utils'; +import { BlockchainLifecycle } from '@0x/dev-utils'; +import * as chai from 'chai'; + +import { TestConstantsContract } from '../generated-wrappers/test_constants'; +import { artifacts } from '../src'; + +chaiSetup.configure(); +const expect = chai.expect; + +const blockchainLifecycle = new BlockchainLifecycle(web3Wrapper); + +describe('Libs', () => { + beforeEach(async () => { + await blockchainLifecycle.startAsync(); + }); + afterEach(async () => { + await blockchainLifecycle.revertAsync(); + }); + + describe('LibConstants', () => { + describe('ZRX_ASSET_DATA', () => { + it('should have the correct ZRX_ASSET_DATA', async () => { + const testConstants = await TestConstantsContract.deployFrom0xArtifactAsync( + artifacts.TestConstants, + provider, + txDefaults, + ); + const isValid = await testConstants.assertValidZrxAssetData.callAsync(); + expect(isValid).to.be.equal(true); + }); + }); + }); +}); diff --git a/contracts/utils/tsconfig.json b/contracts/utils/tsconfig.json new file mode 100644 index 000000000..68251e6b0 --- /dev/null +++ b/contracts/utils/tsconfig.json @@ -0,0 +1,19 @@ +{ + "extends": "../../tsconfig", + "compilerOptions": { + "outDir": "lib", + "rootDir": ".", + "resolveJsonModule": true + }, + "include": ["./src/**/*", "./test/**/*", "./generated-wrappers/**/*"], + "files": [ + "./generated-artifacts/TestConstants.json", + "./generated-artifacts/TestLibBytes.json", + "./generated-artifacts/IOwnable.json", + "./generated-artifacts/Ownable.json", + "./generated-artifacts/LibBytes.json", + "./generated-artifacts/SafeMath.json", + "./generated-artifacts/ReentrancyGuard.json" + ], + "exclude": ["./deploy/solc/solc_bin"] +} diff --git a/contracts/utils/tslint.json b/contracts/utils/tslint.json new file mode 100644 index 000000000..1bb3ac2a2 --- /dev/null +++ b/contracts/utils/tslint.json @@ -0,0 +1,6 @@ +{ + "extends": ["@0x/tslint-config"], + "rules": { + "custom-no-magic-numbers": false + } +} diff --git a/packages/0x.js/.npmignore b/packages/0x.js/.npmignore index d7ee80c97..312a23faa 100644 --- a/packages/0x.js/.npmignore +++ b/packages/0x.js/.npmignore @@ -4,7 +4,7 @@ webpack.config.js yarn-error.log test/ /src/ -/contract_templates/ +/abi-gen-templates/ /generated_docs/ /scripts/ /lib/src/monorepo_scripts/ diff --git a/packages/abi-gen/README.md b/packages/abi-gen/README.md index 20b9d4f30..214d8f257 100644 --- a/packages/abi-gen/README.md +++ b/packages/abi-gen/README.md @@ -4,7 +4,7 @@ This package allows you to generate TypeScript contract wrappers from ABI files. It's heavily inspired by [Geth abigen](https://github.com/ethereum/go-ethereum/wiki/Native-DApps:-Go-bindings-to-Ethereum-contracts) but takes a different approach. You can write your custom handlebars templates which will allow you to seamlessly integrate the generated code into your existing codebase with existing conventions. -[Here](https://github.com/0xProject/0x-monorepo/tree/development/packages/0x.js/contract_templates) are the templates used to generate the contract wrappers used by 0x.js.e +[Here](https://github.com/0xProject/0x-monorepo/tree/development/packages/0x.js/abi-gen-templates) are the templates used to generate the contract wrappers used by 0x.js.e ## Installation @@ -44,7 +44,7 @@ You need to also specify the location of your main template used for every contr ## How to write custom templates? -The best way to get started is to copy [0x.js templates](https://github.com/0xProject/0x-monorepo/tree/development/packages/contract_templates) and start adjusting them for your needs. +The best way to get started is to copy [0x.js templates](https://github.com/0xProject/0x-monorepo/tree/development/packages/abi-gen-templates) and start adjusting them for your needs. We use [handlebars](http://handlebarsjs.com/) template engine under the hood. You need to have a master template called `contract.mustache`. it will be used to generate each contract wrapper. Although - you don't need and probably shouldn't write all your logic in a single template file. You can write [partial templates](http://handlebarsjs.com/partials.html) and as long as they are within a partials folder - they will be registered and available. diff --git a/packages/contract-wrappers/.npmignore b/packages/contract-wrappers/.npmignore index 6a3eb57bd..6a222fd45 100644 --- a/packages/contract-wrappers/.npmignore +++ b/packages/contract-wrappers/.npmignore @@ -5,7 +5,7 @@ yarn-error.log test/ /src/ /_bundles/ -/contract_templates/ +/abi-gen-templates/ /generated_docs/ /scripts/ /lib/src/monorepo_scripts/ diff --git a/packages/instant/.DS_Store b/packages/instant/.DS_Store Binary files differnew file mode 100644 index 000000000..9a0cceca6 --- /dev/null +++ b/packages/instant/.DS_Store diff --git a/packages/instant/.dogfood.discharge.json b/packages/instant/.dogfood.discharge.json index 5d6a09a16..651b3daa6 100644 --- a/packages/instant/.dogfood.discharge.json +++ b/packages/instant/.dogfood.discharge.json @@ -1,6 +1,6 @@ { "domain": "0x-instant-dogfood", - "build_command": "WEBPACK_OUTPUT_PATH=public dotenv yarn build --env.discharge_target=dogfood", + "build_command": "WEBPACK_OUTPUT_PATH=public dotenv yarn build -- --env.discharge_target=dogfood", "upload_directory": "public", "index_key": "index.html", "error_key": "index.html", diff --git a/packages/instant/.production.discharge.json b/packages/instant/.production.discharge.json index 947f68b1a..1ce39fdd8 100644 --- a/packages/instant/.production.discharge.json +++ b/packages/instant/.production.discharge.json @@ -1,6 +1,6 @@ { "domain": "instant.0xproject.com", - "build_command": "dotenv yarn build --env.discharge_target=production", + "build_command": "dotenv yarn build -- --env.discharge_target=production", "upload_directory": "umd", "index_key": "instant.js", "error_key": "404.html", diff --git a/packages/instant/.staging.discharge.json b/packages/instant/.staging.discharge.json index bd5f28ba8..844e3ca4e 100644 --- a/packages/instant/.staging.discharge.json +++ b/packages/instant/.staging.discharge.json @@ -1,6 +1,6 @@ { "domain": "0x-instant-staging", - "build_command": "dotenv WEBPACK_OUTPUT_PATH=public yarn build --env.discharge_target=staging", + "build_command": "WEBPACK_OUTPUT_PATH=public dotenv yarn build -- --env.discharge_target=staging", "upload_directory": "public", "index_key": "index.html", "error_key": "index.html", diff --git a/packages/instant/src/components/buy_order_progress.tsx b/packages/instant/src/components/buy_order_progress.tsx index 6568de91b..a19f5a4d0 100644 --- a/packages/instant/src/components/buy_order_progress.tsx +++ b/packages/instant/src/components/buy_order_progress.tsx @@ -21,7 +21,7 @@ export const BuyOrderProgress: React.StatelessComponent<BuyOrderProgressProps> = const hasEnded = buyOrderState.processState !== OrderProcessState.Processing; const expectedTimeMs = progress.expectedEndTimeUnix - progress.startTimeUnix; return ( - <Container padding="20px 20px 0px 20px" width="100%"> + <Container width="100%" padding="20px 20px 0px 20px"> <Container marginBottom="5px"> <TimeCounter estimatedTimeMs={expectedTimeMs} hasEnded={hasEnded} key={progress.startTimeUnix} /> </Container> diff --git a/packages/instant/src/components/instant_heading.tsx b/packages/instant/src/components/instant_heading.tsx index 808c6dc7f..117f9dd5f 100644 --- a/packages/instant/src/components/instant_heading.tsx +++ b/packages/instant/src/components/instant_heading.tsx @@ -32,7 +32,7 @@ export class InstantHeading extends React.Component<InstantHeadingProps, {}> { public render(): React.ReactNode { const iconOrAmounts = this._renderIcon() || this._renderAmountsSection(); return ( - <Container backgroundColor={ColorOption.primaryColor} padding="20px" width="100%"> + <Container backgroundColor={ColorOption.primaryColor} width="100%" padding="20px"> <Container marginBottom="5px"> <Text letterSpacing="1px" diff --git a/packages/instant/src/components/order_details.tsx b/packages/instant/src/components/order_details.tsx index 5fc956e1c..a8e0e2513 100644 --- a/packages/instant/src/components/order_details.tsx +++ b/packages/instant/src/components/order_details.tsx @@ -34,7 +34,7 @@ export class OrderDetails extends React.Component<OrderDetailsProps> { ? assetEthBaseUnitAmount.div(selectedAssetUnitAmount).ceil() : undefined; return ( - <Container padding="20px" width="100%" flexGrow={1}> + <Container width="100%" flexGrow={1} padding="20px 20px 0px 20px"> <Container marginBottom="10px"> <Text letterSpacing="1px" diff --git a/packages/instant/src/components/payment_method.tsx b/packages/instant/src/components/payment_method.tsx index 4efe5b28e..7c93f1d1c 100644 --- a/packages/instant/src/components/payment_method.tsx +++ b/packages/instant/src/components/payment_method.tsx @@ -26,7 +26,7 @@ export interface PaymentMethodProps { export class PaymentMethod extends React.Component<PaymentMethodProps> { public render(): React.ReactNode { return ( - <Container padding="20px" width="100%" height="133px"> + <Container width="100%" height="120px" padding="20px 20px 0px 20px"> <Container marginBottom="12px"> <Flex justify="space-between"> <Text @@ -88,10 +88,14 @@ export class PaymentMethod extends React.Component<PaymentMethodProps> { return ( <WalletPrompt onClick={this.props.onUnlockWalletClick} - image={<Icon width={13} icon="lock" color={ColorOption.black} />} + image={ + <Container position="relative" top="2px"> + <Icon width={13} icon="lock" color={ColorOption.black} /> + </Container> + } {...colors} > - Please Unlock {this.props.walletDisplayName} + Click to Connect {this.props.walletDisplayName} </WalletPrompt> ); case AccountState.None: diff --git a/packages/instant/src/components/scaling_amount_input.tsx b/packages/instant/src/components/scaling_amount_input.tsx index 0861bbe05..86aca5a65 100644 --- a/packages/instant/src/components/scaling_amount_input.tsx +++ b/packages/instant/src/components/scaling_amount_input.tsx @@ -4,6 +4,7 @@ import * as React from 'react'; import { Maybe } from '../types'; +import { GIT_SHA, MAGIC_TRIGGER_ERROR_INPUT, MAGIC_TRIGGER_ERROR_MESSAGE, NPM_PACKAGE_VERSION } from '../constants'; import { ColorOption } from '../style/theme'; import { maybeBigNumberUtil } from '../util/maybe_big_number'; import { util } from '../util/util'; @@ -71,6 +72,10 @@ export class ScalingAmountInput extends React.Component<ScalingAmountInputProps, ); } private readonly _handleChange = (event: React.ChangeEvent<HTMLInputElement>): void => { + if (event.target.value === MAGIC_TRIGGER_ERROR_INPUT) { + throw new Error(`${MAGIC_TRIGGER_ERROR_MESSAGE} git: ${GIT_SHA}, npm: ${NPM_PACKAGE_VERSION}`); + } + const sanitizedValue = event.target.value.replace(/[^0-9.]/g, ''); // only allow numbers and "." this.setState({ stringValue: sanitizedValue, diff --git a/packages/instant/src/components/ui/overlay.tsx b/packages/instant/src/components/ui/overlay.tsx index 7d311dc2f..0b5eaf299 100644 --- a/packages/instant/src/components/ui/overlay.tsx +++ b/packages/instant/src/components/ui/overlay.tsx @@ -33,7 +33,7 @@ export const Overlay = Overlay.defaultProps = { zIndex: zIndex.overlayDefault, - backgroundColor: generateOverlayBlack(0.9), + backgroundColor: generateOverlayBlack(0.7), }; Overlay.displayName = 'Overlay'; diff --git a/packages/instant/src/components/wallet_prompt.tsx b/packages/instant/src/components/wallet_prompt.tsx index a0b3ae457..c07cfe7b5 100644 --- a/packages/instant/src/components/wallet_prompt.tsx +++ b/packages/instant/src/components/wallet_prompt.tsx @@ -21,7 +21,7 @@ export const WalletPrompt: React.StatelessComponent<WalletPromptProps> = ({ primaryColor, }) => ( <Container - padding="14.5px" + padding="10px" border={`1px solid ${primaryColor}`} backgroundColor={secondaryColor} width="100%" @@ -33,7 +33,7 @@ export const WalletPrompt: React.StatelessComponent<WalletPromptProps> = ({ <Flex> {image} <Container marginLeft="10px"> - <Text fontSize="16px" fontColor={primaryColor}> + <Text fontSize="16px" fontColor={primaryColor} fontWeight="500"> {children} </Text> </Container> diff --git a/packages/instant/src/components/zero_ex_instant_container.tsx b/packages/instant/src/components/zero_ex_instant_container.tsx index 8a809ee31..0337c7714 100644 --- a/packages/instant/src/components/zero_ex_instant_container.tsx +++ b/packages/instant/src/components/zero_ex_instant_container.tsx @@ -70,7 +70,7 @@ export class ZeroExInstantContainer extends React.Component<ZeroExInstantContain marginTop="10px" marginLeft="auto" marginRight="auto" - width="140px" + width="108px" > <a href={ZERO_EX_SITE_URL} target="_blank"> <PoweredByLogo /> diff --git a/packages/instant/src/components/zero_ex_instant_overlay.tsx b/packages/instant/src/components/zero_ex_instant_overlay.tsx index b3fb57dd6..96e560691 100644 --- a/packages/instant/src/components/zero_ex_instant_overlay.tsx +++ b/packages/instant/src/components/zero_ex_instant_overlay.tsx @@ -1,7 +1,7 @@ import * as React from 'react'; import { ZeroExInstantContainer } from '../components/zero_ex_instant_container'; -import { MAIN_CONTAINER_DIV_CLASS, OVERLAY_DIV_CLASS } from '../constants'; +import { MAIN_CONTAINER_DIV_CLASS, OVERLAY_CLOSE_BUTTON_DIV_CLASS, OVERLAY_DIV_CLASS } from '../constants'; import { ColorOption } from '../style/theme'; import { Container } from './ui/container'; @@ -21,7 +21,13 @@ export const ZeroExInstantOverlay: React.StatelessComponent<ZeroExInstantOverlay <ZeroExInstantProvider {...rest}> <Overlay zIndex={zIndex} className={OVERLAY_DIV_CLASS}> <Flex height="100vh"> - <Container position="absolute" top="0px" right="0px" display={{ default: 'initial', sm: 'none' }}> + <Container + className={OVERLAY_CLOSE_BUTTON_DIV_CLASS} + position="absolute" + top="0px" + right="0px" + display={{ default: 'initial', sm: 'none' }} + > <Icon height={18} width={18} diff --git a/packages/instant/src/constants.ts b/packages/instant/src/constants.ts index 2439c7349..506348092 100644 --- a/packages/instant/src/constants.ts +++ b/packages/instant/src/constants.ts @@ -8,15 +8,20 @@ export const DEFAULT_ZERO_EX_CONTAINER_SELECTOR = '#zeroExInstantContainer'; export const INJECTED_DIV_CLASS = 'zeroExInstantResetRoot'; export const INJECTED_DIV_ID = 'zeroExInstant'; export const OVERLAY_DIV_CLASS = 'zeroExInstantOverlay'; +export const OVERLAY_CLOSE_BUTTON_DIV_CLASS = 'zeroExInstantOverlayCloseButton'; export const MAIN_CONTAINER_DIV_CLASS = 'zeroExInstantMainContainer'; export const WEB_3_WRAPPER_TRANSACTION_FAILED_ERROR_MSG_PREFIX = 'Transaction failed'; export const GWEI_IN_WEI = new BigNumber(1000000000); export const ONE_SECOND_MS = 1000; export const ONE_MINUTE_MS = ONE_SECOND_MS * 60; +export const GIT_SHA = process.env.GIT_SHA; +export const NPM_PACKAGE_VERSION = process.env.NPM_PACKAGE_VERSION; export const ACCOUNT_UPDATE_INTERVAL_TIME_MS = ONE_SECOND_MS * 5; export const BUY_QUOTE_UPDATE_INTERVAL_TIME_MS = ONE_SECOND_MS * 15; export const DEFAULT_GAS_PRICE = GWEI_IN_WEI.mul(6); export const DEFAULT_ESTIMATED_TRANSACTION_TIME_MS = ONE_MINUTE_MS * 2; +export const MAGIC_TRIGGER_ERROR_INPUT = '0€'; +export const MAGIC_TRIGGER_ERROR_MESSAGE = 'Triggered error'; export const ETH_GAS_STATION_API_BASE_URL = 'https://ethgasstation.info'; export const HEAP_ANALYTICS_ID = process.env.HEAP_ANALYTICS_ID; export const HEAP_ENABLED = process.env.HEAP_ENABLED; diff --git a/packages/instant/src/index.umd.ts b/packages/instant/src/index.umd.ts index b92fa3a7c..d172f4145 100644 --- a/packages/instant/src/index.umd.ts +++ b/packages/instant/src/index.umd.ts @@ -2,7 +2,13 @@ import * as _ from 'lodash'; import * as React from 'react'; import * as ReactDOM from 'react-dom'; -import { DEFAULT_ZERO_EX_CONTAINER_SELECTOR, INJECTED_DIV_CLASS, INJECTED_DIV_ID } from './constants'; +import { + DEFAULT_ZERO_EX_CONTAINER_SELECTOR, + GIT_SHA as GIT_SHA_FROM_CONSTANT, + INJECTED_DIV_CLASS, + INJECTED_DIV_ID, + NPM_PACKAGE_VERSION, +} from './constants'; import { ZeroExInstantOverlay, ZeroExInstantOverlayProps } from './index'; import { analytics } from './util/analytics'; import { assert } from './util/assert'; @@ -117,5 +123,5 @@ export const render = (config: ZeroExInstantConfig, selector: string = DEFAULT_Z }; // Write version info to the exported object for debugging -export const GIT_SHA = process.env.GIT_SHA; -export const NPM_VERSION = process.env.NPM_PACKAGE_VERSION; +export const GIT_SHA = GIT_SHA_FROM_CONSTANT; +export const NPM_VERSION = NPM_PACKAGE_VERSION; diff --git a/packages/instant/src/redux/analytics_middleware.ts b/packages/instant/src/redux/analytics_middleware.ts index 47876ca2d..3f7a51707 100644 --- a/packages/instant/src/redux/analytics_middleware.ts +++ b/packages/instant/src/redux/analytics_middleware.ts @@ -1,10 +1,11 @@ +import { AssetProxyId } from '@0x/types'; import { Web3Wrapper } from '@0x/web3-wrapper'; import * as _ from 'lodash'; import { Middleware } from 'redux'; import { ETH_DECIMALS } from '../constants'; import { AccountState, StandardSlidingPanelContent } from '../types'; -import { analytics } from '../util/analytics'; +import { analytics, AnalyticsEventOptions } from '../util/analytics'; import { Action, ActionTypes } from './actions'; @@ -29,9 +30,11 @@ export const analyticsMiddleware: Middleware = store => next => middlewareAction if (didJustTurnReady) { analytics.trackAccountReady(ethAddress); analytics.addUserProperties({ lastKnownEthAddress: ethAddress }); + analytics.addEventProperties({ ethAddress }); } else if (didJustUpdateAddress) { analytics.trackAccountAddressChanged(ethAddress); analytics.addUserProperties({ lastKnownEthAddress: ethAddress }); + analytics.addEventProperties({ ethAddress }); } } break; @@ -51,7 +54,8 @@ export const analyticsMiddleware: Middleware = store => next => middlewareAction curAccount.ethBalanceInWei, ETH_DECIMALS, ).toString(); - analytics.addUserProperties({ ethBalanceInUnitAmount }); + analytics.addUserProperties({ lastEthBalanceInUnitAmount: ethBalanceInUnitAmount }); + analytics.addEventProperties({ ethBalanceInUnitAmount }); } break; case ActionTypes.UPDATE_SELECTED_ASSET: @@ -63,10 +67,16 @@ export const analyticsMiddleware: Middleware = store => next => middlewareAction assetName, assetData, }); - analytics.addEventProperties({ + + const selectedAssetEventProperties: AnalyticsEventOptions = { selectedAssetName: assetName, selectedAssetData: assetData, - }); + }; + if (selectedAsset.metaData.assetProxyId === AssetProxyId.ERC20) { + selectedAssetEventProperties.selectedAssetDecimals = selectedAsset.metaData.decimals; + selectedAssetEventProperties.selectedAssetSymbol = selectedAsset.metaData.symbol; + } + analytics.addEventProperties(selectedAssetEventProperties); } break; case ActionTypes.SET_AVAILABLE_ASSETS: diff --git a/packages/instant/src/style/theme.ts b/packages/instant/src/style/theme.ts index 71e4b7052..fd3f03c3f 100644 --- a/packages/instant/src/style/theme.ts +++ b/packages/instant/src/style/theme.ts @@ -38,8 +38,8 @@ export const theme: Theme = { lightestGrey: '#EEEEEE', darkGrey: '#333333', white: 'white', - lightOrange: '#F9F2ED', - darkOrange: '#F2994C', + lightOrange: '#FFF8F2', + darkOrange: '#F7A24F', green: '#3CB34F', red: '#D00000', darkBlue: '#135df6', diff --git a/packages/instant/src/util/analytics.ts b/packages/instant/src/util/analytics.ts index 760ec8b5c..6da37bedb 100644 --- a/packages/instant/src/util/analytics.ts +++ b/packages/instant/src/util/analytics.ts @@ -2,7 +2,7 @@ import { BuyQuote } from '@0x/asset-buyer'; import { BigNumber } from '@0x/utils'; import * as _ from 'lodash'; -import { HEAP_ENABLED, INSTANT_DISCHARGE_TARGET } from '../constants'; +import { GIT_SHA, HEAP_ENABLED, INSTANT_DISCHARGE_TARGET, NPM_PACKAGE_VERSION } from '../constants'; import { AffiliateInfo, Asset, @@ -97,11 +97,13 @@ const buyQuoteEventProperties = (buyQuote: BuyQuote) => { export interface AnalyticsUserOptions { lastKnownEthAddress?: string; - ethBalanceInUnitAmount?: string; + lastEthBalanceInUnitAmount?: string; } export interface AnalyticsEventOptions { embeddedHost?: string; embeddedUrl?: string; + ethBalanceInUnitAmount?: string; + ethAddress?: string; networkId?: number; providerName?: string; gitSha?: string; @@ -112,7 +114,9 @@ export interface AnalyticsEventOptions { affiliateFeePercent?: number; numberAvailableAssets?: number; selectedAssetName?: string; + selectedAssetSymbol?: string; selectedAssetData?: string; + selectedAssetDecimals?: number; } export enum TokenSelectorClosedVia { ClickedX = 'Clicked X', @@ -145,8 +149,8 @@ export const analytics = { embeddedUrl: window.location.href, networkId: network, providerName: providerState.name, - gitSha: process.env.GIT_SHA, - npmVersion: process.env.NPM_PACKAGE_VERSION, + gitSha: GIT_SHA, + npmVersion: NPM_PACKAGE_VERSION, orderSource: orderSourceName, affiliateAddress, affiliateFeePercent, diff --git a/packages/instant/src/util/error_reporter.ts b/packages/instant/src/util/error_reporter.ts index 3ec7b6daa..b1824eaf9 100644 --- a/packages/instant/src/util/error_reporter.ts +++ b/packages/instant/src/util/error_reporter.ts @@ -1,7 +1,7 @@ import { logUtils } from '@0x/utils'; import * as _ from 'lodash'; -import { HOST_DOMAINS, INSTANT_DISCHARGE_TARGET, ROLLBAR_CLIENT_TOKEN, ROLLBAR_ENABLED } from '../constants'; +import { GIT_SHA, HOST_DOMAINS, INSTANT_DISCHARGE_TARGET, ROLLBAR_CLIENT_TOKEN, ROLLBAR_ENABLED } from '../constants'; // Import version of Rollbar designed for embedded components // See https://docs.rollbar.com/docs/using-rollbarjs-inside-an-embedded-component @@ -24,7 +24,7 @@ export const setupRollbar = (): any => { client: { javascript: { source_map_enabled: true, - code_version: process.env.GIT_SHA, + code_version: GIT_SHA, guess_uncaught_frames: true, }, }, diff --git a/packages/monorepo-scripts/CHANGELOG.json b/packages/monorepo-scripts/CHANGELOG.json index 170a97a33..428168437 100644 --- a/packages/monorepo-scripts/CHANGELOG.json +++ b/packages/monorepo-scripts/CHANGELOG.json @@ -13,6 +13,10 @@ { "note": "Add ForwarderError to the IGNORED_EXCESSIVE_TYPES array", "pr": 1147 + }, + { + "note": "Fix a bug when hardcoded CHANGELOG paths cause fetching release notes to fail", + "pr": 1311 } ] }, diff --git a/packages/monorepo-scripts/src/utils/github_release_utils.ts b/packages/monorepo-scripts/src/utils/github_release_utils.ts index 7434d397e..e63244b46 100644 --- a/packages/monorepo-scripts/src/utils/github_release_utils.ts +++ b/packages/monorepo-scripts/src/utils/github_release_utils.ts @@ -41,7 +41,7 @@ export async function publishReleaseNotesAsync(packagesToPublish: Package[], isD let assets: string[] = []; let aggregateNotes = ''; _.each(packagesToPublish, pkg => { - aggregateNotes += getReleaseNotesForPackage(pkg.packageJson.name); + aggregateNotes += getReleaseNotesForPackage(pkg.location, pkg.packageJson.name); const packageAssets = _.get(pkg.packageJson, 'config.postpublish.assets'); if (!_.isUndefined(packageAssets)) { @@ -88,14 +88,8 @@ function adjustAssetPaths(assets: string[]): string[] { return finalAssets; } -function getReleaseNotesForPackage(packageName: string): string { - const packageNameWithoutNamespace = packageName.replace('@0x/', ''); - const changelogJSONPath = path.join( - constants.monorepoRootPath, - 'packages', - packageNameWithoutNamespace, - 'CHANGELOG.json', - ); +function getReleaseNotesForPackage(packageLocation: string, packageName: string): string { + const changelogJSONPath = path.join(packageLocation, 'CHANGELOG.json'); const changelogJSON = readFileSync(changelogJSONPath, 'utf-8'); const changelogs = JSON.parse(changelogJSON); const latestLog = changelogs[0]; diff --git a/packages/pipeline/.npmignore b/packages/pipeline/.npmignore new file mode 100644 index 000000000..89302c908 --- /dev/null +++ b/packages/pipeline/.npmignore @@ -0,0 +1,7 @@ +.* +yarn-error.log +/scripts/ +/generated_docs/ +/src/ +tsconfig.json +/lib/monorepo_scripts/ diff --git a/packages/pipeline/README.md b/packages/pipeline/README.md new file mode 100644 index 000000000..794488cac --- /dev/null +++ b/packages/pipeline/README.md @@ -0,0 +1,166 @@ +## @0xproject/pipeline + +This repository contains scripts used for scraping data from the Ethereum blockchain into SQL tables for analysis by the 0x team. + +## Contributing + +We strongly recommend that the community help us make improvements and determine the future direction of the protocol. To report bugs within this package, please create an issue in this repository. + +Please read our [contribution guidelines](../../CONTRIBUTING.md) before getting started. + +### Install dependencies: + +```bash +yarn install +``` + +### Build + +```bash +yarn build +``` + +### Clean + +```bash +yarn clean +``` + +### Lint + +```bash +yarn lint +``` + +### Migrations + +Create a new migration: `yarn migrate:create --name MigrationNameInCamelCase` +Run migrations: `yarn migrate:run` +Revert the most recent migration (CAUTION: may result in data loss!): `yarn migrate:revert` + +## Testing + +There are several test scripts in **package.json**. You can run all the tests +with `yarn test:all` or run certain tests seprately by following the +instructions below. Some tests may not work out of the box on certain platforms +or operating systems (see the "Database tests" section below). + +### Unit tests + +The unit tests can be run with `yarn test`. These tests don't depend on any +services or databases and will run in any environment that can run Node. + +### Database tests + +Database integration tests can be run with `yarn test:db`. These tests will +attempt to automatically spin up a Postgres database via Docker. If this doesn't +work you have two other options: + +1. Set the `DOCKER_SOCKET` environment variable to a valid socket path to use + for communicating with Docker. +2. Start Postgres manually and set the `ZEROEX_DATA_PIPELINE_TEST_DB_URL` + environment variable. If this is set, the tests will use your existing + Postgres database instead of trying to create one with Docker. + +## Running locally + +`pipeline` requires access to a PostgreSQL database. The easiest way to start +Postgres is via Docker. Depending on your platform, you may need to prepend +`sudo` to the following command: + +``` +docker run --rm -d -p 5432:5432 --name pipeline_postgres postgres:11-alpine +``` + +This will start a Postgres server with the default username and database name +(`postgres` and `postgres`). You should set the environment variable as follows: + +``` +export ZEROEX_DATA_PIPELINE_DB_URL=postgresql://postgres@localhost/postgres +``` + +First thing you will need to do is run the migrations: + +``` +yarn migrate:run +``` + +Now you can run scripts locally: + +``` +node packages/pipeline/lib/src/scripts/pull_radar_relay_orders.js +``` + +To stop the Postgres server (you may need to add `sudo`): + +``` +docker stop pipeline_postgres +``` + +This will remove all data from the database. + +If you prefer, you can also install Postgres with e.g., +[Homebrew](https://wiki.postgresql.org/wiki/Homebrew) or +[Postgress.app](https://postgresapp.com/). Keep in mind that you will need to +set the`ZEROEX_DATA_PIPELINE_DB_URL` environment variable to a valid +[PostgreSQL connection url](https://stackoverflow.com/questions/3582552/postgresql-connection-url) + +## Directory structure + +``` +. +├── lib: Code generated by the TypeScript compiler. Don't edit this directly. +├── migrations: Code for creating and updating database schemas. +├── node_modules: +├── src: All TypeScript source code. +│ ├── data_sources: Code responsible for getting raw data, typically from a third-party source. +│ ├── entities: TypeORM entities which closely mirror our database schemas. Some other ORMs call these "models". +│ ├── parsers: Code for converting raw data into entities. +│ ├── scripts: Executable scripts which put all the pieces together. +│ └── utils: Various utils used across packages/files. +├── test: All tests go here and are organized in the same way as the folder/file that they test. +``` + +## Adding new data to the pipeline + +1. Create an entity in the _entities_ directory. Entities directly mirror our + database schemas. We follow the practice of having "dumb" entities, so + entity classes should typically not have any methods. +2. Create a migration using the `yarn migrate:create` command. Create/update + tables as needed. Remember to fill in both the `up` and `down` methods. Try + to avoid data loss as much as possible in your migrations. +3. Add basic tests for your entity and migrations to the **test/entities/** + directory. +4. Create a class or function in the **data_sources/** directory for getting + raw data. This code should abstract away pagination and rate-limiting as + much as possible. +5. Create a class or function in the **parsers/** directory for converting the + raw data into an entity. Also add tests in the **tests/** directory to test + the parser. +6. Create an executable script in the **scripts/** directory for putting + everything together. Your script can accept environment variables for things + like API keys. It should pull the data, parse it, and save it to the + database. Scripts should be idempotent and atomic (when possible). What this + means is that your script may be responsible for determining _which_ data + needs to be updated. For example, you may need to query the database to find + the most recent block number that we have already pulled, then pull new data + starting from that block number. +7. Run the migrations and then run your new script locally and verify it works + as expected. + +#### Additional guidelines and tips: + +* Table names should be plural and separated by underscores (e.g., + `exchange_fill_events`). +* Any table which contains data which comes directly from a third-party source + should be namespaced in the `raw` PostgreSQL schema. +* Column names in the database should be separated by underscores (e.g., + `maker_asset_type`). +* Field names in entity classes (like any other fields in TypeScript) should + be camel-cased (e.g., `makerAssetType`). +* All timestamps should be stored as milliseconds since the Unix Epoch. +* Use the `BigNumber` type for TypeScript code which deals with 256-bit + numbers from smart contracts or for any case where we are dealing with large + floating point numbers. +* [TypeORM documentation](http://typeorm.io/#/) is pretty robust and can be a + helpful resource. diff --git a/packages/pipeline/coverage/.gitkeep b/packages/pipeline/coverage/.gitkeep new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/packages/pipeline/coverage/.gitkeep diff --git a/packages/pipeline/migrations/1542070840010-InitialSchema.ts b/packages/pipeline/migrations/1542070840010-InitialSchema.ts new file mode 100644 index 000000000..895f9e6c9 --- /dev/null +++ b/packages/pipeline/migrations/1542070840010-InitialSchema.ts @@ -0,0 +1,187 @@ +import { MigrationInterface, QueryRunner, Table } from 'typeorm'; + +const blocks = new Table({ + name: 'raw.blocks', + columns: [ + { name: 'number', type: 'bigint', isPrimary: true }, + { name: 'hash', type: 'varchar', isPrimary: true }, + { name: 'timestamp', type: 'bigint' }, + ], +}); + +const exchange_cancel_events = new Table({ + name: 'raw.exchange_cancel_events', + columns: [ + { name: 'contract_address', type: 'char(42)', isPrimary: true }, + { name: 'log_index', type: 'integer', isPrimary: true }, + { name: 'block_number', type: 'bigint', isPrimary: true }, + + { name: 'raw_data', type: 'varchar' }, + + { name: 'transaction_hash', type: 'varchar' }, + { name: 'maker_address', type: 'char(42)' }, + { name: 'taker_address', type: 'char(42)' }, + { name: 'fee_recipient_address', type: 'char(42)' }, + { name: 'sender_address', type: 'char(42)' }, + { name: 'order_hash', type: 'varchar' }, + + { name: 'raw_maker_asset_data', type: 'varchar' }, + { name: 'maker_asset_type', type: 'varchar' }, + { name: 'maker_asset_proxy_id', type: 'varchar' }, + { name: 'maker_token_address', type: 'char(42)' }, + { name: 'maker_token_id', type: 'varchar', isNullable: true }, + { name: 'raw_taker_asset_data', type: 'varchar' }, + { name: 'taker_asset_type', type: 'varchar' }, + { name: 'taker_asset_proxy_id', type: 'varchar' }, + { name: 'taker_token_address', type: 'char(42)' }, + { name: 'taker_token_id', type: 'varchar', isNullable: true }, + ], +}); + +const exchange_cancel_up_to_events = new Table({ + name: 'raw.exchange_cancel_up_to_events', + columns: [ + { name: 'contract_address', type: 'char(42)', isPrimary: true }, + { name: 'log_index', type: 'integer', isPrimary: true }, + { name: 'block_number', type: 'bigint', isPrimary: true }, + + { name: 'raw_data', type: 'varchar' }, + + { name: 'transaction_hash', type: 'varchar' }, + { name: 'maker_address', type: 'char(42)' }, + { name: 'sender_address', type: 'char(42)' }, + { name: 'order_epoch', type: 'varchar' }, + ], +}); + +const exchange_fill_events = new Table({ + name: 'raw.exchange_fill_events', + columns: [ + { name: 'contract_address', type: 'char(42)', isPrimary: true }, + { name: 'log_index', type: 'integer', isPrimary: true }, + { name: 'block_number', type: 'bigint', isPrimary: true }, + + { name: 'raw_data', type: 'varchar' }, + + { name: 'transaction_hash', type: 'varchar' }, + { name: 'maker_address', type: 'char(42)' }, + { name: 'taker_address', type: 'char(42)' }, + { name: 'fee_recipient_address', type: 'char(42)' }, + { name: 'sender_address', type: 'char(42)' }, + { name: 'maker_asset_filled_amount', type: 'varchar' }, + { name: 'taker_asset_filled_amount', type: 'varchar' }, + { name: 'maker_fee_paid', type: 'varchar' }, + { name: 'taker_fee_paid', type: 'varchar' }, + { name: 'order_hash', type: 'varchar' }, + + { name: 'raw_maker_asset_data', type: 'varchar' }, + { name: 'maker_asset_type', type: 'varchar' }, + { name: 'maker_asset_proxy_id', type: 'varchar' }, + { name: 'maker_token_address', type: 'char(42)' }, + { name: 'maker_token_id', type: 'varchar', isNullable: true }, + { name: 'raw_taker_asset_data', type: 'varchar' }, + { name: 'taker_asset_type', type: 'varchar' }, + { name: 'taker_asset_proxy_id', type: 'varchar' }, + { name: 'taker_token_address', type: 'char(42)' }, + { name: 'taker_token_id', type: 'varchar', isNullable: true }, + ], +}); + +const relayers = new Table({ + name: 'raw.relayers', + columns: [ + { name: 'uuid', type: 'varchar', isPrimary: true }, + { name: 'name', type: 'varchar' }, + { name: 'sra_http_endpoint', type: 'varchar', isNullable: true }, + { name: 'sra_ws_endpoint', type: 'varchar', isNullable: true }, + { name: 'app_url', type: 'varchar', isNullable: true }, + { name: 'fee_recipient_addresses', type: 'char(42)', isArray: true }, + { name: 'taker_addresses', type: 'char(42)', isArray: true }, + ], +}); + +const sra_orders = new Table({ + name: 'raw.sra_orders', + columns: [ + { name: 'exchange_address', type: 'char(42)', isPrimary: true }, + { name: 'order_hash_hex', type: 'varchar', isPrimary: true }, + + { name: 'source_url', type: 'varchar' }, + { name: 'last_updated_timestamp', type: 'bigint' }, + { name: 'first_seen_timestamp', type: 'bigint' }, + + { name: 'maker_address', type: 'char(42)' }, + { name: 'taker_address', type: 'char(42)' }, + { name: 'fee_recipient_address', type: 'char(42)' }, + { name: 'sender_address', type: 'char(42)' }, + { name: 'maker_asset_filled_amount', type: 'varchar' }, + { name: 'taker_asset_filled_amount', type: 'varchar' }, + { name: 'maker_fee', type: 'varchar' }, + { name: 'taker_fee', type: 'varchar' }, + { name: 'expiration_time_seconds', type: 'int' }, + { name: 'salt', type: 'varchar' }, + { name: 'signature', type: 'varchar' }, + + { name: 'raw_maker_asset_data', type: 'varchar' }, + { name: 'maker_asset_type', type: 'varchar' }, + { name: 'maker_asset_proxy_id', type: 'varchar' }, + { name: 'maker_token_address', type: 'char(42)' }, + { name: 'maker_token_id', type: 'varchar', isNullable: true }, + { name: 'raw_taker_asset_data', type: 'varchar' }, + { name: 'taker_asset_type', type: 'varchar' }, + { name: 'taker_asset_proxy_id', type: 'varchar' }, + { name: 'taker_token_address', type: 'char(42)' }, + { name: 'taker_token_id', type: 'varchar', isNullable: true }, + + { name: 'metadata_json', type: 'varchar' }, + ], +}); + +const token_on_chain_metadata = new Table({ + name: 'raw.token_on_chain_metadata', + columns: [ + { name: 'address', type: 'char(42)', isPrimary: true }, + { name: 'decimals', type: 'integer' }, + { name: 'symbol', type: 'varchar' }, + { name: 'name', type: 'varchar' }, + ], +}); + +const transactions = new Table({ + name: 'raw.transactions', + columns: [ + { name: 'block_number', type: 'bigint', isPrimary: true }, + { name: 'block_hash', type: 'varchar', isPrimary: true }, + { name: 'transaction_hash', type: 'varchar', isPrimary: true }, + { name: 'gas_used', type: 'bigint' }, + { name: 'gas_price', type: 'bigint' }, + ], +}); + +export class InitialSchema1542070840010 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise<any> { + await queryRunner.createSchema('raw'); + + await queryRunner.createTable(blocks); + await queryRunner.createTable(exchange_cancel_events); + await queryRunner.createTable(exchange_cancel_up_to_events); + await queryRunner.createTable(exchange_fill_events); + await queryRunner.createTable(relayers); + await queryRunner.createTable(sra_orders); + await queryRunner.createTable(token_on_chain_metadata); + await queryRunner.createTable(transactions); + } + + public async down(queryRunner: QueryRunner): Promise<any> { + await queryRunner.dropTable(blocks.name); + await queryRunner.dropTable(exchange_cancel_events.name); + await queryRunner.dropTable(exchange_cancel_up_to_events.name); + await queryRunner.dropTable(exchange_fill_events.name); + await queryRunner.dropTable(relayers.name); + await queryRunner.dropTable(sra_orders.name); + await queryRunner.dropTable(token_on_chain_metadata.name); + await queryRunner.dropTable(transactions.name); + + await queryRunner.dropSchema('raw'); + } +} diff --git a/packages/pipeline/migrations/1542147915364-NewSraOrderTimestampFormat.ts b/packages/pipeline/migrations/1542147915364-NewSraOrderTimestampFormat.ts new file mode 100644 index 000000000..5a8f3fec8 --- /dev/null +++ b/packages/pipeline/migrations/1542147915364-NewSraOrderTimestampFormat.ts @@ -0,0 +1,48 @@ +import { MigrationInterface, QueryRunner, Table } from 'typeorm'; + +export class NewSraOrderTimestampFormat1542147915364 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise<any> { + await queryRunner.query( + `ALTER TABLE raw.sra_orders + DROP CONSTRAINT "PK_09bfb9980715329563bd53d667e", + ADD PRIMARY KEY (order_hash_hex, exchange_address, source_url); + `, + ); + + await queryRunner.query( + `CREATE TABLE raw.sra_orders_observed_timestamps ( + order_hash_hex varchar NOT NULL, + exchange_address varchar NOT NULL, + source_url varchar NOT NULL, + observed_timestamp bigint NOT NULL, + FOREIGN KEY + (order_hash_hex, exchange_address, source_url) + REFERENCES raw.sra_orders (order_hash_hex, exchange_address, source_url), + PRIMARY KEY (order_hash_hex, exchange_address, source_url, observed_timestamp) + );`, + ); + + await queryRunner.query( + `ALTER TABLE raw.sra_orders + DROP COLUMN last_updated_timestamp, + DROP COLUMN first_seen_timestamp;`, + ); + } + + public async down(queryRunner: QueryRunner): Promise<any> { + await queryRunner.dropTable('raw.sra_orders_observed_timestamps'); + + await queryRunner.query( + `ALTER TABLE raw.sra_orders + ADD COLUMN last_updated_timestamp bigint NOT NULL DEFAULT 0, + ADD COLUMN first_seen_timestamp bigint NOT NULL DEFAULT 0;`, + ); + + await queryRunner.query( + `ALTER TABLE raw.sra_orders + DROP CONSTRAINT sra_orders_pkey, + ADD CONSTRAINT "PK_09bfb9980715329563bd53d667e" PRIMARY KEY ("exchange_address", "order_hash_hex"); + `, + ); + } +} diff --git a/packages/pipeline/migrations/1542152278484-RenameSraOrdersFilledAmounts.ts b/packages/pipeline/migrations/1542152278484-RenameSraOrdersFilledAmounts.ts new file mode 100644 index 000000000..a13e3efa5 --- /dev/null +++ b/packages/pipeline/migrations/1542152278484-RenameSraOrdersFilledAmounts.ts @@ -0,0 +1,13 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class RenameSraOrdersFilledAmounts1542152278484 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise<any> { + await queryRunner.renameColumn('raw.sra_orders', 'maker_asset_filled_amount', 'maker_asset_amount'); + await queryRunner.renameColumn('raw.sra_orders', 'taker_asset_filled_amount', 'taker_asset_amount'); + } + + public async down(queryRunner: QueryRunner): Promise<any> { + await queryRunner.renameColumn('raw.sra_orders', 'maker_asset_amount', 'maker_asset_filled_amount'); + await queryRunner.renameColumn('raw.sra_orders', 'taker_asset_amount', 'taker_asset_filled_amount'); + } +} diff --git a/packages/pipeline/migrations/1542234704666-ConvertBigNumberToNumeric.ts b/packages/pipeline/migrations/1542234704666-ConvertBigNumberToNumeric.ts new file mode 100644 index 000000000..5200ef7cc --- /dev/null +++ b/packages/pipeline/migrations/1542234704666-ConvertBigNumberToNumeric.ts @@ -0,0 +1,53 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class ConvertBigNumberToNumeric1542234704666 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise<any> { + await queryRunner.query( + `ALTER TABLE raw.exchange_fill_events + ALTER COLUMN maker_asset_filled_amount TYPE numeric USING maker_asset_filled_amount::numeric, + ALTER COLUMN taker_asset_filled_amount TYPE numeric USING taker_asset_filled_amount::numeric, + ALTER COLUMN maker_fee_paid TYPE numeric USING maker_fee_paid::numeric, + ALTER COLUMN taker_fee_paid TYPE numeric USING taker_fee_paid::numeric;`, + ); + + await queryRunner.query( + `ALTER TABLE raw.exchange_cancel_up_to_events + ALTER COLUMN order_epoch TYPE numeric USING order_epoch::numeric;`, + ); + + await queryRunner.query( + `ALTER TABLE raw.sra_orders + ALTER COLUMN maker_asset_amount TYPE numeric USING maker_asset_amount::numeric, + ALTER COLUMN taker_asset_amount TYPE numeric USING taker_asset_amount::numeric, + ALTER COLUMN maker_fee TYPE numeric USING maker_fee::numeric, + ALTER COLUMN taker_fee TYPE numeric USING taker_fee::numeric, + ALTER COLUMN expiration_time_seconds TYPE numeric USING expiration_time_seconds::numeric, + ALTER COLUMN salt TYPE numeric USING salt::numeric;`, + ); + } + + public async down(queryRunner: QueryRunner): Promise<any> { + await queryRunner.query( + `ALTER TABLE raw.sra_orders + ALTER COLUMN maker_asset_amount TYPE varchar USING maker_asset_amount::varchar, + ALTER COLUMN taker_asset_amount TYPE varchar USING taker_asset_amount::varchar, + ALTER COLUMN maker_fee TYPE varchar USING maker_fee::varchar, + ALTER COLUMN taker_fee TYPE varchar USING taker_fee::varchar, + ALTER COLUMN expiration_time_seconds TYPE varchar USING expiration_time_seconds::varchar, + ALTER COLUMN salt TYPE varchar USING salt::varchar;`, + ); + + await queryRunner.query( + `ALTER TABLE raw.exchange_cancel_up_to_events + ALTER COLUMN order_epoch TYPE varchar USING order_epoch::varchar;`, + ); + + await queryRunner.query( + `ALTER TABLE raw.exchange_fill_events + ALTER COLUMN maker_asset_filled_amount TYPE varchar USING maker_asset_filled_amount::varchar, + ALTER COLUMN taker_asset_filled_amount TYPE varchar USING taker_asset_filled_amount::varchar, + ALTER COLUMN maker_fee_paid TYPE varchar USING maker_fee_paid::varchar, + ALTER COLUMN taker_fee_paid TYPE varchar USING taker_fee_paid::varchar;`, + ); + } +} diff --git a/packages/pipeline/migrations/1542249766882-AddHomepageUrlToRelayers.ts b/packages/pipeline/migrations/1542249766882-AddHomepageUrlToRelayers.ts new file mode 100644 index 000000000..9a4811ad5 --- /dev/null +++ b/packages/pipeline/migrations/1542249766882-AddHomepageUrlToRelayers.ts @@ -0,0 +1,14 @@ +import { MigrationInterface, QueryRunner, TableColumn } from 'typeorm'; + +export class AddHomepageUrlToRelayers1542249766882 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise<any> { + await queryRunner.addColumn( + 'raw.relayers', + new TableColumn({ name: 'homepage_url', type: 'varchar', default: `'unknown'` }), + ); + } + + public async down(queryRunner: QueryRunner): Promise<any> { + await queryRunner.dropColumn('raw.relayers', 'homepage_url'); + } +} diff --git a/packages/pipeline/migrations/1542401122477-MakeTakerAddressNullable.ts b/packages/pipeline/migrations/1542401122477-MakeTakerAddressNullable.ts new file mode 100644 index 000000000..957c85a36 --- /dev/null +++ b/packages/pipeline/migrations/1542401122477-MakeTakerAddressNullable.ts @@ -0,0 +1,17 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class MakeTakerAddressNullable1542401122477 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise<any> { + await queryRunner.query( + `ALTER TABLE raw.exchange_cancel_events + ALTER COLUMN taker_address DROP NOT NULL;`, + ); + } + + public async down(queryRunner: QueryRunner): Promise<any> { + await queryRunner.query( + `ALTER TABLE raw.exchange_cancel_events + ALTER COLUMN taker_address SET NOT NULL;`, + ); + } +} diff --git a/packages/pipeline/migrations/1542655823221-NewMetadataAndOHLCVTables.ts b/packages/pipeline/migrations/1542655823221-NewMetadataAndOHLCVTables.ts new file mode 100644 index 000000000..838f5ba9c --- /dev/null +++ b/packages/pipeline/migrations/1542655823221-NewMetadataAndOHLCVTables.ts @@ -0,0 +1,60 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class NewMetadataAndOHLCVTables1542655823221 implements MigrationInterface { + // tslint:disable-next-line + public async up(queryRunner: QueryRunner): Promise<any> { + await queryRunner.query(` + CREATE TABLE raw.token_metadata ( + address VARCHAR NOT NULL, + authority VARCHAR NOT NULL, + decimals INT NULL, + symbol VARCHAR NULL, + name VARCHAR NULL, + + PRIMARY KEY (address, authority) + ); + `); + + await queryRunner.dropTable('raw.token_on_chain_metadata'); + + await queryRunner.query(` + CREATE TABLE raw.ohlcv_external ( + exchange VARCHAR NOT NULL, + from_symbol VARCHAR NOT NULL, + to_symbol VARCHAR NOT NULL, + start_time BIGINT NOT NULL, + end_time BIGINT NOT NULL, + + open DOUBLE PRECISION NOT NULL, + close DOUBLE PRECISION NOT NULL, + low DOUBLE PRECISION NOT NULL, + high DOUBLE PRECISION NOT NULL, + volume_from DOUBLE PRECISION NOT NULL, + volume_to DOUBLE PRECISION NOT NULL, + + source VARCHAR NOT NULL, + observed_timestamp BIGINT NOT NULL, + + PRIMARY KEY (exchange, from_symbol, to_symbol, start_time, end_time, source, observed_timestamp) + ); + `); + } + + // tslint:disable-next-line + public async down(queryRunner: QueryRunner): Promise<any> { + await queryRunner.query(` + CREATE TABLE raw.token_on_chain_metadata ( + address VARCHAR NOT NULL, + decimals INT NULL, + symbol VARCHAR NULL, + name VARCHAR NULL, + + PRIMARY KEY (address) + ); + `); + + await queryRunner.dropTable('raw.token_metadata'); + + await queryRunner.dropTable('raw.ohlcv_external'); + } +} diff --git a/packages/pipeline/migrations/1543434472116-TokenOrderbookSnapshots.ts b/packages/pipeline/migrations/1543434472116-TokenOrderbookSnapshots.ts new file mode 100644 index 000000000..a7117c753 --- /dev/null +++ b/packages/pipeline/migrations/1543434472116-TokenOrderbookSnapshots.ts @@ -0,0 +1,30 @@ +import { MigrationInterface, QueryRunner, Table } from 'typeorm'; + +const tokenOrderbookSnapshots = new Table({ + name: 'raw.token_orderbook_snapshots', + columns: [ + { name: 'observed_timestamp', type: 'bigint', isPrimary: true }, + { name: 'source', type: 'varchar', isPrimary: true }, + { name: 'order_type', type: 'order_t' }, + { name: 'price', type: 'numeric', isPrimary: true }, + + { name: 'base_asset_symbol', type: 'varchar', isPrimary: true }, + { name: 'base_asset_address', type: 'char(42)' }, + { name: 'base_volume', type: 'numeric' }, + + { name: 'quote_asset_symbol', type: 'varchar', isPrimary: true }, + { name: 'quote_asset_address', type: 'char(42)' }, + { name: 'quote_volume', type: 'numeric' }, + ], +}); + +export class TokenOrderbookSnapshots1543434472116 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise<any> { + await queryRunner.query(`CREATE TYPE order_t AS enum('bid', 'ask');`); + await queryRunner.createTable(tokenOrderbookSnapshots); + } + + public async down(queryRunner: QueryRunner): Promise<any> { + await queryRunner.dropTable(tokenOrderbookSnapshots.name); + } +} diff --git a/packages/pipeline/migrations/1543446690436-CreateDexTrades.ts b/packages/pipeline/migrations/1543446690436-CreateDexTrades.ts new file mode 100644 index 000000000..267cf144b --- /dev/null +++ b/packages/pipeline/migrations/1543446690436-CreateDexTrades.ts @@ -0,0 +1,41 @@ +import { MigrationInterface, QueryRunner, Table } from 'typeorm'; + +const dexTrades = new Table({ + name: 'raw.dex_trades', + columns: [ + { name: 'source_url', type: 'varchar', isPrimary: true }, + { name: 'tx_hash', type: 'varchar', isPrimary: true }, + + { name: 'tx_timestamp', type: 'bigint' }, + { name: 'tx_date', type: 'varchar' }, + { name: 'tx_sender', type: 'varchar(42)' }, + { name: 'smart_contract_id', type: 'bigint' }, + { name: 'smart_contract_address', type: 'varchar(42)' }, + { name: 'contract_type', type: 'varchar' }, + { name: 'maker', type: 'varchar(42)' }, + { name: 'taker', type: 'varchar(42)' }, + { name: 'amount_buy', type: 'numeric' }, + { name: 'maker_fee_amount', type: 'numeric' }, + { name: 'buy_currency_id', type: 'bigint' }, + { name: 'buy_symbol', type: 'varchar' }, + { name: 'amount_sell', type: 'numeric' }, + { name: 'taker_fee_amount', type: 'numeric' }, + { name: 'sell_currency_id', type: 'bigint' }, + { name: 'sell_symbol', type: 'varchar' }, + { name: 'maker_annotation', type: 'varchar' }, + { name: 'taker_annotation', type: 'varchar' }, + { name: 'protocol', type: 'varchar' }, + { name: 'buy_address', type: 'varchar(42)', isNullable: true }, + { name: 'sell_address', type: 'varchar(42)', isNullable: true }, + ], +}); + +export class CreateDexTrades1543446690436 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise<any> { + await queryRunner.createTable(dexTrades); + } + + public async down(queryRunner: QueryRunner): Promise<any> { + await queryRunner.dropTable(dexTrades); + } +} diff --git a/packages/pipeline/migrations/1543980079179-ConvertTokenMetadataDecimalsToBigNumber.ts b/packages/pipeline/migrations/1543980079179-ConvertTokenMetadataDecimalsToBigNumber.ts new file mode 100644 index 000000000..351bc7eb8 --- /dev/null +++ b/packages/pipeline/migrations/1543980079179-ConvertTokenMetadataDecimalsToBigNumber.ts @@ -0,0 +1,17 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class ConvertTokenMetadataDecimalsToBigNumber1543980079179 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise<any> { + await queryRunner.query( + `ALTER TABLE raw.token_metadata + ALTER COLUMN decimals TYPE numeric USING decimals::numeric;`, + ); + } + + public async down(queryRunner: QueryRunner): Promise<any> { + await queryRunner.query( + `ALTER TABLE raw.token_metadata + ALTER COLUMN decimals TYPE numeric USING decimals::integer;`, + ); + } +} diff --git a/packages/pipeline/migrations/1543983324954-ConvertTransactionGasPriceToBigNumber.ts b/packages/pipeline/migrations/1543983324954-ConvertTransactionGasPriceToBigNumber.ts new file mode 100644 index 000000000..dcb0fd727 --- /dev/null +++ b/packages/pipeline/migrations/1543983324954-ConvertTransactionGasPriceToBigNumber.ts @@ -0,0 +1,19 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class ConvertTransactionGasPriceToBigNumber1543983324954 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise<any> { + await queryRunner.query( + `ALTER TABLE raw.transactions + ALTER COLUMN gas_price TYPE numeric USING gas_price::numeric, + ALTER COLUMN gas_used TYPE numeric USING gas_used::numeric;`, + ); + } + + public async down(queryRunner: QueryRunner): Promise<any> { + await queryRunner.query( + `ALTER TABLE raw.transactions + ALTER COLUMN gas_price TYPE numeric USING gas_price::bigint, + ALTER COLUMN gas_used TYPE numeric USING gas_used::bigint;`, + ); + } +} diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json new file mode 100644 index 000000000..0539618d4 --- /dev/null +++ b/packages/pipeline/package.json @@ -0,0 +1,65 @@ +{ + "name": "@0x/pipeline", + "version": "1.0.0", + "private": true, + "description": "Data pipeline for offline analysis", + "scripts": { + "build": "yarn tsc -b", + "build:ci": "yarn build", + "test": "yarn run_mocha", + "rebuild_and_test": "run-s build test:all", + "test:db": "yarn run_mocha:db", + "test:all": "run-s test test:db", + "test:circleci": "yarn test:coverage", + "run_mocha": "mocha --require source-map-support/register --require make-promises-safe 'lib/test/!(entities)/**/*_test.js' --bail --exit", + "run_mocha:db": "mocha --require source-map-support/register --require make-promises-safe lib/test/db_global_hooks.js 'lib/test/entities/*_test.js' --bail --exit --timeout 60000", + "test:coverage": "nyc npm run test:all --all && yarn coverage:report:lcov", + "coverage:report:lcov": "nyc report --reporter=text-lcov > coverage/lcov.info", + "clean": "shx rm -rf lib", + "lint": "tslint --project . --format stylish --exclude ./migrations/**/*", + "migrate:run": "yarn typeorm migration:run --config ./lib/src/ormconfig", + "migrate:revert": "yarn typeorm migration:revert --config ./lib/src/ormconfig", + "migrate:create": "yarn typeorm migration:create --config ./lib/src/ormconfig --dir migrations" + }, + "repository": { + "type": "git", + "url": "https://github.com/0xProject/0x-monorepo" + }, + "license": "Apache-2.0", + "devDependencies": { + "@0x/tslint-config": "^1.0.9", + "@types/axios": "^0.14.0", + "@types/ramda": "^0.25.38", + "chai": "^4.1.2", + "chai-as-promised": "^7.1.1", + "chai-bignumber": "^2.0.2", + "dirty-chai": "^2.0.1", + "mocha": "^5.2.0", + "tslint": "5.11.0", + "typescript": "3.0.1" + }, + "dependencies": { + "@0x/connect": "^3.0.2", + "@0x/contract-artifacts": "^1.0.1", + "@0x/contract-wrappers": "^3.0.0", + "@0x/dev-utils": "^1.0.13", + "@0x/order-utils": "^2.0.0", + "@0x/subproviders": "^2.1.0", + "@0x/types": "^1.2.0", + "@0x/utils": "^2.0.3", + "@0x/web3-wrapper": "^3.1.0", + "@types/dockerode": "^2.5.9", + "@types/p-limit": "^2.0.0", + "async-parallel": "^1.2.3", + "axios": "^0.18.0", + "dockerode": "^2.5.7", + "ethereum-types": "^1.0.6", + "p-limit": "^2.0.0", + "pg": "^7.5.0", + "prettier": "^1.15.3", + "ramda": "^0.25.0", + "reflect-metadata": "^0.1.12", + "sqlite3": "^4.0.2", + "typeorm": "^0.2.7" + } +} diff --git a/packages/pipeline/src/data_sources/bloxy/index.ts b/packages/pipeline/src/data_sources/bloxy/index.ts new file mode 100644 index 000000000..31cd5bfd6 --- /dev/null +++ b/packages/pipeline/src/data_sources/bloxy/index.ts @@ -0,0 +1,133 @@ +import axios from 'axios'; +import * as R from 'ramda'; + +// URL to use for getting dex trades from Bloxy. +export const BLOXY_DEX_TRADES_URL = 'https://bloxy.info/api/dex/trades'; +// Number of trades to get at once. Must be less than or equal to MAX_OFFSET. +const TRADES_PER_QUERY = 10000; +// Maximum offset supported by the Bloxy API. +const MAX_OFFSET = 100000; +// Buffer to subtract from offset. This means we will request some trades twice +// but we have less chance on missing out on any data. +const OFFSET_BUFFER = 1000; +// Maximum number of days supported by the Bloxy API. +const MAX_DAYS = 30; +// Buffer used for comparing the last seen timestamp to the last returned +// timestamp. Increasing this reduces chances of data loss but also creates more +// redundancy and can impact performance. +// tslint:disable-next-line:custom-no-magic-numbers +const LAST_SEEN_TIMESTAMP_BUFFER_MS = 1000 * 60 * 30; // 30 minutes + +// tslint:disable-next-line:custom-no-magic-numbers +const millisecondsPerDay = 1000 * 60 * 60 * 24; // ms/d = ms/s * s/m * m/h * h/d + +export interface BloxyTrade { + tx_hash: string; + tx_time: string; + tx_date: string; + tx_sender: string; + smart_contract_id: number; + smart_contract_address: string; + contract_type: string; + maker: string; + taker: string; + amountBuy: number; + makerFee: number; + buyCurrencyId: number; + buySymbol: string; + amountSell: number; + takerFee: number; + sellCurrencyId: number; + sellSymbol: string; + maker_annotation: string; + taker_annotation: string; + protocol: string; + buyAddress: string | null; + sellAddress: string | null; +} + +interface BloxyError { + error: string; +} + +type BloxyResponse<T> = T | BloxyError; +type BloxyTradeResponse = BloxyResponse<BloxyTrade[]>; + +function isError<T>(response: BloxyResponse<T>): response is BloxyError { + return (response as BloxyError).error !== undefined; +} + +export class BloxySource { + private readonly _apiKey: string; + + constructor(apiKey: string) { + this._apiKey = apiKey; + } + + /** + * Gets all latest trades between the lastSeenTimestamp (minus some buffer) + * and the current time. Note that because the Bloxy API has some hard + * limits it might not always be possible to get *all* the trades in the + * desired time range. + * @param lastSeenTimestamp The latest timestamp for trades that have + * already been seen. + */ + public async getDexTradesAsync(lastSeenTimestamp: number): Promise<BloxyTrade[]> { + let allTrades: BloxyTrade[] = []; + + // Clamp numberOfDays so that it is always between 1 and MAX_DAYS (inclusive) + const numberOfDays = R.clamp(1, MAX_DAYS, getDaysSinceTimestamp(lastSeenTimestamp)); + + // Keep getting trades until we hit one of the following conditions: + // + // 1. Offset hits MAX_OFFSET (we can't go back any further). + // 2. There are no more trades in the response. + // 3. We see a tx_time equal to or earlier than lastSeenTimestamp (plus + // some buffer). + // + for (let offset = 0; offset <= MAX_OFFSET; offset += TRADES_PER_QUERY - OFFSET_BUFFER) { + const trades = await this._getTradesWithOffsetAsync(numberOfDays, offset); + if (trades.length === 0) { + // There are no more trades left for the days we are querying. + // This means we are done. + return filterDuplicateTrades(allTrades); + } + const sortedTrades = R.reverse(R.sortBy(trade => trade.tx_time, trades)); + allTrades = allTrades.concat(sortedTrades); + + // Check if lastReturnedTimestamp < lastSeenTimestamp + const lastReturnedTimestamp = new Date(sortedTrades[0].tx_time).getTime(); + if (lastReturnedTimestamp < lastSeenTimestamp - LAST_SEEN_TIMESTAMP_BUFFER_MS) { + // We are at the point where we have already seen trades for the + // timestamp range that is being returned. We're done. + return filterDuplicateTrades(allTrades); + } + } + return filterDuplicateTrades(allTrades); + } + + private async _getTradesWithOffsetAsync(numberOfDays: number, offset: number): Promise<BloxyTrade[]> { + const resp = await axios.get<BloxyTradeResponse>(BLOXY_DEX_TRADES_URL, { + params: { + key: this._apiKey, + days: numberOfDays, + limit: TRADES_PER_QUERY, + offset, + }, + }); + if (isError(resp.data)) { + throw new Error('Error in Bloxy API response: ' + resp.data.error); + } + return resp.data; + } +} + +// Computes the number of days between the given timestamp and the current +// timestamp (rounded up). +function getDaysSinceTimestamp(timestamp: number): number { + const msSinceTimestamp = Date.now() - timestamp; + const daysSinceTimestamp = msSinceTimestamp / millisecondsPerDay; + return Math.ceil(daysSinceTimestamp); +} + +const filterDuplicateTrades = R.uniqBy((trade: BloxyTrade) => trade.tx_hash); diff --git a/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts b/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts new file mode 100644 index 000000000..1717eb8b3 --- /dev/null +++ b/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts @@ -0,0 +1,85 @@ +import { + ContractWrappers, + ExchangeCancelEventArgs, + ExchangeCancelUpToEventArgs, + ExchangeEventArgs, + ExchangeEvents, + ExchangeFillEventArgs, + ExchangeWrapper, +} from '@0x/contract-wrappers'; +import { Web3ProviderEngine } from '@0x/subproviders'; +import { Web3Wrapper } from '@0x/web3-wrapper'; +import { LogWithDecodedArgs } from 'ethereum-types'; + +import { EXCHANGE_START_BLOCK } from '../../utils'; + +const BLOCK_FINALITY_THRESHOLD = 10; // When to consider blocks as final. Used to compute default toBlock. +const NUM_BLOCKS_PER_QUERY = 20000; // Number of blocks to query for events at a time. + +export class ExchangeEventsSource { + private readonly _exchangeWrapper: ExchangeWrapper; + private readonly _web3Wrapper: Web3Wrapper; + constructor(provider: Web3ProviderEngine, networkId: number) { + this._web3Wrapper = new Web3Wrapper(provider); + const contractWrappers = new ContractWrappers(provider, { networkId }); + this._exchangeWrapper = contractWrappers.exchange; + } + + public async getFillEventsAsync( + fromBlock?: number, + toBlock?: number, + ): Promise<Array<LogWithDecodedArgs<ExchangeFillEventArgs>>> { + return this._getEventsAsync<ExchangeFillEventArgs>(ExchangeEvents.Fill, fromBlock, toBlock); + } + + public async getCancelEventsAsync( + fromBlock?: number, + toBlock?: number, + ): Promise<Array<LogWithDecodedArgs<ExchangeCancelEventArgs>>> { + return this._getEventsAsync<ExchangeCancelEventArgs>(ExchangeEvents.Cancel, fromBlock, toBlock); + } + + public async getCancelUpToEventsAsync( + fromBlock?: number, + toBlock?: number, + ): Promise<Array<LogWithDecodedArgs<ExchangeCancelUpToEventArgs>>> { + return this._getEventsAsync<ExchangeCancelUpToEventArgs>(ExchangeEvents.CancelUpTo, fromBlock, toBlock); + } + + private async _getEventsAsync<ArgsType extends ExchangeEventArgs>( + eventName: ExchangeEvents, + fromBlock: number = EXCHANGE_START_BLOCK, + toBlock?: number, + ): Promise<Array<LogWithDecodedArgs<ArgsType>>> { + const calculatedToBlock = + toBlock === undefined + ? (await this._web3Wrapper.getBlockNumberAsync()) - BLOCK_FINALITY_THRESHOLD + : toBlock; + let events: Array<LogWithDecodedArgs<ArgsType>> = []; + for (let currFromBlock = fromBlock; currFromBlock <= calculatedToBlock; currFromBlock += NUM_BLOCKS_PER_QUERY) { + events = events.concat( + await this._getEventsForRangeAsync<ArgsType>( + eventName, + currFromBlock, + Math.min(currFromBlock + NUM_BLOCKS_PER_QUERY - 1, calculatedToBlock), + ), + ); + } + return events; + } + + private async _getEventsForRangeAsync<ArgsType extends ExchangeEventArgs>( + eventName: ExchangeEvents, + fromBlock: number, + toBlock: number, + ): Promise<Array<LogWithDecodedArgs<ArgsType>>> { + return this._exchangeWrapper.getLogsAsync<ArgsType>( + eventName, + { + fromBlock, + toBlock, + }, + {}, + ); + } +} diff --git a/packages/pipeline/src/data_sources/ddex/index.ts b/packages/pipeline/src/data_sources/ddex/index.ts new file mode 100644 index 000000000..2bbd8c29b --- /dev/null +++ b/packages/pipeline/src/data_sources/ddex/index.ts @@ -0,0 +1,78 @@ +import { fetchAsync, logUtils } from '@0x/utils'; + +const DDEX_BASE_URL = 'https://api.ddex.io/v2'; +const ACTIVE_MARKETS_URL = `${DDEX_BASE_URL}/markets`; +const NO_AGGREGATION_LEVEL = 3; // See https://docs.ddex.io/#get-orderbook +const ORDERBOOK_ENDPOINT = `/orderbook?level=${NO_AGGREGATION_LEVEL}`; +export const DDEX_SOURCE = 'ddex'; + +export interface DdexActiveMarketsResponse { + status: number; + desc: string; + data: { + markets: DdexMarket[]; + }; +} + +export interface DdexMarket { + id: string; + quoteToken: string; + quoteTokenDecimals: number; + quoteTokenAddress: string; + baseToken: string; + baseTokenDecimals: number; + baseTokenAddress: string; + minOrderSize: string; + maxOrderSize: string; + pricePrecision: number; + priceDecimals: number; + amountDecimals: number; +} + +export interface DdexOrderbookResponse { + status: number; + desc: string; + data: { + orderBook: DdexOrderbook; + }; +} + +export interface DdexOrderbook { + marketId: string; + bids: DdexOrder[]; + asks: DdexOrder[]; +} + +export interface DdexOrder { + price: string; + amount: string; + orderId: string; +} + +// tslint:disable:prefer-function-over-method +// ^ Keep consistency with other sources and help logical organization +export class DdexSource { + /** + * Call Ddex API to find out which markets they are maintaining orderbooks for. + */ + public async getActiveMarketsAsync(): Promise<DdexMarket[]> { + logUtils.log('Getting all active DDEX markets'); + const resp = await fetchAsync(ACTIVE_MARKETS_URL); + const respJson: DdexActiveMarketsResponse = await resp.json(); + const markets = respJson.data.markets; + logUtils.log(`Got ${markets.length} markets.`); + return markets; + } + + /** + * Retrieve orderbook from Ddex API for a given market. + * @param marketId String identifying the market we want data for. Eg. 'REP/AUG' + */ + public async getMarketOrderbookAsync(marketId: string): Promise<DdexOrderbook> { + logUtils.log(`${marketId}: Retrieving orderbook.`); + const marketOrderbookUrl = `${ACTIVE_MARKETS_URL}/${marketId}${ORDERBOOK_ENDPOINT}`; + const resp = await fetchAsync(marketOrderbookUrl); + const respJson: DdexOrderbookResponse = await resp.json(); + return respJson.data.orderBook; + } +} diff --git a/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts b/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts new file mode 100644 index 000000000..6b10c29c5 --- /dev/null +++ b/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts @@ -0,0 +1,94 @@ +// tslint:disable:no-duplicate-imports +import { fetchAsync } from '@0x/utils'; +import promiseLimit = require('p-limit'); +import { stringify } from 'querystring'; +import * as R from 'ramda'; + +import { TradingPair } from '../../utils/get_ohlcv_trading_pairs'; + +export interface CryptoCompareOHLCVResponse { + Data: Map<string, CryptoCompareOHLCVRecord[]>; + Response: string; + Message: string; +} + +export interface CryptoCompareOHLCVRecord { + time: number; // in seconds, not milliseconds + close: number; + high: number; + low: number; + open: number; + volumefrom: number; + volumeto: number; +} + +export interface CryptoCompareOHLCVParams { + fsym: string; + tsym: string; + e?: string; + aggregate?: string; + aggregatePredictableTimePeriods?: boolean; + limit?: number; + toTs?: number; +} + +const ONE_WEEK = 7 * 24 * 60 * 60 * 1000; // tslint:disable-line:custom-no-magic-numbers +const ONE_HOUR = 60 * 60 * 1000; // tslint:disable-line:custom-no-magic-numbers +const ONE_SECOND = 1000; +const HTTP_OK_STATUS = 200; + +export class CryptoCompareOHLCVSource { + public readonly interval = ONE_WEEK; // the hourly API returns data for one week at a time + public readonly default_exchange = 'CCCAGG'; + public readonly intervalBetweenRecords = ONE_HOUR; + private readonly _url: string = 'https://min-api.cryptocompare.com/data/histohour?'; + + // rate-limit for all API calls through this class instance + private readonly _promiseLimit: (fetchFn: () => Promise<Response>) => Promise<Response>; + constructor(maxConcurrentRequests: number = 50) { + this._promiseLimit = promiseLimit(maxConcurrentRequests); + } + + // gets OHLCV records starting from pair.latest + public async getHourlyOHLCVAsync(pair: TradingPair): Promise<CryptoCompareOHLCVRecord[]> { + const params = { + e: this.default_exchange, + fsym: pair.fromSymbol, + tsym: pair.toSymbol, + toTs: Math.floor((pair.latestSavedTime + this.interval) / ONE_SECOND), // CryptoCompare uses timestamp in seconds. not ms + }; + const url = this._url + stringify(params); + + // go through the instance-wide rate-limit + const fetchPromise: Promise<Response> = this._promiseLimit(() => { + // tslint:disable-next-line:no-console + console.log(`Scraping Crypto Compare at ${url}`); + return fetchAsync(url); + }); + + const response = await Promise.resolve(fetchPromise); + if (response.status !== HTTP_OK_STATUS) { + // tslint:disable-next-line:no-console + console.log(`Error scraping ${url}`); + return []; + } + const json: CryptoCompareOHLCVResponse = await response.json(); + if (json.Response === 'Error' || Object.keys(json.Data).length === 0) { + // tslint:disable-next-line:no-console + console.log(`Error scraping ${url}: ${json.Message}`); + return []; + } + return Object.values(json.Data).filter(rec => rec.time * ONE_SECOND >= pair.latestSavedTime); + } + public generateBackfillIntervals(pair: TradingPair): TradingPair[] { + const now = new Date().getTime(); + const f = (p: TradingPair): false | [TradingPair, TradingPair] => { + if (p.latestSavedTime > now) { + return false; + } else { + return [p, R.merge(p, { latestSavedTime: p.latestSavedTime + this.interval })]; + } + }; + return R.unfold(f, pair); + } +} diff --git a/packages/pipeline/src/data_sources/paradex/index.ts b/packages/pipeline/src/data_sources/paradex/index.ts new file mode 100644 index 000000000..69a03d553 --- /dev/null +++ b/packages/pipeline/src/data_sources/paradex/index.ts @@ -0,0 +1,92 @@ +import { fetchAsync, logUtils } from '@0x/utils'; + +const PARADEX_BASE_URL = 'https://api.paradex.io/consumer/v0'; +const ACTIVE_MARKETS_URL = PARADEX_BASE_URL + '/markets'; +const ORDERBOOK_ENDPOINT = PARADEX_BASE_URL + '/orderbook'; +const TOKEN_INFO_ENDPOINT = PARADEX_BASE_URL + '/tokens'; +export const PARADEX_SOURCE = 'paradex'; + +export type ParadexActiveMarketsResponse = ParadexMarket[]; + +export interface ParadexMarket { + id: string; + symbol: string; + baseToken: string; + quoteToken: string; + minOrderSize: string; + maxOrderSize: string; + priceMaxDecimals: number; + amountMaxDecimals: number; + // These are not native to the Paradex API response. We tag them on later + // by calling the token endpoint and joining on symbol. + baseTokenAddress?: string; + quoteTokenAddress?: string; +} + +export interface ParadexOrderbookResponse { + marketId: number; + marketSymbol: string; + bids: ParadexOrder[]; + asks: ParadexOrder[]; +} + +export interface ParadexOrder { + amount: string; + price: string; +} + +export type ParadexTokenInfoResponse = ParadexTokenInfo[]; + +export interface ParadexTokenInfo { + name: string; + symbol: string; + address: string; +} + +export class ParadexSource { + private readonly _apiKey: string; + + constructor(apiKey: string) { + this._apiKey = apiKey; + } + + /** + * Call Paradex API to find out which markets they are maintaining orderbooks for. + */ + public async getActiveMarketsAsync(): Promise<ParadexActiveMarketsResponse> { + logUtils.log('Getting all active Paradex markets.'); + const resp = await fetchAsync(ACTIVE_MARKETS_URL, { + headers: { 'API-KEY': this._apiKey }, + }); + const markets: ParadexActiveMarketsResponse = await resp.json(); + logUtils.log(`Got ${markets.length} markets.`); + return markets; + } + + /** + * Call Paradex API to find out their token information. + */ + public async getTokenInfoAsync(): Promise<ParadexTokenInfoResponse> { + logUtils.log('Getting token information from Paradex.'); + const resp = await fetchAsync(TOKEN_INFO_ENDPOINT, { + headers: { 'API-KEY': this._apiKey }, + }); + const tokens: ParadexTokenInfoResponse = await resp.json(); + logUtils.log(`Got information for ${tokens.length} tokens.`); + return tokens; + } + + /** + * Retrieve orderbook from Paradex API for a given market. + * @param marketSymbol String representing the market we want data for. + */ + public async getMarketOrderbookAsync(marketSymbol: string): Promise<ParadexOrderbookResponse> { + logUtils.log(`${marketSymbol}: Retrieving orderbook.`); + const marketOrderbookUrl = `${ORDERBOOK_ENDPOINT}?market=${marketSymbol}`; + const resp = await fetchAsync(marketOrderbookUrl, { + headers: { 'API-KEY': this._apiKey }, + }); + const orderbookResponse: ParadexOrderbookResponse = await resp.json(); + return orderbookResponse; + } +} diff --git a/packages/pipeline/src/data_sources/relayer-registry/index.ts b/packages/pipeline/src/data_sources/relayer-registry/index.ts new file mode 100644 index 000000000..8133f5eae --- /dev/null +++ b/packages/pipeline/src/data_sources/relayer-registry/index.ts @@ -0,0 +1,33 @@ +import axios from 'axios'; + +export interface RelayerResponse { + name: string; + homepage_url: string; + app_url: string; + header_img: string; + logo_img: string; + networks: RelayerResponseNetwork[]; +} + +export interface RelayerResponseNetwork { + networkId: number; + sra_http_endpoint?: string; + sra_ws_endpoint?: string; + static_order_fields?: { + fee_recipient_addresses?: string[]; + taker_addresses?: string[]; + }; +} + +export class RelayerRegistrySource { + private readonly _url: string; + + constructor(url: string) { + this._url = url; + } + + public async getRelayerInfoAsync(): Promise<Map<string, RelayerResponse>> { + const resp = await axios.get<Map<string, RelayerResponse>>(this._url); + return resp.data; + } +} diff --git a/packages/pipeline/src/data_sources/trusted_tokens/index.ts b/packages/pipeline/src/data_sources/trusted_tokens/index.ts new file mode 100644 index 000000000..552739fb9 --- /dev/null +++ b/packages/pipeline/src/data_sources/trusted_tokens/index.ts @@ -0,0 +1,29 @@ +import axios from 'axios'; + +export interface ZeroExTrustedTokenMeta { + address: string; + name: string; + symbol: string; + decimals: number; +} + +export interface MetamaskTrustedTokenMeta { + address: string; + name: string; + erc20: boolean; + symbol: string; + decimals: number; +} + +export class TrustedTokenSource<T> { + private readonly _url: string; + + constructor(url: string) { + this._url = url; + } + + public async getTrustedTokenMetaAsync(): Promise<T> { + const resp = await axios.get<T>(this._url); + return resp.data; + } +} diff --git a/packages/pipeline/src/data_sources/web3/index.ts b/packages/pipeline/src/data_sources/web3/index.ts new file mode 100644 index 000000000..45a9ea161 --- /dev/null +++ b/packages/pipeline/src/data_sources/web3/index.ts @@ -0,0 +1,22 @@ +import { Web3ProviderEngine } from '@0x/subproviders'; +import { Web3Wrapper } from '@0x/web3-wrapper'; +import { BlockWithoutTransactionData, Transaction } from 'ethereum-types'; + +export class Web3Source { + private readonly _web3Wrapper: Web3Wrapper; + constructor(provider: Web3ProviderEngine) { + this._web3Wrapper = new Web3Wrapper(provider); + } + + public async getBlockInfoAsync(blockNumber: number): Promise<BlockWithoutTransactionData> { + const block = await this._web3Wrapper.getBlockIfExistsAsync(blockNumber); + if (block == null) { + return Promise.reject(new Error(`Could not find block for given block number: ${blockNumber}`)); + } + return block; + } + + public async getTransactionInfoAsync(txHash: string): Promise<Transaction> { + return this._web3Wrapper.getTransactionByHashAsync(txHash); + } +} diff --git a/packages/pipeline/src/entities/block.ts b/packages/pipeline/src/entities/block.ts new file mode 100644 index 000000000..398946622 --- /dev/null +++ b/packages/pipeline/src/entities/block.ts @@ -0,0 +1,13 @@ +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +import { numberToBigIntTransformer } from '../utils'; + +@Entity({ name: 'blocks', schema: 'raw' }) +export class Block { + @PrimaryColumn() public hash!: string; + @PrimaryColumn({ transformer: numberToBigIntTransformer }) + public number!: number; + + @Column({ name: 'timestamp', transformer: numberToBigIntTransformer }) + public timestamp!: number; +} diff --git a/packages/pipeline/src/entities/dex_trade.ts b/packages/pipeline/src/entities/dex_trade.ts new file mode 100644 index 000000000..9d288cb51 --- /dev/null +++ b/packages/pipeline/src/entities/dex_trade.ts @@ -0,0 +1,54 @@ +import { BigNumber } from '@0x/utils'; +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +import { bigNumberTransformer, numberToBigIntTransformer } from '../utils'; + +@Entity({ name: 'dex_trades', schema: 'raw' }) +export class DexTrade { + @PrimaryColumn({ name: 'source_url' }) + public sourceUrl!: string; + @PrimaryColumn({ name: 'tx_hash' }) + public txHash!: string; + + @Column({ name: 'tx_timestamp', type: 'bigint', transformer: numberToBigIntTransformer }) + public txTimestamp!: number; + @Column({ name: 'tx_date' }) + public txDate!: string; + @Column({ name: 'tx_sender' }) + public txSender!: string; + @Column({ name: 'smart_contract_id', type: 'bigint', transformer: numberToBigIntTransformer }) + public smartContractId!: number; + @Column({ name: 'smart_contract_address' }) + public smartContractAddress!: string; + @Column({ name: 'contract_type' }) + public contractType!: string; + @Column({ type: 'varchar' }) + public maker!: string; + @Column({ type: 'varchar' }) + public taker!: string; + @Column({ name: 'amount_buy', type: 'numeric', transformer: bigNumberTransformer }) + public amountBuy!: BigNumber; + @Column({ name: 'maker_fee_amount', type: 'numeric', transformer: bigNumberTransformer }) + public makerFeeAmount!: BigNumber; + @Column({ name: 'buy_currency_id', type: 'bigint', transformer: numberToBigIntTransformer }) + public buyCurrencyId!: number; + @Column({ name: 'buy_symbol' }) + public buySymbol!: string; + @Column({ name: 'amount_sell', type: 'numeric', transformer: bigNumberTransformer }) + public amountSell!: BigNumber; + @Column({ name: 'taker_fee_amount', type: 'numeric', transformer: bigNumberTransformer }) + public takerFeeAmount!: BigNumber; + @Column({ name: 'sell_currency_id', type: 'bigint', transformer: numberToBigIntTransformer }) + public sellCurrencyId!: number; + @Column({ name: 'sell_symbol' }) + public sellSymbol!: string; + @Column({ name: 'maker_annotation' }) + public makerAnnotation!: string; + @Column({ name: 'taker_annotation' }) + public takerAnnotation!: string; + @Column() public protocol!: string; + @Column({ name: 'buy_address', type: 'varchar', nullable: true }) + public buyAddress!: string | null; + @Column({ name: 'sell_address', type: 'varchar', nullable: true }) + public sellAddress!: string | null; +} diff --git a/packages/pipeline/src/entities/exchange_cancel_event.ts b/packages/pipeline/src/entities/exchange_cancel_event.ts new file mode 100644 index 000000000..38f99c903 --- /dev/null +++ b/packages/pipeline/src/entities/exchange_cancel_event.ts @@ -0,0 +1,51 @@ +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +import { AssetType } from '../types'; +import { numberToBigIntTransformer } from '../utils'; + +@Entity({ name: 'exchange_cancel_events', schema: 'raw' }) +export class ExchangeCancelEvent { + @PrimaryColumn({ name: 'contract_address' }) + public contractAddress!: string; + @PrimaryColumn({ name: 'log_index' }) + public logIndex!: number; + @PrimaryColumn({ name: 'block_number', transformer: numberToBigIntTransformer }) + public blockNumber!: number; + + @Column({ name: 'raw_data' }) + public rawData!: string; + + @Column({ name: 'transaction_hash' }) + public transactionHash!: string; + @Column({ name: 'maker_address' }) + public makerAddress!: string; + @Column({ nullable: true, type: String, name: 'taker_address' }) + public takerAddress!: string; + @Column({ name: 'fee_recipient_address' }) + public feeRecipientAddress!: string; + @Column({ name: 'sender_address' }) + public senderAddress!: string; + @Column({ name: 'order_hash' }) + public orderHash!: string; + + @Column({ name: 'raw_maker_asset_data' }) + public rawMakerAssetData!: string; + @Column({ name: 'maker_asset_type' }) + public makerAssetType!: AssetType; + @Column({ name: 'maker_asset_proxy_id' }) + public makerAssetProxyId!: string; + @Column({ name: 'maker_token_address' }) + public makerTokenAddress!: string; + @Column({ nullable: true, type: String, name: 'maker_token_id' }) + public makerTokenId!: string | null; + @Column({ name: 'raw_taker_asset_data' }) + public rawTakerAssetData!: string; + @Column({ name: 'taker_asset_type' }) + public takerAssetType!: AssetType; + @Column({ name: 'taker_asset_proxy_id' }) + public takerAssetProxyId!: string; + @Column({ name: 'taker_token_address' }) + public takerTokenAddress!: string; + @Column({ nullable: true, type: String, name: 'taker_token_id' }) + public takerTokenId!: string | null; +} diff --git a/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts b/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts new file mode 100644 index 000000000..27580305e --- /dev/null +++ b/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts @@ -0,0 +1,26 @@ +import { BigNumber } from '@0x/utils'; +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +import { bigNumberTransformer, numberToBigIntTransformer } from '../utils'; + +@Entity({ name: 'exchange_cancel_up_to_events', schema: 'raw' }) +export class ExchangeCancelUpToEvent { + @PrimaryColumn({ name: 'contract_address' }) + public contractAddress!: string; + @PrimaryColumn({ name: 'log_index' }) + public logIndex!: number; + @PrimaryColumn({ name: 'block_number', transformer: numberToBigIntTransformer }) + public blockNumber!: number; + + @Column({ name: 'raw_data' }) + public rawData!: string; + + @Column({ name: 'transaction_hash' }) + public transactionHash!: string; + @Column({ name: 'maker_address' }) + public makerAddress!: string; + @Column({ name: 'sender_address' }) + public senderAddress!: string; + @Column({ name: 'order_epoch', type: 'numeric', transformer: bigNumberTransformer }) + public orderEpoch!: BigNumber; +} diff --git a/packages/pipeline/src/entities/exchange_fill_event.ts b/packages/pipeline/src/entities/exchange_fill_event.ts new file mode 100644 index 000000000..9b7727615 --- /dev/null +++ b/packages/pipeline/src/entities/exchange_fill_event.ts @@ -0,0 +1,60 @@ +import { BigNumber } from '@0x/utils'; +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +import { AssetType } from '../types'; +import { bigNumberTransformer, numberToBigIntTransformer } from '../utils'; + +@Entity({ name: 'exchange_fill_events', schema: 'raw' }) +export class ExchangeFillEvent { + @PrimaryColumn({ name: 'contract_address' }) + public contractAddress!: string; + @PrimaryColumn({ name: 'log_index' }) + public logIndex!: number; + @PrimaryColumn({ name: 'block_number', transformer: numberToBigIntTransformer }) + public blockNumber!: number; + + @Column({ name: 'raw_data' }) + public rawData!: string; + + @Column({ name: 'transaction_hash' }) + public transactionHash!: string; + @Column({ name: 'maker_address' }) + public makerAddress!: string; + @Column({ name: 'taker_address' }) + public takerAddress!: string; + @Column({ name: 'fee_recipient_address' }) + public feeRecipientAddress!: string; + @Column({ name: 'sender_address' }) + public senderAddress!: string; + @Column({ name: 'maker_asset_filled_amount', type: 'numeric', transformer: bigNumberTransformer }) + public makerAssetFilledAmount!: BigNumber; + @Column({ name: 'taker_asset_filled_amount', type: 'numeric', transformer: bigNumberTransformer }) + public takerAssetFilledAmount!: BigNumber; + @Column({ name: 'maker_fee_paid', type: 'numeric', transformer: bigNumberTransformer }) + public makerFeePaid!: BigNumber; + @Column({ name: 'taker_fee_paid', type: 'numeric', transformer: bigNumberTransformer }) + public takerFeePaid!: BigNumber; + @Column({ name: 'order_hash' }) + public orderHash!: string; + + @Column({ name: 'raw_maker_asset_data' }) + public rawMakerAssetData!: string; + @Column({ name: 'maker_asset_type' }) + public makerAssetType!: AssetType; + @Column({ name: 'maker_asset_proxy_id' }) + public makerAssetProxyId!: string; + @Column({ name: 'maker_token_address' }) + public makerTokenAddress!: string; + @Column({ nullable: true, type: String, name: 'maker_token_id' }) + public makerTokenId!: string | null; + @Column({ name: 'raw_taker_asset_data' }) + public rawTakerAssetData!: string; + @Column({ name: 'taker_asset_type' }) + public takerAssetType!: AssetType; + @Column({ name: 'taker_asset_proxy_id' }) + public takerAssetProxyId!: string; + @Column({ name: 'taker_token_address' }) + public takerTokenAddress!: string; + @Column({ nullable: true, type: String, name: 'taker_token_id' }) + public takerTokenId!: string | null; +} diff --git a/packages/pipeline/src/entities/index.ts b/packages/pipeline/src/entities/index.ts new file mode 100644 index 000000000..db0814e38 --- /dev/null +++ b/packages/pipeline/src/entities/index.ts @@ -0,0 +1,18 @@ +import { ExchangeCancelEvent } from './exchange_cancel_event'; +import { ExchangeCancelUpToEvent } from './exchange_cancel_up_to_event'; +import { ExchangeFillEvent } from './exchange_fill_event'; + +export { Block } from './block'; +export { DexTrade } from './dex_trade'; +export { ExchangeCancelEvent } from './exchange_cancel_event'; +export { ExchangeCancelUpToEvent } from './exchange_cancel_up_to_event'; +export { ExchangeFillEvent } from './exchange_fill_event'; +export { OHLCVExternal } from './ohlcv_external'; +export { Relayer } from './relayer'; +export { SraOrder } from './sra_order'; +export { SraOrdersObservedTimeStamp, createObservedTimestampForOrder } from './sra_order_observed_timestamp'; +export { TokenMetadata } from './token_metadata'; +export { TokenOrderbookSnapshot } from './token_order'; +export { Transaction } from './transaction'; + +export type ExchangeEvent = ExchangeFillEvent | ExchangeCancelEvent | ExchangeCancelUpToEvent; diff --git a/packages/pipeline/src/entities/ohlcv_external.ts b/packages/pipeline/src/entities/ohlcv_external.ts new file mode 100644 index 000000000..4f55dd930 --- /dev/null +++ b/packages/pipeline/src/entities/ohlcv_external.ts @@ -0,0 +1,30 @@ +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +import { numberToBigIntTransformer } from '../utils'; + +@Entity({ name: 'ohlcv_external', schema: 'raw' }) +export class OHLCVExternal { + @PrimaryColumn() public exchange!: string; + + @PrimaryColumn({ name: 'from_symbol', type: 'varchar' }) + public fromSymbol!: string; + @PrimaryColumn({ name: 'to_symbol', type: 'varchar' }) + public toSymbol!: string; + @PrimaryColumn({ name: 'start_time', transformer: numberToBigIntTransformer }) + public startTime!: number; + @PrimaryColumn({ name: 'end_time', transformer: numberToBigIntTransformer }) + public endTime!: number; + + @Column() public open!: number; + @Column() public close!: number; + @Column() public low!: number; + @Column() public high!: number; + @Column({ name: 'volume_from' }) + public volumeFrom!: number; + @Column({ name: 'volume_to' }) + public volumeTo!: number; + + @PrimaryColumn() public source!: string; + @PrimaryColumn({ name: 'observed_timestamp', transformer: numberToBigIntTransformer }) + public observedTimestamp!: number; +} diff --git a/packages/pipeline/src/entities/relayer.ts b/packages/pipeline/src/entities/relayer.ts new file mode 100644 index 000000000..5af8578b4 --- /dev/null +++ b/packages/pipeline/src/entities/relayer.ts @@ -0,0 +1,21 @@ +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +@Entity({ name: 'relayers', schema: 'raw' }) +export class Relayer { + @PrimaryColumn() public uuid!: string; + + @Column() public name!: string; + @Column({ name: 'homepage_url', type: 'varchar' }) + public homepageUrl!: string; + @Column({ name: 'sra_http_endpoint', type: 'varchar', nullable: true }) + public sraHttpEndpoint!: string | null; + @Column({ name: 'sra_ws_endpoint', type: 'varchar', nullable: true }) + public sraWsEndpoint!: string | null; + @Column({ name: 'app_url', type: 'varchar', nullable: true }) + public appUrl!: string | null; + + @Column({ name: 'fee_recipient_addresses', type: 'varchar', array: true }) + public feeRecipientAddresses!: string[]; + @Column({ name: 'taker_addresses', type: 'varchar', array: true }) + public takerAddresses!: string[]; +} diff --git a/packages/pipeline/src/entities/sra_order.ts b/packages/pipeline/src/entities/sra_order.ts new file mode 100644 index 000000000..9c730a0bb --- /dev/null +++ b/packages/pipeline/src/entities/sra_order.ts @@ -0,0 +1,63 @@ +import { BigNumber } from '@0x/utils'; +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +import { AssetType } from '../types'; +import { bigNumberTransformer } from '../utils'; + +@Entity({ name: 'sra_orders', schema: 'raw' }) +export class SraOrder { + @PrimaryColumn({ name: 'exchange_address' }) + public exchangeAddress!: string; + @PrimaryColumn({ name: 'order_hash_hex' }) + public orderHashHex!: string; + @PrimaryColumn({ name: 'source_url' }) + public sourceUrl!: string; + + @Column({ name: 'maker_address' }) + public makerAddress!: string; + @Column({ name: 'taker_address' }) + public takerAddress!: string; + @Column({ name: 'fee_recipient_address' }) + public feeRecipientAddress!: string; + @Column({ name: 'sender_address' }) + public senderAddress!: string; + @Column({ name: 'maker_asset_amount', type: 'numeric', transformer: bigNumberTransformer }) + public makerAssetAmount!: BigNumber; + @Column({ name: 'taker_asset_amount', type: 'numeric', transformer: bigNumberTransformer }) + public takerAssetAmount!: BigNumber; + @Column({ name: 'maker_fee', type: 'numeric', transformer: bigNumberTransformer }) + public makerFee!: BigNumber; + @Column({ name: 'taker_fee', type: 'numeric', transformer: bigNumberTransformer }) + public takerFee!: BigNumber; + @Column({ name: 'expiration_time_seconds', type: 'numeric', transformer: bigNumberTransformer }) + public expirationTimeSeconds!: BigNumber; + @Column({ name: 'salt', type: 'numeric', transformer: bigNumberTransformer }) + public salt!: BigNumber; + @Column({ name: 'signature' }) + public signature!: string; + + @Column({ name: 'raw_maker_asset_data' }) + public rawMakerAssetData!: string; + @Column({ name: 'maker_asset_type' }) + public makerAssetType!: AssetType; + @Column({ name: 'maker_asset_proxy_id' }) + public makerAssetProxyId!: string; + @Column({ name: 'maker_token_address' }) + public makerTokenAddress!: string; + @Column({ nullable: true, type: String, name: 'maker_token_id' }) + public makerTokenId!: string | null; + @Column({ name: 'raw_taker_asset_data' }) + public rawTakerAssetData!: string; + @Column({ name: 'taker_asset_type' }) + public takerAssetType!: AssetType; + @Column({ name: 'taker_asset_proxy_id' }) + public takerAssetProxyId!: string; + @Column({ name: 'taker_token_address' }) + public takerTokenAddress!: string; + @Column({ nullable: true, type: String, name: 'taker_token_id' }) + public takerTokenId!: string | null; + + // TODO(albrow): Make this optional? + @Column({ name: 'metadata_json' }) + public metadataJson!: string; +} diff --git a/packages/pipeline/src/entities/sra_order_observed_timestamp.ts b/packages/pipeline/src/entities/sra_order_observed_timestamp.ts new file mode 100644 index 000000000..cbec1c6d0 --- /dev/null +++ b/packages/pipeline/src/entities/sra_order_observed_timestamp.ts @@ -0,0 +1,35 @@ +import { Entity, PrimaryColumn } from 'typeorm'; + +import { numberToBigIntTransformer } from '../utils'; + +import { SraOrder } from './sra_order'; + +@Entity({ name: 'sra_orders_observed_timestamps', schema: 'raw' }) +export class SraOrdersObservedTimeStamp { + @PrimaryColumn({ name: 'exchange_address' }) + public exchangeAddress!: string; + @PrimaryColumn({ name: 'order_hash_hex' }) + public orderHashHex!: string; + @PrimaryColumn({ name: 'source_url' }) + public sourceUrl!: string; + + @PrimaryColumn({ name: 'observed_timestamp', transformer: numberToBigIntTransformer }) + public observedTimestamp!: number; +} + +/** + * Returns a new SraOrdersObservedTimeStamp for the given order based on the + * current time. + * @param order The order to generate a timestamp for. + */ +export function createObservedTimestampForOrder( + order: SraOrder, + observedTimestamp: number, +): SraOrdersObservedTimeStamp { + const observed = new SraOrdersObservedTimeStamp(); + observed.exchangeAddress = order.exchangeAddress; + observed.orderHashHex = order.orderHashHex; + observed.sourceUrl = order.sourceUrl; + observed.observedTimestamp = observedTimestamp; + return observed; +} diff --git a/packages/pipeline/src/entities/token_metadata.ts b/packages/pipeline/src/entities/token_metadata.ts new file mode 100644 index 000000000..911b53972 --- /dev/null +++ b/packages/pipeline/src/entities/token_metadata.ts @@ -0,0 +1,22 @@ +import { BigNumber } from '@0x/utils'; +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +import { bigNumberTransformer } from '../utils/transformers'; + +@Entity({ name: 'token_metadata', schema: 'raw' }) +export class TokenMetadata { + @PrimaryColumn({ type: 'varchar', nullable: false }) + public address!: string; + + @PrimaryColumn({ type: 'varchar', nullable: false }) + public authority!: string; + + @Column({ type: 'numeric', transformer: bigNumberTransformer, nullable: true }) + public decimals!: BigNumber | null; + + @Column({ type: 'varchar', nullable: true }) + public symbol!: string | null; + + @Column({ type: 'varchar', nullable: true }) + public name!: string | null; +} diff --git a/packages/pipeline/src/entities/token_order.ts b/packages/pipeline/src/entities/token_order.ts new file mode 100644 index 000000000..557705767 --- /dev/null +++ b/packages/pipeline/src/entities/token_order.ts @@ -0,0 +1,29 @@ +import { BigNumber } from '@0x/utils'; +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +import { OrderType } from '../types'; +import { bigNumberTransformer, numberToBigIntTransformer } from '../utils'; + +@Entity({ name: 'token_orderbook_snapshots', schema: 'raw' }) +export class TokenOrderbookSnapshot { + @PrimaryColumn({ name: 'observed_timestamp', type: 'bigint', transformer: numberToBigIntTransformer }) + public observedTimestamp!: number; + @PrimaryColumn({ name: 'source' }) + public source!: string; + @Column({ name: 'order_type' }) + public orderType!: OrderType; + @PrimaryColumn({ name: 'price', type: 'numeric', transformer: bigNumberTransformer }) + public price!: BigNumber; + @PrimaryColumn({ name: 'base_asset_symbol' }) + public baseAssetSymbol!: string; + @Column({ name: 'base_asset_address' }) + public baseAssetAddress!: string; + @Column({ name: 'base_volume', type: 'numeric', transformer: bigNumberTransformer }) + public baseVolume!: BigNumber; + @PrimaryColumn({ name: 'quote_asset_symbol' }) + public quoteAssetSymbol!: string; + @Column({ name: 'quote_asset_address' }) + public quoteAssetAddress!: string; + @Column({ name: 'quote_volume', type: 'numeric', transformer: bigNumberTransformer }) + public quoteVolume!: BigNumber; +} diff --git a/packages/pipeline/src/entities/transaction.ts b/packages/pipeline/src/entities/transaction.ts new file mode 100644 index 000000000..742050177 --- /dev/null +++ b/packages/pipeline/src/entities/transaction.ts @@ -0,0 +1,19 @@ +import { BigNumber } from '@0x/utils'; +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +import { bigNumberTransformer, numberToBigIntTransformer } from '../utils'; + +@Entity({ name: 'transactions', schema: 'raw' }) +export class Transaction { + @PrimaryColumn({ name: 'transaction_hash' }) + public transactionHash!: string; + @PrimaryColumn({ name: 'block_hash' }) + public blockHash!: string; + @PrimaryColumn({ name: 'block_number', transformer: numberToBigIntTransformer }) + public blockNumber!: number; + + @Column({ type: 'numeric', name: 'gas_used', transformer: bigNumberTransformer }) + public gasUsed!: BigNumber; + @Column({ type: 'numeric', name: 'gas_price', transformer: bigNumberTransformer }) + public gasPrice!: BigNumber; +} diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts new file mode 100644 index 000000000..9f7815b4e --- /dev/null +++ b/packages/pipeline/src/ormconfig.ts @@ -0,0 +1,42 @@ +import { ConnectionOptions } from 'typeorm'; + +import { + Block, + DexTrade, + ExchangeCancelEvent, + ExchangeCancelUpToEvent, + ExchangeFillEvent, + OHLCVExternal, + Relayer, + SraOrder, + SraOrdersObservedTimeStamp, + TokenMetadata, + TokenOrderbookSnapshot, + Transaction, +} from './entities'; + +const entities = [ + Block, + DexTrade, + ExchangeCancelEvent, + ExchangeCancelUpToEvent, + ExchangeFillEvent, + OHLCVExternal, + Relayer, + SraOrder, + SraOrdersObservedTimeStamp, + TokenMetadata, + TokenOrderbookSnapshot, + Transaction, +]; + +const config: ConnectionOptions = { + type: 'postgres', + url: process.env.ZEROEX_DATA_PIPELINE_DB_URL, + synchronize: false, + logging: ['error'], + entities, + migrations: ['./lib/migrations/**/*.js'], +}; + +module.exports = config; diff --git a/packages/pipeline/src/parsers/bloxy/index.ts b/packages/pipeline/src/parsers/bloxy/index.ts new file mode 100644 index 000000000..caa55d289 --- /dev/null +++ b/packages/pipeline/src/parsers/bloxy/index.ts @@ -0,0 +1,53 @@ +import { BigNumber } from '@0x/utils'; +import * as R from 'ramda'; + +import { BLOXY_DEX_TRADES_URL, BloxyTrade } from '../../data_sources/bloxy'; +import { DexTrade } from '../../entities'; + +/** + * Parses a raw trades response from the Bloxy Dex API and returns an array of + * DexTrade entities. + * @param rawTrades A raw order response from an SRA endpoint. + */ +export function parseBloxyTrades(rawTrades: BloxyTrade[]): DexTrade[] { + return R.map(_parseBloxyTrade, rawTrades); +} + +/** + * Converts a single Bloxy trade into a DexTrade entity. + * @param rawTrade A single trade from the response from the Bloxy API. + */ +export function _parseBloxyTrade(rawTrade: BloxyTrade): DexTrade { + const dexTrade = new DexTrade(); + dexTrade.sourceUrl = BLOXY_DEX_TRADES_URL; + dexTrade.txHash = rawTrade.tx_hash; + dexTrade.txTimestamp = new Date(rawTrade.tx_time).getTime(); + dexTrade.txDate = rawTrade.tx_date; + dexTrade.txSender = rawTrade.tx_sender; + dexTrade.smartContractId = rawTrade.smart_contract_id; + dexTrade.smartContractAddress = rawTrade.smart_contract_address; + dexTrade.contractType = rawTrade.contract_type; + dexTrade.maker = rawTrade.maker; + dexTrade.taker = rawTrade.taker; + // TODO(albrow): The Bloxy API returns amounts and fees as a `number` type + // but some of their values have too many significant digits to be + // represented that way. Ideally they will switch to using strings and then + // we can update this code. + dexTrade.amountBuy = new BigNumber(rawTrade.amountBuy.toString()); + dexTrade.makerFeeAmount = new BigNumber(rawTrade.makerFee.toString()); + dexTrade.buyCurrencyId = rawTrade.buyCurrencyId; + dexTrade.buySymbol = filterNullCharacters(rawTrade.buySymbol); + dexTrade.amountSell = new BigNumber(rawTrade.amountSell.toString()); + dexTrade.takerFeeAmount = new BigNumber(rawTrade.takerFee.toString()); + dexTrade.sellCurrencyId = rawTrade.sellCurrencyId; + dexTrade.sellSymbol = filterNullCharacters(rawTrade.sellSymbol); + dexTrade.makerAnnotation = rawTrade.maker_annotation; + dexTrade.takerAnnotation = rawTrade.taker_annotation; + dexTrade.protocol = rawTrade.protocol; + dexTrade.buyAddress = rawTrade.buyAddress; + dexTrade.sellAddress = rawTrade.sellAddress; + return dexTrade; +} + +// Works with any form of escaped null character (e.g., '\0' and '\u0000'). +const filterNullCharacters = R.replace(/\0/g, ''); diff --git a/packages/pipeline/src/parsers/ddex_orders/index.ts b/packages/pipeline/src/parsers/ddex_orders/index.ts new file mode 100644 index 000000000..81132e8f0 --- /dev/null +++ b/packages/pipeline/src/parsers/ddex_orders/index.ts @@ -0,0 +1,77 @@ +import { BigNumber } from '@0x/utils'; +import * as R from 'ramda'; + +import { DdexMarket, DdexOrder, DdexOrderbook } from '../../data_sources/ddex'; +import { TokenOrderbookSnapshot as TokenOrder } from '../../entities'; +import { OrderType } from '../../types'; + +/** + * Marque function of this file. + * 1) Takes in orders from an orderbook, + * other information attached. + * @param ddexOrderbook A raw orderbook that we pull from the Ddex API. + * @param ddexMarket An object containing market data also directly from the API. + * @param observedTimestamp Time at which the orders for the market were pulled. + * @param source The exchange where these orders are placed. In this case 'ddex'. + */ +export function parseDdexOrders( + ddexOrderbook: DdexOrderbook, + ddexMarket: DdexMarket, + observedTimestamp: number, + source: string, +): TokenOrder[] { + const aggregatedBids = aggregateOrders(ddexOrderbook.bids); + const aggregatedAsks = aggregateOrders(ddexOrderbook.asks); + const parsedBids = aggregatedBids.map(order => parseDdexOrder(ddexMarket, observedTimestamp, 'bid', source, order)); + const parsedAsks = aggregatedAsks.map(order => parseDdexOrder(ddexMarket, observedTimestamp, 'ask', source, order)); + return parsedBids.concat(parsedAsks); +} + +/** + * Aggregates orders by price point for consistency with other exchanges. + * Querying the Ddex API at level 3 setting returns a breakdown of + * individual orders at each price point. Other exchanges only give total amount + * at each price point. Returns an array of <price, amount> tuples. + * @param ddexOrders A list of Ddex orders awaiting aggregation. + */ +export function aggregateOrders(ddexOrders: DdexOrder[]): Array<[string, BigNumber]> { + const sumAmount = (acc: BigNumber, order: DdexOrder): BigNumber => acc.plus(order.amount); + const aggregatedPricePoints = R.reduceBy(sumAmount, new BigNumber(0), R.prop('price'), ddexOrders); + return Object.entries(aggregatedPricePoints); +} + +/** + * Parse a single aggregated Ddex order in order to form a tokenOrder entity + * which can be saved into the database. + * @param ddexMarket An object containing information about the market where these + * trades have been placed. + * @param observedTimestamp The time when the API response returned back to us. + * @param orderType 'bid' or 'ask' enum. + * @param source Exchange where these orders were placed. + * @param ddexOrder A <price, amount> tuple which we will convert to volume-basis. + */ +export function parseDdexOrder( + ddexMarket: DdexMarket, + observedTimestamp: number, + orderType: OrderType, + source: string, + ddexOrder: [string, BigNumber], +): TokenOrder { + const tokenOrder = new TokenOrder(); + const price = new BigNumber(ddexOrder[0]); + const amount = ddexOrder[1]; + + tokenOrder.source = source; + tokenOrder.observedTimestamp = observedTimestamp; + tokenOrder.orderType = orderType; + tokenOrder.price = price; + + tokenOrder.baseAssetSymbol = ddexMarket.baseToken; + tokenOrder.baseAssetAddress = ddexMarket.baseTokenAddress; + tokenOrder.baseVolume = price.times(amount); + + tokenOrder.quoteAssetSymbol = ddexMarket.quoteToken; + tokenOrder.quoteAssetAddress = ddexMarket.quoteTokenAddress; + tokenOrder.quoteVolume = amount; + return tokenOrder; +} diff --git a/packages/pipeline/src/parsers/events/index.ts b/packages/pipeline/src/parsers/events/index.ts new file mode 100644 index 000000000..e18106c75 --- /dev/null +++ b/packages/pipeline/src/parsers/events/index.ts @@ -0,0 +1,133 @@ +import { ExchangeCancelEventArgs, ExchangeCancelUpToEventArgs, ExchangeFillEventArgs } from '@0x/contract-wrappers'; +import { assetDataUtils } from '@0x/order-utils'; +import { AssetProxyId, ERC721AssetData } from '@0x/types'; +import { LogWithDecodedArgs } from 'ethereum-types'; +import * as R from 'ramda'; + +import { ExchangeCancelEvent, ExchangeCancelUpToEvent, ExchangeFillEvent } from '../../entities'; +import { bigNumbertoStringOrNull } from '../../utils'; + +/** + * Parses raw event logs for a fill event and returns an array of + * ExchangeFillEvent entities. + * @param eventLogs Raw event logs (e.g. returned from contract-wrappers). + */ +export const parseExchangeFillEvents: ( + eventLogs: Array<LogWithDecodedArgs<ExchangeFillEventArgs>>, +) => ExchangeFillEvent[] = R.map(_convertToExchangeFillEvent); + +/** + * Parses raw event logs for a cancel event and returns an array of + * ExchangeCancelEvent entities. + * @param eventLogs Raw event logs (e.g. returned from contract-wrappers). + */ +export const parseExchangeCancelEvents: ( + eventLogs: Array<LogWithDecodedArgs<ExchangeCancelEventArgs>>, +) => ExchangeCancelEvent[] = R.map(_convertToExchangeCancelEvent); + +/** + * Parses raw event logs for a CancelUpTo event and returns an array of + * ExchangeCancelUpToEvent entities. + * @param eventLogs Raw event logs (e.g. returned from contract-wrappers). + */ +export const parseExchangeCancelUpToEvents: ( + eventLogs: Array<LogWithDecodedArgs<ExchangeCancelUpToEventArgs>>, +) => ExchangeCancelUpToEvent[] = R.map(_convertToExchangeCancelUpToEvent); + +/** + * Converts a raw event log for a fill event into an ExchangeFillEvent entity. + * @param eventLog Raw event log (e.g. returned from contract-wrappers). + */ +export function _convertToExchangeFillEvent(eventLog: LogWithDecodedArgs<ExchangeFillEventArgs>): ExchangeFillEvent { + const makerAssetData = assetDataUtils.decodeAssetDataOrThrow(eventLog.args.makerAssetData); + const makerAssetType = makerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721'; + const takerAssetData = assetDataUtils.decodeAssetDataOrThrow(eventLog.args.takerAssetData); + const takerAssetType = takerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721'; + const exchangeFillEvent = new ExchangeFillEvent(); + exchangeFillEvent.contractAddress = eventLog.address as string; + exchangeFillEvent.blockNumber = eventLog.blockNumber as number; + exchangeFillEvent.logIndex = eventLog.logIndex as number; + exchangeFillEvent.rawData = eventLog.data as string; + exchangeFillEvent.transactionHash = eventLog.transactionHash; + exchangeFillEvent.makerAddress = eventLog.args.makerAddress; + exchangeFillEvent.takerAddress = eventLog.args.takerAddress; + exchangeFillEvent.feeRecipientAddress = eventLog.args.feeRecipientAddress; + exchangeFillEvent.senderAddress = eventLog.args.senderAddress; + exchangeFillEvent.makerAssetFilledAmount = eventLog.args.makerAssetFilledAmount; + exchangeFillEvent.takerAssetFilledAmount = eventLog.args.takerAssetFilledAmount; + exchangeFillEvent.makerFeePaid = eventLog.args.makerFeePaid; + exchangeFillEvent.takerFeePaid = eventLog.args.takerFeePaid; + exchangeFillEvent.orderHash = eventLog.args.orderHash; + exchangeFillEvent.rawMakerAssetData = eventLog.args.makerAssetData; + exchangeFillEvent.makerAssetType = makerAssetType; + exchangeFillEvent.makerAssetProxyId = makerAssetData.assetProxyId; + exchangeFillEvent.makerTokenAddress = makerAssetData.tokenAddress; + // tslint has a false positive here. Type assertion is required. + // tslint:disable-next-line:no-unnecessary-type-assertion + exchangeFillEvent.makerTokenId = bigNumbertoStringOrNull((makerAssetData as ERC721AssetData).tokenId); + exchangeFillEvent.rawTakerAssetData = eventLog.args.takerAssetData; + exchangeFillEvent.takerAssetType = takerAssetType; + exchangeFillEvent.takerAssetProxyId = takerAssetData.assetProxyId; + exchangeFillEvent.takerTokenAddress = takerAssetData.tokenAddress; + // tslint:disable-next-line:no-unnecessary-type-assertion + exchangeFillEvent.takerTokenId = bigNumbertoStringOrNull((takerAssetData as ERC721AssetData).tokenId); + return exchangeFillEvent; +} + +/** + * Converts a raw event log for a cancel event into an ExchangeCancelEvent + * entity. + * @param eventLog Raw event log (e.g. returned from contract-wrappers). + */ +export function _convertToExchangeCancelEvent( + eventLog: LogWithDecodedArgs<ExchangeCancelEventArgs>, +): ExchangeCancelEvent { + const makerAssetData = assetDataUtils.decodeAssetDataOrThrow(eventLog.args.makerAssetData); + const makerAssetType = makerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721'; + const takerAssetData = assetDataUtils.decodeAssetDataOrThrow(eventLog.args.takerAssetData); + const takerAssetType = takerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721'; + const exchangeCancelEvent = new ExchangeCancelEvent(); + exchangeCancelEvent.contractAddress = eventLog.address as string; + exchangeCancelEvent.blockNumber = eventLog.blockNumber as number; + exchangeCancelEvent.logIndex = eventLog.logIndex as number; + exchangeCancelEvent.rawData = eventLog.data as string; + exchangeCancelEvent.transactionHash = eventLog.transactionHash; + exchangeCancelEvent.makerAddress = eventLog.args.makerAddress; + exchangeCancelEvent.takerAddress = eventLog.args.takerAddress; + exchangeCancelEvent.feeRecipientAddress = eventLog.args.feeRecipientAddress; + exchangeCancelEvent.senderAddress = eventLog.args.senderAddress; + exchangeCancelEvent.orderHash = eventLog.args.orderHash; + exchangeCancelEvent.rawMakerAssetData = eventLog.args.makerAssetData; + exchangeCancelEvent.makerAssetType = makerAssetType; + exchangeCancelEvent.makerAssetProxyId = makerAssetData.assetProxyId; + exchangeCancelEvent.makerTokenAddress = makerAssetData.tokenAddress; + // tslint:disable-next-line:no-unnecessary-type-assertion + exchangeCancelEvent.makerTokenId = bigNumbertoStringOrNull((makerAssetData as ERC721AssetData).tokenId); + exchangeCancelEvent.rawTakerAssetData = eventLog.args.takerAssetData; + exchangeCancelEvent.takerAssetType = takerAssetType; + exchangeCancelEvent.takerAssetProxyId = takerAssetData.assetProxyId; + exchangeCancelEvent.takerTokenAddress = takerAssetData.tokenAddress; + // tslint:disable-next-line:no-unnecessary-type-assertion + exchangeCancelEvent.takerTokenId = bigNumbertoStringOrNull((takerAssetData as ERC721AssetData).tokenId); + return exchangeCancelEvent; +} + +/** + * Converts a raw event log for a cancelUpTo event into an + * ExchangeCancelUpToEvent entity. + * @param eventLog Raw event log (e.g. returned from contract-wrappers). + */ +export function _convertToExchangeCancelUpToEvent( + eventLog: LogWithDecodedArgs<ExchangeCancelUpToEventArgs>, +): ExchangeCancelUpToEvent { + const exchangeCancelUpToEvent = new ExchangeCancelUpToEvent(); + exchangeCancelUpToEvent.contractAddress = eventLog.address as string; + exchangeCancelUpToEvent.blockNumber = eventLog.blockNumber as number; + exchangeCancelUpToEvent.logIndex = eventLog.logIndex as number; + exchangeCancelUpToEvent.rawData = eventLog.data as string; + exchangeCancelUpToEvent.transactionHash = eventLog.transactionHash; + exchangeCancelUpToEvent.makerAddress = eventLog.args.makerAddress; + exchangeCancelUpToEvent.senderAddress = eventLog.args.senderAddress; + exchangeCancelUpToEvent.orderEpoch = eventLog.args.orderEpoch; + return exchangeCancelUpToEvent; +} diff --git a/packages/pipeline/src/parsers/ohlcv_external/crypto_compare.ts b/packages/pipeline/src/parsers/ohlcv_external/crypto_compare.ts new file mode 100644 index 000000000..3efb90384 --- /dev/null +++ b/packages/pipeline/src/parsers/ohlcv_external/crypto_compare.ts @@ -0,0 +1,38 @@ +import { CryptoCompareOHLCVRecord } from '../../data_sources/ohlcv_external/crypto_compare'; +import { OHLCVExternal } from '../../entities'; + +const ONE_SECOND = 1000; // Crypto Compare uses timestamps in seconds instead of milliseconds + +export interface OHLCVMetadata { + exchange: string; + fromSymbol: string; + toSymbol: string; + source: string; + observedTimestamp: number; + interval: number; +} +/** + * Parses OHLCV records from Crypto Compare into an array of OHLCVExternal entities + * @param rawRecords an array of OHLCV records from Crypto Compare (not the full response) + */ +export function parseRecords(rawRecords: CryptoCompareOHLCVRecord[], metadata: OHLCVMetadata): OHLCVExternal[] { + return rawRecords.map(rec => { + const ohlcvRecord = new OHLCVExternal(); + ohlcvRecord.exchange = metadata.exchange; + ohlcvRecord.fromSymbol = metadata.fromSymbol; + ohlcvRecord.toSymbol = metadata.toSymbol; + ohlcvRecord.startTime = rec.time * ONE_SECOND - metadata.interval; + ohlcvRecord.endTime = rec.time * ONE_SECOND; + + ohlcvRecord.open = rec.open; + ohlcvRecord.close = rec.close; + ohlcvRecord.low = rec.low; + ohlcvRecord.high = rec.high; + ohlcvRecord.volumeFrom = rec.volumefrom; + ohlcvRecord.volumeTo = rec.volumeto; + + ohlcvRecord.source = metadata.source; + ohlcvRecord.observedTimestamp = metadata.observedTimestamp; + return ohlcvRecord; + }); +} diff --git a/packages/pipeline/src/parsers/paradex_orders/index.ts b/packages/pipeline/src/parsers/paradex_orders/index.ts new file mode 100644 index 000000000..7966658a7 --- /dev/null +++ b/packages/pipeline/src/parsers/paradex_orders/index.ts @@ -0,0 +1,66 @@ +import { BigNumber } from '@0x/utils'; + +import { ParadexMarket, ParadexOrder, ParadexOrderbookResponse } from '../../data_sources/paradex'; +import { TokenOrderbookSnapshot as TokenOrder } from '../../entities'; +import { OrderType } from '../../types'; + +/** + * Marque function of this file. + * 1) Takes in orders from an orderbook (orders are already aggregated by price point), + * 2) For each aggregated order, forms a TokenOrder entity with market data and + * other information attached. + * @param paradexOrderbookResponse An orderbook response from the Paradex API. + * @param paradexMarket An object containing market data also directly from the API. + * @param observedTimestamp Time at which the orders for the market were pulled. + * @param source The exchange where these orders are placed. In this case 'paradex'. + */ +export function parseParadexOrders( + paradexOrderbookResponse: ParadexOrderbookResponse, + paradexMarket: ParadexMarket, + observedTimestamp: number, + source: string, +): TokenOrder[] { + const parsedBids = paradexOrderbookResponse.bids.map(order => + parseParadexOrder(paradexMarket, observedTimestamp, 'bid', source, order), + ); + const parsedAsks = paradexOrderbookResponse.asks.map(order => + parseParadexOrder(paradexMarket, observedTimestamp, 'ask', source, order), + ); + return parsedBids.concat(parsedAsks); +} + +/** + * Parse a single aggregated Ddex order in order to form a tokenOrder entity + * which can be saved into the database. + * @param paradexMarket An object containing information about the market where these + * orders have been placed. + * @param observedTimestamp The time when the API response returned back to us. + * @param orderType 'bid' or 'ask' enum. + * @param source Exchange where these orders were placed. + * @param paradexOrder A ParadexOrder object; basically price, amount tuple. + */ +export function parseParadexOrder( + paradexMarket: ParadexMarket, + observedTimestamp: number, + orderType: OrderType, + source: string, + paradexOrder: ParadexOrder, +): TokenOrder { + const tokenOrder = new TokenOrder(); + const price = new BigNumber(paradexOrder.price); + const amount = new BigNumber(paradexOrder.amount); + + tokenOrder.source = source; + tokenOrder.observedTimestamp = observedTimestamp; + tokenOrder.orderType = orderType; + tokenOrder.price = price; + + tokenOrder.baseAssetSymbol = paradexMarket.baseToken; + tokenOrder.baseAssetAddress = paradexMarket.baseTokenAddress as string; + tokenOrder.baseVolume = price.times(amount); + + tokenOrder.quoteAssetSymbol = paradexMarket.quoteToken; + tokenOrder.quoteAssetAddress = paradexMarket.quoteTokenAddress as string; + tokenOrder.quoteVolume = amount; + return tokenOrder; +} diff --git a/packages/pipeline/src/parsers/relayer_registry/index.ts b/packages/pipeline/src/parsers/relayer_registry/index.ts new file mode 100644 index 000000000..9723880a4 --- /dev/null +++ b/packages/pipeline/src/parsers/relayer_registry/index.ts @@ -0,0 +1,37 @@ +import * as R from 'ramda'; + +import { RelayerResponse, RelayerResponseNetwork } from '../../data_sources/relayer-registry'; +import { Relayer } from '../../entities'; + +/** + * Parses a raw relayer registry response into an array of Relayer entities. + * @param rawResp raw response from the relayer-registry json file. + */ +export function parseRelayers(rawResp: Map<string, RelayerResponse>): Relayer[] { + const parsedAsObject = R.mapObjIndexed(parseRelayer, rawResp); + return R.values(parsedAsObject); +} + +function parseRelayer(relayerResp: RelayerResponse, uuid: string): Relayer { + const relayer = new Relayer(); + relayer.uuid = uuid; + relayer.name = relayerResp.name; + relayer.homepageUrl = relayerResp.homepage_url; + relayer.appUrl = relayerResp.app_url; + const mainNetworkRelayerInfo = getMainNetwork(relayerResp); + if (mainNetworkRelayerInfo !== undefined) { + relayer.sraHttpEndpoint = mainNetworkRelayerInfo.sra_http_endpoint || null; + relayer.sraWsEndpoint = mainNetworkRelayerInfo.sra_ws_endpoint || null; + relayer.feeRecipientAddresses = + R.path(['static_order_fields', 'fee_recipient_addresses'], mainNetworkRelayerInfo) || []; + relayer.takerAddresses = R.path(['static_order_fields', 'taker_addresses'], mainNetworkRelayerInfo) || []; + } else { + relayer.feeRecipientAddresses = []; + relayer.takerAddresses = []; + } + return relayer; +} + +function getMainNetwork(relayerResp: RelayerResponse): RelayerResponseNetwork | undefined { + return R.find(network => network.networkId === 1, relayerResp.networks); +} diff --git a/packages/pipeline/src/parsers/sra_orders/index.ts b/packages/pipeline/src/parsers/sra_orders/index.ts new file mode 100644 index 000000000..ef8901e40 --- /dev/null +++ b/packages/pipeline/src/parsers/sra_orders/index.ts @@ -0,0 +1,62 @@ +import { APIOrder, OrdersResponse } from '@0x/connect'; +import { assetDataUtils, orderHashUtils } from '@0x/order-utils'; +import { AssetProxyId, ERC721AssetData } from '@0x/types'; +import * as R from 'ramda'; + +import { SraOrder } from '../../entities'; +import { bigNumbertoStringOrNull } from '../../utils'; + +/** + * Parses a raw order response from an SRA endpoint and returns an array of + * SraOrder entities. + * @param rawOrdersResponse A raw order response from an SRA endpoint. + */ +export function parseSraOrders(rawOrdersResponse: OrdersResponse): SraOrder[] { + return R.map(_convertToEntity, rawOrdersResponse.records); +} + +/** + * Converts a single APIOrder into an SraOrder entity. + * @param apiOrder A single order from the response from an SRA endpoint. + */ +export function _convertToEntity(apiOrder: APIOrder): SraOrder { + // TODO(albrow): refactor out common asset data decoding code. + const makerAssetData = assetDataUtils.decodeAssetDataOrThrow(apiOrder.order.makerAssetData); + const makerAssetType = makerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721'; + const takerAssetData = assetDataUtils.decodeAssetDataOrThrow(apiOrder.order.takerAssetData); + const takerAssetType = takerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721'; + + const sraOrder = new SraOrder(); + sraOrder.exchangeAddress = apiOrder.order.exchangeAddress; + sraOrder.orderHashHex = orderHashUtils.getOrderHashHex(apiOrder.order); + + sraOrder.makerAddress = apiOrder.order.makerAddress; + sraOrder.takerAddress = apiOrder.order.takerAddress; + sraOrder.feeRecipientAddress = apiOrder.order.feeRecipientAddress; + sraOrder.senderAddress = apiOrder.order.senderAddress; + sraOrder.makerAssetAmount = apiOrder.order.makerAssetAmount; + sraOrder.takerAssetAmount = apiOrder.order.takerAssetAmount; + sraOrder.makerFee = apiOrder.order.makerFee; + sraOrder.takerFee = apiOrder.order.takerFee; + sraOrder.expirationTimeSeconds = apiOrder.order.expirationTimeSeconds; + sraOrder.salt = apiOrder.order.salt; + sraOrder.signature = apiOrder.order.signature; + + sraOrder.rawMakerAssetData = apiOrder.order.makerAssetData; + sraOrder.makerAssetType = makerAssetType; + sraOrder.makerAssetProxyId = makerAssetData.assetProxyId; + sraOrder.makerTokenAddress = makerAssetData.tokenAddress; + // tslint has a false positive here. Type assertion is required. + // tslint:disable-next-line:no-unnecessary-type-assertion + sraOrder.makerTokenId = bigNumbertoStringOrNull((makerAssetData as ERC721AssetData).tokenId); + sraOrder.rawTakerAssetData = apiOrder.order.takerAssetData; + sraOrder.takerAssetType = takerAssetType; + sraOrder.takerAssetProxyId = takerAssetData.assetProxyId; + sraOrder.takerTokenAddress = takerAssetData.tokenAddress; + // tslint:disable-next-line:no-unnecessary-type-assertion + sraOrder.takerTokenId = bigNumbertoStringOrNull((takerAssetData as ERC721AssetData).tokenId); + + sraOrder.metadataJson = JSON.stringify(apiOrder.metaData); + + return sraOrder; +} diff --git a/packages/pipeline/src/parsers/token_metadata/index.ts b/packages/pipeline/src/parsers/token_metadata/index.ts new file mode 100644 index 000000000..3b3e05d76 --- /dev/null +++ b/packages/pipeline/src/parsers/token_metadata/index.ts @@ -0,0 +1,47 @@ +import { BigNumber } from '@0x/utils'; +import * as R from 'ramda'; + +import { MetamaskTrustedTokenMeta, ZeroExTrustedTokenMeta } from '../../data_sources/trusted_tokens'; +import { TokenMetadata } from '../../entities'; +import {} from '../../utils'; + +/** + * Parses Metamask's trusted tokens list. + * @param rawResp raw response from the metamask json file. + */ +export function parseMetamaskTrustedTokens(rawResp: Map<string, MetamaskTrustedTokenMeta>): TokenMetadata[] { + const parsedAsObject = R.mapObjIndexed(parseMetamaskTrustedToken, rawResp); + return R.values(parsedAsObject); +} + +/** + * Parses 0x's trusted tokens list. + * @param rawResp raw response from the 0x trusted tokens file. + */ +export function parseZeroExTrustedTokens(rawResp: ZeroExTrustedTokenMeta[]): TokenMetadata[] { + return R.map(parseZeroExTrustedToken, rawResp); +} + +function parseMetamaskTrustedToken(resp: MetamaskTrustedTokenMeta, address: string): TokenMetadata { + const trustedToken = new TokenMetadata(); + + trustedToken.address = address; + trustedToken.decimals = new BigNumber(resp.decimals); + trustedToken.symbol = resp.symbol; + trustedToken.name = resp.name; + trustedToken.authority = 'metamask'; + + return trustedToken; +} + +function parseZeroExTrustedToken(resp: ZeroExTrustedTokenMeta): TokenMetadata { + const trustedToken = new TokenMetadata(); + + trustedToken.address = resp.address; + trustedToken.decimals = new BigNumber(resp.decimals); + trustedToken.symbol = resp.symbol; + trustedToken.name = resp.name; + trustedToken.authority = '0x'; + + return trustedToken; +} diff --git a/packages/pipeline/src/parsers/web3/index.ts b/packages/pipeline/src/parsers/web3/index.ts new file mode 100644 index 000000000..f986efc59 --- /dev/null +++ b/packages/pipeline/src/parsers/web3/index.ts @@ -0,0 +1,49 @@ +import { BigNumber } from '@0x/utils'; +import { BlockWithoutTransactionData, Transaction as EthTransaction } from 'ethereum-types'; + +import { Block, Transaction } from '../../entities'; + +const MILLISECONDS_PER_SECOND = 1000; + +/** + * Parses a raw block and returns a Block entity. + * @param rawBlock a raw block (e.g. returned from web3-wrapper). + */ +export function parseBlock(rawBlock: BlockWithoutTransactionData): Block { + if (rawBlock.hash == null) { + throw new Error('Tried to parse raw block but hash was null'); + } + if (rawBlock.number == null) { + throw new Error('Tried to parse raw block but number was null'); + } + + const block = new Block(); + block.hash = rawBlock.hash; + block.number = rawBlock.number; + // Block timestamps are in seconds, but we use milliseconds everywhere else. + block.timestamp = rawBlock.timestamp * MILLISECONDS_PER_SECOND; + return block; +} + +/** + * Parses a raw transaction and returns a Transaction entity. + * @param rawBlock a raw transaction (e.g. returned from web3-wrapper). + */ +export function parseTransaction(rawTransaction: EthTransaction): Transaction { + if (rawTransaction.blockHash == null) { + throw new Error('Tried to parse raw transaction but blockHash was null'); + } + if (rawTransaction.blockNumber == null) { + throw new Error('Tried to parse raw transaction but blockNumber was null'); + } + + const tx = new Transaction(); + tx.transactionHash = rawTransaction.hash; + tx.blockHash = rawTransaction.blockHash; + tx.blockNumber = rawTransaction.blockNumber; + + tx.gasUsed = new BigNumber(rawTransaction.gas); + tx.gasPrice = rawTransaction.gasPrice; + + return tx; +} diff --git a/packages/pipeline/src/scripts/pull_competing_dex_trades.ts b/packages/pipeline/src/scripts/pull_competing_dex_trades.ts new file mode 100644 index 000000000..4e4c12dd0 --- /dev/null +++ b/packages/pipeline/src/scripts/pull_competing_dex_trades.ts @@ -0,0 +1,51 @@ +// tslint:disable:no-console +import 'reflect-metadata'; +import { Connection, ConnectionOptions, createConnection, Repository } from 'typeorm'; + +import { BloxySource } from '../data_sources/bloxy'; +import { DexTrade } from '../entities'; +import * as ormConfig from '../ormconfig'; +import { parseBloxyTrades } from '../parsers/bloxy'; +import { handleError } from '../utils'; + +// Number of trades to save at once. +const BATCH_SAVE_SIZE = 1000; + +let connection: Connection; + +(async () => { + connection = await createConnection(ormConfig as ConnectionOptions); + await getAndSaveTrades(); + process.exit(0); +})().catch(handleError); + +async function getAndSaveTrades(): Promise<void> { + const apiKey = process.env.BLOXY_API_KEY; + if (apiKey === undefined) { + throw new Error('Missing required env var: BLOXY_API_KEY'); + } + const bloxySource = new BloxySource(apiKey); + const tradesRepository = connection.getRepository(DexTrade); + const lastSeenTimestamp = await getLastSeenTimestampAsync(tradesRepository); + console.log(`Last seen timestamp: ${lastSeenTimestamp === 0 ? 'none' : lastSeenTimestamp}`); + console.log('Getting latest dex trades...'); + const rawTrades = await bloxySource.getDexTradesAsync(lastSeenTimestamp); + console.log(`Parsing ${rawTrades.length} trades...`); + const trades = parseBloxyTrades(rawTrades); + console.log(`Saving ${trades.length} trades...`); + await tradesRepository.save(trades, { chunk: Math.ceil(trades.length / BATCH_SAVE_SIZE) }); + console.log('Done saving trades.'); +} + +async function getLastSeenTimestampAsync(tradesRepository: Repository<DexTrade>): Promise<number> { + if ((await tradesRepository.count()) === 0) { + return 0; + } + const response = (await connection.query( + 'SELECT tx_timestamp FROM raw.dex_trades ORDER BY tx_timestamp DESC LIMIT 1', + )) as Array<{ tx_timestamp: number }>; + if (response.length === 0) { + return 0; + } + return response[0].tx_timestamp; +} diff --git a/packages/pipeline/src/scripts/pull_ddex_orderbook_snapshots.ts b/packages/pipeline/src/scripts/pull_ddex_orderbook_snapshots.ts new file mode 100644 index 000000000..7868e9c5a --- /dev/null +++ b/packages/pipeline/src/scripts/pull_ddex_orderbook_snapshots.ts @@ -0,0 +1,55 @@ +import { logUtils } from '@0x/utils'; +import * as R from 'ramda'; +import { Connection, ConnectionOptions, createConnection } from 'typeorm'; + +import { DDEX_SOURCE, DdexMarket, DdexSource } from '../data_sources/ddex'; +import { TokenOrderbookSnapshot as TokenOrder } from '../entities'; +import * as ormConfig from '../ormconfig'; +import { parseDdexOrders } from '../parsers/ddex_orders'; +import { handleError } from '../utils'; + +// Number of orders to save at once. +const BATCH_SAVE_SIZE = 1000; + +// Number of markets to retrieve orderbooks for at once. +const MARKET_ORDERBOOK_REQUEST_BATCH_SIZE = 50; + +// Delay between market orderbook requests. +const MILLISEC_MARKET_ORDERBOOK_REQUEST_DELAY = 5000; + +let connection: Connection; + +(async () => { + connection = await createConnection(ormConfig as ConnectionOptions); + const ddexSource = new DdexSource(); + const markets = await ddexSource.getActiveMarketsAsync(); + for (const marketsChunk of R.splitEvery(MARKET_ORDERBOOK_REQUEST_BATCH_SIZE, markets)) { + await Promise.all( + marketsChunk.map(async (market: DdexMarket) => getAndSaveMarketOrderbook(ddexSource, market)), + ); + await new Promise<void>(resolve => setTimeout(resolve, MILLISEC_MARKET_ORDERBOOK_REQUEST_DELAY)); + } + process.exit(0); +})().catch(handleError); + +/** + * Retrieve orderbook from Ddex API for a given market. Parse orders and insert + * them into our database. + * @param ddexSource Data source which can query Ddex API. + * @param market Object from Ddex API containing market data. + */ +async function getAndSaveMarketOrderbook(ddexSource: DdexSource, market: DdexMarket): Promise<void> { + const orderBook = await ddexSource.getMarketOrderbookAsync(market.id); + const observedTimestamp = Date.now(); + + logUtils.log(`${market.id}: Parsing orders.`); + const orders = parseDdexOrders(orderBook, market, observedTimestamp, DDEX_SOURCE); + + if (orders.length > 0) { + logUtils.log(`${market.id}: Saving ${orders.length} orders.`); + const TokenOrderRepository = connection.getRepository(TokenOrder); + await TokenOrderRepository.save(orders, { chunk: Math.ceil(orders.length / BATCH_SAVE_SIZE) }); + } else { + logUtils.log(`${market.id}: 0 orders to save.`); + } +} diff --git a/packages/pipeline/src/scripts/pull_missing_blocks.ts b/packages/pipeline/src/scripts/pull_missing_blocks.ts new file mode 100644 index 000000000..b7bd51f08 --- /dev/null +++ b/packages/pipeline/src/scripts/pull_missing_blocks.ts @@ -0,0 +1,80 @@ +// tslint:disable:no-console +import { web3Factory } from '@0x/dev-utils'; +import * as Parallel from 'async-parallel'; +import R = require('ramda'); +import 'reflect-metadata'; +import { Connection, ConnectionOptions, createConnection, Repository } from 'typeorm'; + +import { Web3Source } from '../data_sources/web3'; +import { Block } from '../entities'; +import * as ormConfig from '../ormconfig'; +import { parseBlock } from '../parsers/web3'; +import { EXCHANGE_START_BLOCK, handleError, INFURA_ROOT_URL } from '../utils'; + +// Number of blocks to save at once. +const BATCH_SAVE_SIZE = 1000; +// Maximum number of requests to send at once. +const MAX_CONCURRENCY = 10; +// Maximum number of blocks to query for at once. This is also the maximum +// number of blocks we will hold in memory prior to being saved to the database. +const MAX_BLOCKS_PER_QUERY = 1000; + +let connection: Connection; + +(async () => { + connection = await createConnection(ormConfig as ConnectionOptions); + const provider = web3Factory.getRpcProvider({ + rpcUrl: `${INFURA_ROOT_URL}/${process.env.INFURA_API_KEY}`, + }); + const web3Source = new Web3Source(provider); + await getAllMissingBlocks(web3Source); + process.exit(0); +})().catch(handleError); + +interface MissingBlocksResponse { + block_number: string; +} + +async function getAllMissingBlocks(web3Source: Web3Source): Promise<void> { + const blocksRepository = connection.getRepository(Block); + let fromBlock = EXCHANGE_START_BLOCK; + while (true) { + const blockNumbers = await getMissingBlockNumbers(fromBlock); + if (blockNumbers.length === 0) { + // There are no more missing blocks. We're done. + break; + } + await getAndSaveBlocks(web3Source, blocksRepository, blockNumbers); + fromBlock = Math.max(...blockNumbers) + 1; + } + const totalBlocks = await blocksRepository.count(); + console.log(`Done saving blocks. There are now ${totalBlocks} total blocks.`); +} + +async function getMissingBlockNumbers(fromBlock: number): Promise<number[]> { + console.log(`Checking for missing blocks starting at ${fromBlock}...`); + const response = (await connection.query( + 'SELECT DISTINCT(block_number) FROM raw.exchange_fill_events WHERE block_number NOT IN (SELECT number FROM raw.blocks) AND block_number >= $1 ORDER BY block_number ASC LIMIT $2', + [fromBlock, MAX_BLOCKS_PER_QUERY], + )) as MissingBlocksResponse[]; + const blockNumberStrings = R.pluck('block_number', response); + const blockNumbers = R.map(parseInt, blockNumberStrings); + console.log(`Found ${blockNumbers.length} missing blocks in the given range.`); + return blockNumbers; +} + +async function getAndSaveBlocks( + web3Source: Web3Source, + blocksRepository: Repository<Block>, + blockNumbers: number[], +): Promise<void> { + console.log(`Getting block data for ${blockNumbers.length} blocks...`); + Parallel.setConcurrency(MAX_CONCURRENCY); + const rawBlocks = await Parallel.map(blockNumbers, async (blockNumber: number) => + web3Source.getBlockInfoAsync(blockNumber), + ); + console.log(`Parsing ${rawBlocks.length} blocks...`); + const blocks = R.map(parseBlock, rawBlocks); + console.log(`Saving ${blocks.length} blocks...`); + await blocksRepository.save(blocks, { chunk: Math.ceil(blocks.length / BATCH_SAVE_SIZE) }); +} diff --git a/packages/pipeline/src/scripts/pull_missing_events.ts b/packages/pipeline/src/scripts/pull_missing_events.ts new file mode 100644 index 000000000..80abbb8b0 --- /dev/null +++ b/packages/pipeline/src/scripts/pull_missing_events.ts @@ -0,0 +1,136 @@ +// tslint:disable:no-console +import { web3Factory } from '@0x/dev-utils'; +import R = require('ramda'); +import 'reflect-metadata'; +import { Connection, ConnectionOptions, createConnection, Repository } from 'typeorm'; + +import { ExchangeEventsSource } from '../data_sources/contract-wrappers/exchange_events'; +import { ExchangeCancelEvent, ExchangeCancelUpToEvent, ExchangeEvent, ExchangeFillEvent } from '../entities'; +import * as ormConfig from '../ormconfig'; +import { parseExchangeCancelEvents, parseExchangeCancelUpToEvents, parseExchangeFillEvents } from '../parsers/events'; +import { EXCHANGE_START_BLOCK, handleError, INFURA_ROOT_URL } from '../utils'; + +const START_BLOCK_OFFSET = 100; // Number of blocks before the last known block to consider when updating fill events. +const BATCH_SAVE_SIZE = 1000; // Number of events to save at once. + +let connection: Connection; + +(async () => { + connection = await createConnection(ormConfig as ConnectionOptions); + const provider = web3Factory.getRpcProvider({ + rpcUrl: INFURA_ROOT_URL, + }); + const eventsSource = new ExchangeEventsSource(provider, 1); + await getFillEventsAsync(eventsSource); + await getCancelEventsAsync(eventsSource); + await getCancelUpToEventsAsync(eventsSource); + process.exit(0); +})().catch(handleError); + +async function getFillEventsAsync(eventsSource: ExchangeEventsSource): Promise<void> { + console.log('Checking existing fill events...'); + const repository = connection.getRepository(ExchangeFillEvent); + const startBlock = await getStartBlockAsync(repository); + console.log(`Getting fill events starting at ${startBlock}...`); + const eventLogs = await eventsSource.getFillEventsAsync(startBlock); + console.log('Parsing fill events...'); + const events = parseExchangeFillEvents(eventLogs); + console.log(`Retrieved and parsed ${events.length} total fill events.`); + await saveEventsAsync(startBlock === EXCHANGE_START_BLOCK, repository, events); +} + +async function getCancelEventsAsync(eventsSource: ExchangeEventsSource): Promise<void> { + console.log('Checking existing cancel events...'); + const repository = connection.getRepository(ExchangeCancelEvent); + const startBlock = await getStartBlockAsync(repository); + console.log(`Getting cancel events starting at ${startBlock}...`); + const eventLogs = await eventsSource.getCancelEventsAsync(startBlock); + console.log('Parsing cancel events...'); + const events = parseExchangeCancelEvents(eventLogs); + console.log(`Retrieved and parsed ${events.length} total cancel events.`); + await saveEventsAsync(startBlock === EXCHANGE_START_BLOCK, repository, events); +} + +async function getCancelUpToEventsAsync(eventsSource: ExchangeEventsSource): Promise<void> { + console.log('Checking existing CancelUpTo events...'); + const repository = connection.getRepository(ExchangeCancelUpToEvent); + const startBlock = await getStartBlockAsync(repository); + console.log(`Getting CancelUpTo events starting at ${startBlock}...`); + const eventLogs = await eventsSource.getCancelUpToEventsAsync(startBlock); + console.log('Parsing CancelUpTo events...'); + const events = parseExchangeCancelUpToEvents(eventLogs); + console.log(`Retrieved and parsed ${events.length} total CancelUpTo events.`); + await saveEventsAsync(startBlock === EXCHANGE_START_BLOCK, repository, events); +} + +const tableNameRegex = /^[a-zA-Z_]*$/; + +async function getStartBlockAsync<T extends ExchangeEvent>(repository: Repository<T>): Promise<number> { + const fillEventCount = await repository.count(); + if (fillEventCount === 0) { + console.log(`No existing ${repository.metadata.name}s found.`); + return EXCHANGE_START_BLOCK; + } + const tableName = repository.metadata.tableName; + if (!tableNameRegex.test(tableName)) { + throw new Error(`Unexpected special character in table name: ${tableName}`); + } + const queryResult = await connection.query( + `SELECT block_number FROM raw.${tableName} ORDER BY block_number DESC LIMIT 1`, + ); + const lastKnownBlock = queryResult[0].block_number; + return lastKnownBlock - START_BLOCK_OFFSET; +} + +async function saveEventsAsync<T extends ExchangeEvent>( + isInitialPull: boolean, + repository: Repository<T>, + events: T[], +): Promise<void> { + console.log(`Saving ${repository.metadata.name}s...`); + if (isInitialPull) { + // Split data into numChunks pieces of maximum size BATCH_SAVE_SIZE + // each. + for (const eventsBatch of R.splitEvery(BATCH_SAVE_SIZE, events)) { + await repository.insert(eventsBatch); + } + } else { + // If we possibly have some overlap where we need to update some + // existing events, we need to use our workaround/fallback. + await saveIndividuallyWithFallbackAsync(repository, events); + } + const totalEvents = await repository.count(); + console.log(`Done saving events. There are now ${totalEvents} total ${repository.metadata.name}s.`); +} + +async function saveIndividuallyWithFallbackAsync<T extends ExchangeEvent>( + repository: Repository<T>, + events: T[], +): Promise<void> { + // Note(albrow): This is a temporary hack because `save` is not working as + // documented and is causing a foreign key constraint violation. Hopefully + // can remove later because this "poor man's upsert" implementation operates + // on one event at a time and is therefore much slower. + for (const event of events) { + try { + // First try an insert. + await repository.insert(event); + } catch { + // If it fails, assume it was a foreign key constraint error and try + // doing an update instead. + // Note(albrow): Unfortunately the `as any` hack here seems + // required. I can't figure out how to convince the type-checker + // that the criteria and the entity itself are the correct type for + // the given repository. If we can remove the `save` hack then this + // will probably no longer be necessary. + await repository.update( + { + contractAddress: event.contractAddress, + blockNumber: event.blockNumber, + logIndex: event.logIndex, + } as any, + event as any, + ); + } + } +} diff --git a/packages/pipeline/src/scripts/pull_ohlcv_cryptocompare.ts b/packages/pipeline/src/scripts/pull_ohlcv_cryptocompare.ts new file mode 100644 index 000000000..6979cd10e --- /dev/null +++ b/packages/pipeline/src/scripts/pull_ohlcv_cryptocompare.ts @@ -0,0 +1,101 @@ +// tslint:disable:no-console +import { Connection, ConnectionOptions, createConnection, Repository } from 'typeorm'; + +import { CryptoCompareOHLCVSource } from '../data_sources/ohlcv_external/crypto_compare'; +import { OHLCVExternal } from '../entities'; +import * as ormConfig from '../ormconfig'; +import { OHLCVMetadata, parseRecords } from '../parsers/ohlcv_external/crypto_compare'; +import { handleError } from '../utils'; +import { fetchOHLCVTradingPairsAsync, TradingPair } from '../utils/get_ohlcv_trading_pairs'; + +const SOURCE_NAME = 'CryptoCompare'; +const TWO_HOURS_AGO = new Date().getTime() - 2 * 60 * 60 * 1000; // tslint:disable-line:custom-no-magic-numbers +const ONE_HOUR_AGO = new Date().getTime() - 60 * 60 * 1000; // tslint:disable-line:custom-no-magic-numbers +const ONE_SECOND = 1000; + +const MAX_CONCURRENT_REQUESTS = parseInt(process.env.CRYPTOCOMPARE_MAX_CONCURRENT_REQUESTS || '14', 10); // tslint:disable-line:custom-no-magic-numbers +const EARLIEST_BACKFILL_DATE = process.env.OHLCV_EARLIEST_BACKFILL_DATE || '2010-09-01'; // the time when BTC/USD info starts appearing on Crypto Compare +const EARLIEST_BACKFILL_TIME = new Date(EARLIEST_BACKFILL_DATE).getTime(); + +let connection: Connection; + +(async () => { + connection = await createConnection(ormConfig as ConnectionOptions); + const repository = connection.getRepository(OHLCVExternal); + const source = new CryptoCompareOHLCVSource(MAX_CONCURRENT_REQUESTS); + + const jobTime = new Date().getTime(); + const tradingPairs = await fetchOHLCVTradingPairsAsync(connection, SOURCE_NAME, EARLIEST_BACKFILL_TIME); + console.log(`Starting ${tradingPairs.length} job(s) to scrape Crypto Compare for OHLCV records...`); + + const fetchAndSavePromises = tradingPairs.map(async pair => { + const pairs = source.generateBackfillIntervals(pair); + return fetchAndSaveAsync(source, repository, jobTime, pairs); + }); + await Promise.all(fetchAndSavePromises); + console.log(`Finished scraping OHLCV records from Crypto Compare, exiting...`); + process.exit(0); +})().catch(handleError); + +async function fetchAndSaveAsync( + source: CryptoCompareOHLCVSource, + repository: Repository<OHLCVExternal>, + jobTime: number, + pairs: TradingPair[], +): Promise<void> { + const sortAscTimestamp = (a: TradingPair, b: TradingPair): number => { + if (a.latestSavedTime < b.latestSavedTime) { + return -1; + } else if (a.latestSavedTime > b.latestSavedTime) { + return 1; + } else { + return 0; + } + }; + pairs.sort(sortAscTimestamp); + + let i = 0; + while (i < pairs.length) { + const pair = pairs[i]; + if (pair.latestSavedTime > TWO_HOURS_AGO) { + break; + } + const rawRecords = await source.getHourlyOHLCVAsync(pair); + const records = rawRecords.filter(rec => { + return rec.time * ONE_SECOND < ONE_HOUR_AGO && rec.time * ONE_SECOND > pair.latestSavedTime; + }); // Crypto Compare can take ~30mins to finalise records + if (records.length === 0) { + console.log(`No more records, stopping task for ${JSON.stringify(pair)}`); + break; + } + const metadata: OHLCVMetadata = { + exchange: source.default_exchange, + fromSymbol: pair.fromSymbol, + toSymbol: pair.toSymbol, + source: SOURCE_NAME, + observedTimestamp: jobTime, + interval: source.intervalBetweenRecords, + }; + const parsedRecords = parseRecords(records, metadata); + try { + await saveRecordsAsync(repository, parsedRecords); + i++; + } catch (err) { + console.log(`Error saving OHLCVRecords, stopping task for ${JSON.stringify(pair)} [${err}]`); + break; + } + } + return Promise.resolve(); +} + +async function saveRecordsAsync(repository: Repository<OHLCVExternal>, records: OHLCVExternal[]): Promise<void> { + const metadata = [ + records[0].fromSymbol, + records[0].toSymbol, + new Date(records[0].startTime), + new Date(records[records.length - 1].endTime), + ]; + + console.log(`Saving ${records.length} records to ${repository.metadata.name}... ${JSON.stringify(metadata)}`); + await repository.save(records); +} diff --git a/packages/pipeline/src/scripts/pull_paradex_orderbook_snapshots.ts b/packages/pipeline/src/scripts/pull_paradex_orderbook_snapshots.ts new file mode 100644 index 000000000..bae1fbede --- /dev/null +++ b/packages/pipeline/src/scripts/pull_paradex_orderbook_snapshots.ts @@ -0,0 +1,87 @@ +import { logUtils } from '@0x/utils'; +import { Connection, ConnectionOptions, createConnection } from 'typeorm'; + +import { + PARADEX_SOURCE, + ParadexActiveMarketsResponse, + ParadexMarket, + ParadexSource, + ParadexTokenInfoResponse, +} from '../data_sources/paradex'; +import { TokenOrderbookSnapshot as TokenOrder } from '../entities'; +import * as ormConfig from '../ormconfig'; +import { parseParadexOrders } from '../parsers/paradex_orders'; +import { handleError } from '../utils'; + +// Number of orders to save at once. +const BATCH_SAVE_SIZE = 1000; + +let connection: Connection; + +(async () => { + connection = await createConnection(ormConfig as ConnectionOptions); + const apiKey = process.env.PARADEX_DATA_PIPELINE_API_KEY; + if (apiKey === undefined) { + throw new Error('Missing required env var: PARADEX_DATA_PIPELINE_API_KEY'); + } + const paradexSource = new ParadexSource(apiKey); + const markets = await paradexSource.getActiveMarketsAsync(); + const tokenInfoResponse = await paradexSource.getTokenInfoAsync(); + const extendedMarkets = addTokenAddresses(markets, tokenInfoResponse); + await Promise.all( + extendedMarkets.map(async (market: ParadexMarket) => getAndSaveMarketOrderbook(paradexSource, market)), + ); + process.exit(0); +})().catch(handleError); + +/** + * Extend the default ParadexMarket objects with token addresses. + * @param markets An array of ParadexMarket objects. + * @param tokenInfoResponse An array of ParadexTokenInfo containing the addresses. + */ +function addTokenAddresses( + markets: ParadexActiveMarketsResponse, + tokenInfoResponse: ParadexTokenInfoResponse, +): ParadexMarket[] { + const symbolAddressMapping = new Map<string, string>(); + tokenInfoResponse.forEach(tokenInfo => symbolAddressMapping.set(tokenInfo.symbol, tokenInfo.address)); + + markets.forEach((market: ParadexMarket) => { + if (symbolAddressMapping.has(market.baseToken)) { + market.baseTokenAddress = symbolAddressMapping.get(market.baseToken); + } else { + market.quoteTokenAddress = ''; + logUtils.warn(`${market.baseToken}: No address found.`); + } + + if (symbolAddressMapping.has(market.quoteToken)) { + market.quoteTokenAddress = symbolAddressMapping.get(market.quoteToken); + } else { + market.quoteTokenAddress = ''; + logUtils.warn(`${market.quoteToken}: No address found.`); + } + }); + return markets; +} + +/** + * Retrieve orderbook from Paradex API for a given market. Parse orders and insert + * them into our database. + * @param paradexSource Data source which can query the Paradex API. + * @param market Object from the Paradex API with information about the market in question. + */ +async function getAndSaveMarketOrderbook(paradexSource: ParadexSource, market: ParadexMarket): Promise<void> { + const paradexOrderbookResponse = await paradexSource.getMarketOrderbookAsync(market.symbol); + const observedTimestamp = Date.now(); + + logUtils.log(`${market.symbol}: Parsing orders.`); + const orders = parseParadexOrders(paradexOrderbookResponse, market, observedTimestamp, PARADEX_SOURCE); + + if (orders.length > 0) { + logUtils.log(`${market.symbol}: Saving ${orders.length} orders.`); + const tokenOrderRepository = connection.getRepository(TokenOrder); + await tokenOrderRepository.save(orders, { chunk: Math.ceil(orders.length / BATCH_SAVE_SIZE) }); + } else { + logUtils.log(`${market.symbol}: 0 orders to save.`); + } +} diff --git a/packages/pipeline/src/scripts/pull_radar_relay_orders.ts b/packages/pipeline/src/scripts/pull_radar_relay_orders.ts new file mode 100644 index 000000000..40bb6fc97 --- /dev/null +++ b/packages/pipeline/src/scripts/pull_radar_relay_orders.ts @@ -0,0 +1,54 @@ +// tslint:disable:no-console +import { HttpClient } from '@0x/connect'; +import * as R from 'ramda'; +import 'reflect-metadata'; +import { Connection, ConnectionOptions, createConnection, EntityManager } from 'typeorm'; + +import { createObservedTimestampForOrder, SraOrder } from '../entities'; +import * as ormConfig from '../ormconfig'; +import { parseSraOrders } from '../parsers/sra_orders'; +import { handleError } from '../utils'; + +const RADAR_RELAY_URL = 'https://api.radarrelay.com/0x/v2'; +const ORDERS_PER_PAGE = 10000; // Number of orders to get per request. + +let connection: Connection; + +(async () => { + connection = await createConnection(ormConfig as ConnectionOptions); + await getOrderbookAsync(); + process.exit(0); +})().catch(handleError); + +async function getOrderbookAsync(): Promise<void> { + console.log('Getting all orders...'); + const connectClient = new HttpClient(RADAR_RELAY_URL); + const rawOrders = await connectClient.getOrdersAsync({ + perPage: ORDERS_PER_PAGE, + }); + console.log(`Got ${rawOrders.records.length} orders.`); + console.log('Parsing orders...'); + // Parse the sra orders, then add source url to each. + const orders = R.pipe(parseSraOrders, R.map(setSourceUrl(RADAR_RELAY_URL)))(rawOrders); + // Save all the orders and update the observed time stamps in a single + // transaction. + console.log('Saving orders and updating timestamps...'); + const observedTimestamp = Date.now(); + await connection.transaction(async (manager: EntityManager): Promise<void> => { + for (const order of orders) { + await manager.save(SraOrder, order); + const orderObservation = createObservedTimestampForOrder(order, observedTimestamp); + await manager.save(orderObservation); + } + }); +} + +const sourceUrlProp = R.lensProp('sourceUrl'); + +/** + * Sets the source url for a single order. Returns a new order instead of + * mutating the given one. + */ +const setSourceUrl = R.curry((sourceURL: string, order: SraOrder): SraOrder => { + return R.set(sourceUrlProp, sourceURL, order); +}); diff --git a/packages/pipeline/src/scripts/pull_trusted_tokens.ts b/packages/pipeline/src/scripts/pull_trusted_tokens.ts new file mode 100644 index 000000000..1befc4437 --- /dev/null +++ b/packages/pipeline/src/scripts/pull_trusted_tokens.ts @@ -0,0 +1,52 @@ +import 'reflect-metadata'; +import { Connection, ConnectionOptions, createConnection } from 'typeorm'; + +import { MetamaskTrustedTokenMeta, TrustedTokenSource, ZeroExTrustedTokenMeta } from '../data_sources/trusted_tokens'; +import { TokenMetadata } from '../entities'; +import * as ormConfig from '../ormconfig'; +import { parseMetamaskTrustedTokens, parseZeroExTrustedTokens } from '../parsers/token_metadata'; +import { handleError } from '../utils'; + +const METAMASK_TRUSTED_TOKENS_URL = + 'https://raw.githubusercontent.com/MetaMask/eth-contract-metadata/d45916c533116510cc8e9e048a8b5fc3732a6b6d/contract-map.json'; + +const ZEROEX_TRUSTED_TOKENS_URL = 'https://website-api.0xproject.com/tokens'; + +let connection: Connection; + +(async () => { + connection = await createConnection(ormConfig as ConnectionOptions); + await getMetamaskTrustedTokens(); + await getZeroExTrustedTokens(); + process.exit(0); +})().catch(handleError); + +async function getMetamaskTrustedTokens(): Promise<void> { + // tslint:disable-next-line:no-console + console.log('Getting latest metamask trusted tokens list ...'); + const trustedTokensRepository = connection.getRepository(TokenMetadata); + const trustedTokensSource = new TrustedTokenSource<Map<string, MetamaskTrustedTokenMeta>>( + METAMASK_TRUSTED_TOKENS_URL, + ); + const resp = await trustedTokensSource.getTrustedTokenMetaAsync(); + const trustedTokens = parseMetamaskTrustedTokens(resp); + // tslint:disable-next-line:no-console + console.log('Saving metamask trusted tokens list'); + await trustedTokensRepository.save(trustedTokens); + // tslint:disable-next-line:no-console + console.log('Done saving metamask trusted tokens.'); +} + +async function getZeroExTrustedTokens(): Promise<void> { + // tslint:disable-next-line:no-console + console.log('Getting latest 0x trusted tokens list ...'); + const trustedTokensRepository = connection.getRepository(TokenMetadata); + const trustedTokensSource = new TrustedTokenSource<ZeroExTrustedTokenMeta[]>(ZEROEX_TRUSTED_TOKENS_URL); + const resp = await trustedTokensSource.getTrustedTokenMetaAsync(); + const trustedTokens = parseZeroExTrustedTokens(resp); + // tslint:disable-next-line:no-console + console.log('Saving metamask trusted tokens list'); + await trustedTokensRepository.save(trustedTokens); + // tslint:disable-next-line:no-console + console.log('Done saving metamask trusted tokens.'); +} diff --git a/packages/pipeline/src/scripts/update_relayer_info.ts b/packages/pipeline/src/scripts/update_relayer_info.ts new file mode 100644 index 000000000..f8918728d --- /dev/null +++ b/packages/pipeline/src/scripts/update_relayer_info.ts @@ -0,0 +1,33 @@ +// tslint:disable:no-console +import 'reflect-metadata'; +import { Connection, ConnectionOptions, createConnection } from 'typeorm'; + +import { RelayerRegistrySource } from '../data_sources/relayer-registry'; +import { Relayer } from '../entities'; +import * as ormConfig from '../ormconfig'; +import { parseRelayers } from '../parsers/relayer_registry'; +import { handleError } from '../utils'; + +// NOTE(albrow): We need to manually update this URL for now. Fix this when we +// have the relayer-registry behind semantic versioning. +const RELAYER_REGISTRY_URL = + 'https://raw.githubusercontent.com/0xProject/0x-relayer-registry/4701c85677d161ea729a466aebbc1826c6aa2c0b/relayers.json'; + +let connection: Connection; + +(async () => { + connection = await createConnection(ormConfig as ConnectionOptions); + await getRelayers(); + process.exit(0); +})().catch(handleError); + +async function getRelayers(): Promise<void> { + console.log('Getting latest relayer info...'); + const relayerRepository = connection.getRepository(Relayer); + const relayerSource = new RelayerRegistrySource(RELAYER_REGISTRY_URL); + const relayersResp = await relayerSource.getRelayerInfoAsync(); + const relayers = parseRelayers(relayersResp); + console.log('Saving relayer info...'); + await relayerRepository.save(relayers); + console.log('Done saving relayer info.'); +} diff --git a/packages/pipeline/src/types.ts b/packages/pipeline/src/types.ts new file mode 100644 index 000000000..e02b42a40 --- /dev/null +++ b/packages/pipeline/src/types.ts @@ -0,0 +1,2 @@ +export type AssetType = 'erc20' | 'erc721'; +export type OrderType = 'bid' | 'ask'; diff --git a/packages/pipeline/src/utils/constants.ts b/packages/pipeline/src/utils/constants.ts new file mode 100644 index 000000000..56f3e82d8 --- /dev/null +++ b/packages/pipeline/src/utils/constants.ts @@ -0,0 +1,3 @@ +// Block number when the Exchange contract was deployed to mainnet. +export const EXCHANGE_START_BLOCK = 6271590; +export const INFURA_ROOT_URL = 'https://mainnet.infura.io'; diff --git a/packages/pipeline/src/utils/get_ohlcv_trading_pairs.ts b/packages/pipeline/src/utils/get_ohlcv_trading_pairs.ts new file mode 100644 index 000000000..9d3ef2fba --- /dev/null +++ b/packages/pipeline/src/utils/get_ohlcv_trading_pairs.ts @@ -0,0 +1,92 @@ +import { fetchAsync } from '@0x/utils'; +import * as R from 'ramda'; +import { Connection } from 'typeorm'; + +export interface TradingPair { + fromSymbol: string; + toSymbol: string; + latestSavedTime: number; +} + +const COINLIST_API = 'https://min-api.cryptocompare.com/data/all/coinlist?BuiltOn=7605'; + +interface CryptoCompareCoinListResp { + Data: Map<string, CryptoCompareCoin>; +} + +interface CryptoCompareCoin { + Symbol: string; + BuiltOn: string; + SmartContractAddress: string; +} + +const TO_CURRENCIES = ['USD', 'EUR', 'ETH', 'USDT']; +const ETHEREUM_IDENTIFIER = '7605'; +const HTTP_OK_STATUS = 200; +/** + * Get trading pairs with latest scraped time for OHLCV records + * @param conn a typeorm Connection to postgres + */ +export async function fetchOHLCVTradingPairsAsync( + conn: Connection, + source: string, + earliestBackfillTime: number, +): Promise<TradingPair[]> { + // fetch existing ohlcv records + const latestTradingPairs: Array<{ + from_symbol: string; + to_symbol: string; + latest: string; + }> = await conn.query(`SELECT + MAX(end_time) as latest, + from_symbol, + to_symbol + FROM raw.ohlcv_external + GROUP BY from_symbol, to_symbol;`); + + const latestTradingPairsIndex: { [fromSym: string]: { [toSym: string]: number } } = {}; + latestTradingPairs.forEach(pair => { + const latestIndex: { [toSym: string]: number } = latestTradingPairsIndex[pair.from_symbol] || {}; + latestIndex[pair.to_symbol] = parseInt(pair.latest, 10); // tslint:disable-line:custom-no-magic-numbers + latestTradingPairsIndex[pair.from_symbol] = latestIndex; + }); + + // get token symbols used by Crypto Compare + const allCoinsResp = await fetchAsync(COINLIST_API); + if (allCoinsResp.status !== HTTP_OK_STATUS) { + return []; + } + const allCoins: CryptoCompareCoinListResp = await allCoinsResp.json(); + const erc20CoinsIndex: Map<string, string> = new Map(); + Object.entries(allCoins.Data).forEach(pair => { + const [symbol, coinData] = pair; + if (coinData.BuiltOn === ETHEREUM_IDENTIFIER && coinData.SmartContractAddress !== 'N/A') { + erc20CoinsIndex.set(coinData.SmartContractAddress.toLowerCase(), symbol); + } + }); + + // fetch all tokens that are traded on 0x + const rawTokenAddresses: Array<{ tokenaddress: string }> = await conn.query( + `SELECT DISTINCT(maker_token_address) as tokenaddress FROM raw.exchange_fill_events UNION + SELECT DISTINCT(taker_token_address) as tokenaddress FROM raw.exchange_fill_events`, + ); + const tokenAddresses = R.pluck('tokenaddress', rawTokenAddresses); + + // join token addresses with CC symbols + const allTokenSymbols: string[] = tokenAddresses + .map(tokenAddress => erc20CoinsIndex.get(tokenAddress.toLowerCase()) || '') + .filter(x => x); + + // generate list of all tokens with time of latest existing record OR default earliest time + const allTradingPairCombinations: TradingPair[] = R.chain(sym => { + return TO_CURRENCIES.map(fiat => { + return { + fromSymbol: sym, + toSymbol: fiat, + latestSavedTime: R.path<number>([sym, fiat], latestTradingPairsIndex) || earliestBackfillTime, + }; + }); + }, allTokenSymbols); + + return allTradingPairCombinations; +} diff --git a/packages/pipeline/src/utils/index.ts b/packages/pipeline/src/utils/index.ts new file mode 100644 index 000000000..2096a0a39 --- /dev/null +++ b/packages/pipeline/src/utils/index.ts @@ -0,0 +1,38 @@ +import { BigNumber } from '@0x/utils'; +export * from './transformers'; +export * from './constants'; + +/** + * If the given BigNumber is not null, returns the string representation of that + * number. Otherwise, returns null. + * @param n The number to convert. + */ +export function bigNumbertoStringOrNull(n: BigNumber): string | null { + if (n == null) { + return null; + } + return n.toString(); +} + +/** + * Logs an error by intelligently checking for `message` and `stack` properties. + * Intended for use with top-level immediately invoked asynchronous functions. + * @param e the error to log. + */ +export function handleError(e: any): void { + if (e.message != null) { + // tslint:disable-next-line:no-console + console.error(e.message); + } else { + // tslint:disable-next-line:no-console + console.error('Unknown error'); + } + if (e.stack != null) { + // tslint:disable-next-line:no-console + console.error(e.stack); + } else { + // tslint:disable-next-line:no-console + console.error('(No stack trace)'); + } + process.exit(1); +} diff --git a/packages/pipeline/src/utils/transformers/big_number.ts b/packages/pipeline/src/utils/transformers/big_number.ts new file mode 100644 index 000000000..5f2e4d565 --- /dev/null +++ b/packages/pipeline/src/utils/transformers/big_number.ts @@ -0,0 +1,16 @@ +import { BigNumber } from '@0x/utils'; +import { ValueTransformer } from 'typeorm/decorator/options/ValueTransformer'; + +export class BigNumberTransformer implements ValueTransformer { + // tslint:disable-next-line:prefer-function-over-method + public to(value: BigNumber | null): string | null { + return value === null ? null : value.toString(); + } + + // tslint:disable-next-line:prefer-function-over-method + public from(value: string | null): BigNumber | null { + return value === null ? null : new BigNumber(value); + } +} + +export const bigNumberTransformer = new BigNumberTransformer(); diff --git a/packages/pipeline/src/utils/transformers/index.ts b/packages/pipeline/src/utils/transformers/index.ts new file mode 100644 index 000000000..232c1c5de --- /dev/null +++ b/packages/pipeline/src/utils/transformers/index.ts @@ -0,0 +1,2 @@ +export * from './big_number'; +export * from './number_to_bigint'; diff --git a/packages/pipeline/src/utils/transformers/number_to_bigint.ts b/packages/pipeline/src/utils/transformers/number_to_bigint.ts new file mode 100644 index 000000000..85560c1f0 --- /dev/null +++ b/packages/pipeline/src/utils/transformers/number_to_bigint.ts @@ -0,0 +1,27 @@ +import { BigNumber } from '@0x/utils'; +import { ValueTransformer } from 'typeorm/decorator/options/ValueTransformer'; + +const decimalRadix = 10; + +// Can be used to convert a JavaScript number type to a Postgres bigint type and +// vice versa. By default TypeORM will silently convert number types to string +// if the corresponding Postgres type is bigint. See +// https://github.com/typeorm/typeorm/issues/2400 for more information. +export class NumberToBigIntTransformer implements ValueTransformer { + // tslint:disable-next-line:prefer-function-over-method + public to(value: number): string { + return value.toString(); + } + + // tslint:disable-next-line:prefer-function-over-method + public from(value: string): number { + if (new BigNumber(value).greaterThan(Number.MAX_SAFE_INTEGER)) { + throw new Error( + `Attempted to convert PostgreSQL bigint value (${value}) to JavaScript number type but it is too big to safely convert`, + ); + } + return Number.parseInt(value, decimalRadix); + } +} + +export const numberToBigIntTransformer = new NumberToBigIntTransformer(); diff --git a/packages/pipeline/test/data_sources/ohlcv_external/crypto_compare_test.ts b/packages/pipeline/test/data_sources/ohlcv_external/crypto_compare_test.ts new file mode 100644 index 000000000..cb374bbb1 --- /dev/null +++ b/packages/pipeline/test/data_sources/ohlcv_external/crypto_compare_test.ts @@ -0,0 +1,47 @@ +import * as chai from 'chai'; +import 'mocha'; +import * as R from 'ramda'; + +import { CryptoCompareOHLCVSource } from '../../../src/data_sources/ohlcv_external/crypto_compare'; +import { TradingPair } from '../../../src/utils/get_ohlcv_trading_pairs'; +import { chaiSetup } from '../../utils/chai_setup'; + +chaiSetup.configure(); +const expect = chai.expect; + +// tslint:disable:custom-no-magic-numbers +describe('ohlcv_external data source (Crypto Compare)', () => { + describe('generateBackfillIntervals', () => { + it('generates pairs with intervals to query', () => { + const source = new CryptoCompareOHLCVSource(); + const pair: TradingPair = { + fromSymbol: 'ETH', + toSymbol: 'ZRX', + latestSavedTime: new Date().getTime() - source.interval * 2, + }; + + const expected = [ + pair, + R.merge(pair, { latestSavedTime: pair.latestSavedTime + source.interval }), + R.merge(pair, { latestSavedTime: pair.latestSavedTime + source.interval * 2 }), + ]; + + const actual = source.generateBackfillIntervals(pair); + expect(actual).deep.equal(expected); + }); + + it('returns single pair if no backfill is needed', () => { + const source = new CryptoCompareOHLCVSource(); + const pair: TradingPair = { + fromSymbol: 'ETH', + toSymbol: 'ZRX', + latestSavedTime: new Date().getTime() - source.interval + 5000, + }; + + const expected = [pair]; + + const actual = source.generateBackfillIntervals(pair); + expect(actual).deep.equal(expected); + }); + }); +}); diff --git a/packages/pipeline/test/db_global_hooks.ts b/packages/pipeline/test/db_global_hooks.ts new file mode 100644 index 000000000..dfee02c45 --- /dev/null +++ b/packages/pipeline/test/db_global_hooks.ts @@ -0,0 +1,9 @@ +import { setUpDbAsync, tearDownDbAsync } from './db_setup'; + +before('set up database', async () => { + await setUpDbAsync(); +}); + +after('tear down database', async () => { + await tearDownDbAsync(); +}); diff --git a/packages/pipeline/test/db_setup.ts b/packages/pipeline/test/db_setup.ts new file mode 100644 index 000000000..bf31d15b6 --- /dev/null +++ b/packages/pipeline/test/db_setup.ts @@ -0,0 +1,174 @@ +import * as Docker from 'dockerode'; +import * as fs from 'fs'; +import * as R from 'ramda'; +import { Connection, ConnectionOptions, createConnection } from 'typeorm'; + +import * as ormConfig from '../src/ormconfig'; + +// The name of the image to pull and use for the container. This also affects +// which version of Postgres we use. +const DOCKER_IMAGE_NAME = 'postgres:11-alpine'; +// The name to use for the Docker container which will run Postgres. +const DOCKER_CONTAINER_NAME = '0x_pipeline_postgres_test'; +// The port which will be exposed on the Docker container. +const POSTGRES_HOST_PORT = '15432'; +// Number of milliseconds to wait for postgres to finish initializing after +// starting the docker container. +const POSTGRES_SETUP_DELAY_MS = 5000; + +/** + * Sets up the database for testing purposes. If the + * ZEROEX_DATA_PIPELINE_TEST_DB_URL env var is specified, it will create a + * connection using that url. Otherwise it will spin up a new Docker container + * with a Postgres database and then create a connection to that database. + */ +export async function setUpDbAsync(): Promise<void> { + const connection = await createDbConnectionOnceAsync(); + await connection.runMigrations({ transaction: true }); +} + +/** + * Tears down the database used for testing. This completely destroys any data. + * If a docker container was created, it destroys that container too. + */ +export async function tearDownDbAsync(): Promise<void> { + const connection = await createDbConnectionOnceAsync(); + for (const _ of connection.migrations) { + await connection.undoLastMigration({ transaction: true }); + } + if (needsDocker()) { + const docker = initDockerOnce(); + const postgresContainer = docker.getContainer(DOCKER_CONTAINER_NAME); + await postgresContainer.kill(); + await postgresContainer.remove(); + } +} + +let savedConnection: Connection; + +/** + * The first time this is run, it creates and returns a new TypeORM connection. + * Each subsequent time, it returns the existing connection. This is helpful + * because only one TypeORM connection can be active at a time. + */ +export async function createDbConnectionOnceAsync(): Promise<Connection> { + if (savedConnection !== undefined) { + return savedConnection; + } + + if (needsDocker()) { + await initContainerAsync(); + } + const testDbUrl = + process.env.ZEROEX_DATA_PIPELINE_TEST_DB_URL || + `postgresql://postgres@localhost:${POSTGRES_HOST_PORT}/postgres`; + const testOrmConfig = R.merge(ormConfig, { url: testDbUrl }) as ConnectionOptions; + + savedConnection = await createConnection(testOrmConfig); + return savedConnection; +} + +async function sleepAsync(ms: number): Promise<{}> { + return new Promise<{}>(resolve => setTimeout(resolve, ms)); +} + +let savedDocker: Docker; + +function initDockerOnce(): Docker { + if (savedDocker !== undefined) { + return savedDocker; + } + + // Note(albrow): Code for determining the right socket path is partially + // based on https://github.com/apocas/dockerode/blob/8f3aa85311fab64d58eca08fef49aa1da5b5f60b/test/spec_helper.js + const isWin = require('os').type() === 'Windows_NT'; + const socketPath = process.env.DOCKER_SOCKET || (isWin ? '//./pipe/docker_engine' : '/var/run/docker.sock'); + const isSocket = fs.existsSync(socketPath) ? fs.statSync(socketPath).isSocket() : false; + if (!isSocket) { + throw new Error(`Failed to connect to Docker using socket path: "${socketPath}". + +The database integration tests need to be able to connect to a Postgres database. Make sure that Docker is running and accessible at the expected socket path. If Docker isn't working you have two options: + + 1) Set the DOCKER_SOCKET environment variable to a socket path that can be used to connect to Docker or + 2) Set the ZEROEX_DATA_PIPELINE_TEST_DB_URL environment variable to connect directly to an existing Postgres database instead of trying to start Postgres via Docker +`); + } + savedDocker = new Docker({ + socketPath, + }); + return savedDocker; +} + +// Creates the container, waits for it to initialize, and returns it. +async function initContainerAsync(): Promise<Docker.Container> { + const docker = initDockerOnce(); + + // Tear down any existing containers with the same name. + await tearDownExistingContainerIfAnyAsync(); + + // Pull the image we need. + await pullImageAsync(docker, DOCKER_IMAGE_NAME); + + // Create the container. + const postgresContainer = await docker.createContainer({ + name: DOCKER_CONTAINER_NAME, + Image: DOCKER_IMAGE_NAME, + ExposedPorts: { + '5432': {}, + }, + HostConfig: { + PortBindings: { + '5432': [ + { + HostPort: POSTGRES_HOST_PORT, + }, + ], + }, + }, + }); + await postgresContainer.start(); + await sleepAsync(POSTGRES_SETUP_DELAY_MS); + return postgresContainer; +} + +async function tearDownExistingContainerIfAnyAsync(): Promise<void> { + const docker = initDockerOnce(); + + // Check if a container with the desired name already exists. If so, this + // probably means we didn't clean up properly on the last test run. + const existingContainer = docker.getContainer(DOCKER_CONTAINER_NAME); + if (existingContainer != null) { + try { + await existingContainer.kill(); + } catch { + // If this fails, it's fine. The container was probably already + // killed. + } + try { + await existingContainer.remove(); + } catch { + // If this fails, it's fine. The container was probably already + // removed. + } + } +} + +function needsDocker(): boolean { + return process.env.ZEROEX_DATA_PIPELINE_TEST_DB_URL === undefined; +} + +// Note(albrow): This is partially based on +// https://stackoverflow.com/questions/38258263/how-do-i-wait-for-a-pull +async function pullImageAsync(docker: Docker, imageName: string): Promise<void> { + return new Promise<void>((resolve, reject) => { + docker.pull(imageName, {}, (err, stream) => { + if (err != null) { + reject(err); + return; + } + docker.modem.followProgress(stream, () => { + resolve(); + }); + }); + }); +} diff --git a/packages/pipeline/test/entities/block_test.ts b/packages/pipeline/test/entities/block_test.ts new file mode 100644 index 000000000..503f284f0 --- /dev/null +++ b/packages/pipeline/test/entities/block_test.ts @@ -0,0 +1,23 @@ +import 'mocha'; +import 'reflect-metadata'; + +import { Block } from '../../src/entities'; +import { createDbConnectionOnceAsync } from '../db_setup'; +import { chaiSetup } from '../utils/chai_setup'; + +import { testSaveAndFindEntityAsync } from './util'; + +chaiSetup.configure(); + +// tslint:disable:custom-no-magic-numbers +describe('Block entity', () => { + it('save/find', async () => { + const connection = await createDbConnectionOnceAsync(); + const block = new Block(); + block.hash = '0x12345'; + block.number = 1234567; + block.timestamp = 5432154321; + const blocksRepository = connection.getRepository(Block); + await testSaveAndFindEntityAsync(blocksRepository, block); + }); +}); diff --git a/packages/pipeline/test/entities/dex_trades_test.ts b/packages/pipeline/test/entities/dex_trades_test.ts new file mode 100644 index 000000000..83aaeec8f --- /dev/null +++ b/packages/pipeline/test/entities/dex_trades_test.ts @@ -0,0 +1,60 @@ +import { BigNumber } from '@0x/utils'; +import 'mocha'; +import * as R from 'ramda'; +import 'reflect-metadata'; + +import { DexTrade } from '../../src/entities'; +import { createDbConnectionOnceAsync } from '../db_setup'; +import { chaiSetup } from '../utils/chai_setup'; + +import { testSaveAndFindEntityAsync } from './util'; + +chaiSetup.configure(); + +const baseTrade = { + sourceUrl: 'https://bloxy.info/api/dex/trades', + txTimestamp: 1543447585938, + txDate: '2018-11-21', + txSender: '0x00923b9a074762b93650716333b3e1473a15048e', + smartContractId: 7091917, + smartContractAddress: '0x818e6fecd516ecc3849daf6845e3ec868087b755', + contractType: 'DEX/Kyber Network Proxy', + maker: '0xbf2179859fc6d5bee9bf9158632dc51678a4100c', + taker: '0xbf2179859fc6d5bee9bf9158632dc51678a4100d', + amountBuy: new BigNumber('1.011943163078103'), + makerFeeAmount: new BigNumber(0), + buyCurrencyId: 1, + buySymbol: 'ETH', + amountSell: new BigNumber('941.4997928436911'), + takerFeeAmount: new BigNumber(0), + sellCurrencyId: 16610, + sellSymbol: 'ELF', + makerAnnotation: '', + takerAnnotation: '', + protocol: 'Kyber Network Proxy', + sellAddress: '0xbf2179859fc6d5bee9bf9158632dc51678a4100e', +}; + +const tradeWithNullAddresses: DexTrade = R.merge(baseTrade, { + txHash: '0xb93a7faf92efbbb5405c9a73cd4efd99702fe27c03ff22baee1f1b1e37b3a0bf', + buyAddress: '0xbf2179859fc6d5bee9bf9158632dc51678a4100e', + sellAddress: '0xbf2179859fc6d5bee9bf9158632dc51678a4100f', +}); + +const tradeWithNonNullAddresses: DexTrade = R.merge(baseTrade, { + txHash: '0xb93a7faf92efbbb5405c9a73cd4efd99702fe27c03ff22baee1f1b1e37b3a0be', + buyAddress: null, + sellAddress: null, +}); + +// tslint:disable:custom-no-magic-numbers +describe('DexTrade entity', () => { + it('save/find', async () => { + const connection = await createDbConnectionOnceAsync(); + const trades = [tradeWithNullAddresses, tradeWithNonNullAddresses]; + const tradesRepository = connection.getRepository(DexTrade); + for (const trade of trades) { + await testSaveAndFindEntityAsync(tradesRepository, trade); + } + }); +}); diff --git a/packages/pipeline/test/entities/exchange_cancel_event_test.ts b/packages/pipeline/test/entities/exchange_cancel_event_test.ts new file mode 100644 index 000000000..f3b306d69 --- /dev/null +++ b/packages/pipeline/test/entities/exchange_cancel_event_test.ts @@ -0,0 +1,57 @@ +import 'mocha'; +import * as R from 'ramda'; +import 'reflect-metadata'; + +import { ExchangeCancelEvent } from '../../src/entities'; +import { AssetType } from '../../src/types'; +import { createDbConnectionOnceAsync } from '../db_setup'; +import { chaiSetup } from '../utils/chai_setup'; + +import { testSaveAndFindEntityAsync } from './util'; + +chaiSetup.configure(); + +const baseCancelEvent = { + contractAddress: '0x4f833a24e1f95d70f028921e27040ca56e09ab0b', + logIndex: 1234, + blockNumber: 6276262, + rawData: '0x000000000000000000000000f6da68519f78b0d0bc93c701e86affcb75c92428', + transactionHash: '0x6dd106d002873746072fc5e496dd0fb2541b68c77bcf9184ae19a42fd33657fe', + makerAddress: '0xf6da68519f78b0d0bc93c701e86affcb75c92428', + takerAddress: '0xf6da68519f78b0d0bc93c701e86affcb75c92428', + feeRecipientAddress: '0xc370d2a5920344aa6b7d8d11250e3e861434cbdd', + senderAddress: '0xf6da68519f78b0d0bc93c701e86affcb75c92428', + orderHash: '0xab12ed2cbaa5615ab690b9da75a46e53ddfcf3f1a68655b5fe0d94c75a1aac4a', + rawMakerAssetData: '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2', + makerAssetProxyId: '0xf47261b0', + makerTokenAddress: '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2', + rawTakerAssetData: '0xf47261b0000000000000000000000000e41d2489571d322189246dafa5ebde1f4699f498', + takerAssetProxyId: '0xf47261b0', + takerTokenAddress: '0xe41d2489571d322189246dafa5ebde1f4699f498', +}; + +const erc20CancelEvent = R.merge(baseCancelEvent, { + makerAssetType: 'erc20' as AssetType, + makerTokenId: null, + takerAssetType: 'erc20' as AssetType, + takerTokenId: null, +}); + +const erc721CancelEvent = R.merge(baseCancelEvent, { + makerAssetType: 'erc721' as AssetType, + makerTokenId: '19378573', + takerAssetType: 'erc721' as AssetType, + takerTokenId: '63885673888', +}); + +// tslint:disable:custom-no-magic-numbers +describe('ExchangeCancelEvent entity', () => { + it('save/find', async () => { + const connection = await createDbConnectionOnceAsync(); + const events = [erc20CancelEvent, erc721CancelEvent]; + const cancelEventRepository = connection.getRepository(ExchangeCancelEvent); + for (const event of events) { + await testSaveAndFindEntityAsync(cancelEventRepository, event); + } + }); +}); diff --git a/packages/pipeline/test/entities/exchange_cancel_up_to_event_test.ts b/packages/pipeline/test/entities/exchange_cancel_up_to_event_test.ts new file mode 100644 index 000000000..aa34f8c1c --- /dev/null +++ b/packages/pipeline/test/entities/exchange_cancel_up_to_event_test.ts @@ -0,0 +1,29 @@ +import { BigNumber } from '@0x/utils'; +import 'mocha'; +import 'reflect-metadata'; + +import { ExchangeCancelUpToEvent } from '../../src/entities'; +import { createDbConnectionOnceAsync } from '../db_setup'; +import { chaiSetup } from '../utils/chai_setup'; + +import { testSaveAndFindEntityAsync } from './util'; + +chaiSetup.configure(); + +// tslint:disable:custom-no-magic-numbers +describe('ExchangeCancelUpToEvent entity', () => { + it('save/find', async () => { + const connection = await createDbConnectionOnceAsync(); + const cancelUpToEventRepository = connection.getRepository(ExchangeCancelUpToEvent); + const cancelUpToEvent = new ExchangeCancelUpToEvent(); + cancelUpToEvent.blockNumber = 6276262; + cancelUpToEvent.contractAddress = '0x4f833a24e1f95d70f028921e27040ca56e09ab0b'; + cancelUpToEvent.logIndex = 42; + cancelUpToEvent.makerAddress = '0xf6da68519f78b0d0bc93c701e86affcb75c92428'; + cancelUpToEvent.orderEpoch = new BigNumber('123456789123456789'); + cancelUpToEvent.rawData = '0x000000000000000000000000f6da68519f78b0d0bc93c701e86affcb75c92428'; + cancelUpToEvent.senderAddress = '0xf6da68519f78b0d0bc93c701e86affcb75c92428'; + cancelUpToEvent.transactionHash = '0x6dd106d002873746072fc5e496dd0fb2541b68c77bcf9184ae19a42fd33657fe'; + await testSaveAndFindEntityAsync(cancelUpToEventRepository, cancelUpToEvent); + }); +}); diff --git a/packages/pipeline/test/entities/exchange_fill_event_test.ts b/packages/pipeline/test/entities/exchange_fill_event_test.ts new file mode 100644 index 000000000..b2cb8c5e0 --- /dev/null +++ b/packages/pipeline/test/entities/exchange_fill_event_test.ts @@ -0,0 +1,62 @@ +import { BigNumber } from '@0x/utils'; +import 'mocha'; +import * as R from 'ramda'; +import 'reflect-metadata'; + +import { ExchangeFillEvent } from '../../src/entities'; +import { AssetType } from '../../src/types'; +import { createDbConnectionOnceAsync } from '../db_setup'; +import { chaiSetup } from '../utils/chai_setup'; + +import { testSaveAndFindEntityAsync } from './util'; + +chaiSetup.configure(); + +const baseFillEvent = { + contractAddress: '0x4f833a24e1f95d70f028921e27040ca56e09ab0b', + blockNumber: 6276262, + logIndex: 102, + rawData: '0x000000000000000000000000f6da68519f78b0d0bc93c701e86affcb75c92428', + transactionHash: '0x6dd106d002873746072fc5e496dd0fb2541b68c77bcf9184ae19a42fd33657fe', + makerAddress: '0xf6da68519f78b0d0bc93c701e86affcb75c92428', + takerAddress: '0xf6da68519f78b0d0bc93c701e86affcb75c92428', + feeRecipientAddress: '0xc370d2a5920344aa6b7d8d11250e3e861434cbdd', + senderAddress: '0xf6da68519f78b0d0bc93c701e86affcb75c92428', + makerAssetFilledAmount: new BigNumber('10000000000000000'), + takerAssetFilledAmount: new BigNumber('100000000000000000'), + makerFeePaid: new BigNumber('0'), + takerFeePaid: new BigNumber('12345'), + orderHash: '0xab12ed2cbaa5615ab690b9da75a46e53ddfcf3f1a68655b5fe0d94c75a1aac4a', + rawMakerAssetData: '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2', + makerAssetProxyId: '0xf47261b0', + makerTokenAddress: '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2', + rawTakerAssetData: '0xf47261b0000000000000000000000000e41d2489571d322189246dafa5ebde1f4699f498', + takerAssetProxyId: '0xf47261b0', + takerTokenAddress: '0xe41d2489571d322189246dafa5ebde1f4699f498', +}; + +const erc20FillEvent = R.merge(baseFillEvent, { + makerAssetType: 'erc20' as AssetType, + makerTokenId: null, + takerAssetType: 'erc20' as AssetType, + takerTokenId: null, +}); + +const erc721FillEvent = R.merge(baseFillEvent, { + makerAssetType: 'erc721' as AssetType, + makerTokenId: '19378573', + takerAssetType: 'erc721' as AssetType, + takerTokenId: '63885673888', +}); + +// tslint:disable:custom-no-magic-numbers +describe('ExchangeFillEvent entity', () => { + it('save/find', async () => { + const connection = await createDbConnectionOnceAsync(); + const events = [erc20FillEvent, erc721FillEvent]; + const fillEventsRepository = connection.getRepository(ExchangeFillEvent); + for (const event of events) { + await testSaveAndFindEntityAsync(fillEventsRepository, event); + } + }); +}); diff --git a/packages/pipeline/test/entities/ohlcv_external_test.ts b/packages/pipeline/test/entities/ohlcv_external_test.ts new file mode 100644 index 000000000..8b995db50 --- /dev/null +++ b/packages/pipeline/test/entities/ohlcv_external_test.ts @@ -0,0 +1,35 @@ +import 'mocha'; +import 'reflect-metadata'; + +import { OHLCVExternal } from '../../src/entities'; +import { createDbConnectionOnceAsync } from '../db_setup'; +import { chaiSetup } from '../utils/chai_setup'; + +import { testSaveAndFindEntityAsync } from './util'; + +chaiSetup.configure(); + +const ohlcvExternal: OHLCVExternal = { + exchange: 'CCCAGG', + fromSymbol: 'ETH', + toSymbol: 'ZRX', + startTime: 1543352400000, + endTime: 1543356000000, + open: 307.41, + close: 310.08, + low: 304.6, + high: 310.27, + volumeFrom: 904.6, + volumeTo: 278238.5, + source: 'Crypto Compare', + observedTimestamp: 1543442338074, +}; + +// tslint:disable:custom-no-magic-numbers +describe('OHLCVExternal entity', () => { + it('save/find', async () => { + const connection = await createDbConnectionOnceAsync(); + const repository = connection.getRepository(OHLCVExternal); + await testSaveAndFindEntityAsync(repository, ohlcvExternal); + }); +}); diff --git a/packages/pipeline/test/entities/relayer_test.ts b/packages/pipeline/test/entities/relayer_test.ts new file mode 100644 index 000000000..760ffb6f9 --- /dev/null +++ b/packages/pipeline/test/entities/relayer_test.ts @@ -0,0 +1,55 @@ +import 'mocha'; +import * as R from 'ramda'; +import 'reflect-metadata'; + +import { Relayer } from '../../src/entities'; +import { createDbConnectionOnceAsync } from '../db_setup'; +import { chaiSetup } from '../utils/chai_setup'; + +import { testSaveAndFindEntityAsync } from './util'; + +chaiSetup.configure(); + +const baseRelayer = { + uuid: 'e8d27d8d-ddf6-48b1-9663-60b0a3ddc716', + name: 'Radar Relay', + homepageUrl: 'https://radarrelay.com', + appUrl: null, + sraHttpEndpoint: null, + sraWsEndpoint: null, + feeRecipientAddresses: [], + takerAddresses: [], +}; + +const relayerWithUrls = R.merge(baseRelayer, { + uuid: 'e8d27d8d-ddf6-48b1-9663-60b0a3ddc717', + appUrl: 'https://app.radarrelay.com', + sraHttpEndpoint: 'https://api.radarrelay.com/0x/v2/', + sraWsEndpoint: 'wss://ws.radarrelay.com/0x/v2', +}); + +const relayerWithAddresses = R.merge(baseRelayer, { + uuid: 'e8d27d8d-ddf6-48b1-9663-60b0a3ddc718', + feeRecipientAddresses: [ + '0xa258b39954cef5cb142fd567a46cddb31a670124', + '0xa258b39954cef5cb142fd567a46cddb31a670125', + '0xa258b39954cef5cb142fd567a46cddb31a670126', + ], + takerAddresses: [ + '0xa258b39954cef5cb142fd567a46cddb31a670127', + '0xa258b39954cef5cb142fd567a46cddb31a670128', + '0xa258b39954cef5cb142fd567a46cddb31a670129', + ], +}); + +// tslint:disable:custom-no-magic-numbers +describe('Relayer entity', () => { + it('save/find', async () => { + const connection = await createDbConnectionOnceAsync(); + const relayers = [baseRelayer, relayerWithUrls, relayerWithAddresses]; + const relayerRepository = connection.getRepository(Relayer); + for (const relayer of relayers) { + await testSaveAndFindEntityAsync(relayerRepository, relayer); + } + }); +}); diff --git a/packages/pipeline/test/entities/sra_order_test.ts b/packages/pipeline/test/entities/sra_order_test.ts new file mode 100644 index 000000000..c43de8ce8 --- /dev/null +++ b/packages/pipeline/test/entities/sra_order_test.ts @@ -0,0 +1,84 @@ +import { BigNumber } from '@0x/utils'; +import 'mocha'; +import * as R from 'ramda'; +import 'reflect-metadata'; +import { Repository } from 'typeorm'; + +import { SraOrder, SraOrdersObservedTimeStamp } from '../../src/entities'; +import { AssetType } from '../../src/types'; +import { createDbConnectionOnceAsync } from '../db_setup'; +import { chaiSetup } from '../utils/chai_setup'; + +import { testSaveAndFindEntityAsync } from './util'; + +chaiSetup.configure(); + +const baseOrder = { + sourceUrl: 'https://api.radarrelay.com/0x/v2', + exchangeAddress: '0x4f833a24e1f95d70f028921e27040ca56e09ab0b', + makerAddress: '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81', + takerAddress: '0x0000000000000000000000000000000000000000', + feeRecipientAddress: '0xa258b39954cef5cb142fd567a46cddb31a670124', + senderAddress: '0x0000000000000000000000000000000000000000', + makerAssetAmount: new BigNumber('1619310371000000000'), + takerAssetAmount: new BigNumber('8178335207070707070707'), + makerFee: new BigNumber('100'), + takerFee: new BigNumber('200'), + expirationTimeSeconds: new BigNumber('1538529488'), + salt: new BigNumber('1537924688891'), + signature: '0x1b5a5d672b0d647b5797387ccbb89d8', + rawMakerAssetData: '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2', + makerAssetProxyId: '0xf47261b0', + makerTokenAddress: '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2', + rawTakerAssetData: '0xf47261b000000000000000000000000042d6622dece394b54999fbd73d108123806f6a18', + takerAssetProxyId: '0xf47261b0', + takerTokenAddress: '0x42d6622dece394b54999fbd73d108123806f6a18', + metadataJson: '{"isThisArbitraryData":true,"powerLevel":9001}', +}; + +const erc20Order = R.merge(baseOrder, { + orderHashHex: '0x1bdbeb0d088a33da28b9ee6d94e8771452f90f4a69107da2fa75195d61b9a1c9', + makerAssetType: 'erc20' as AssetType, + makerTokenId: null, + takerAssetType: 'erc20' as AssetType, + takerTokenId: null, +}); + +const erc721Order = R.merge(baseOrder, { + orderHashHex: '0x1bdbeb0d088a33da28b9ee6d94e8771452f90f4a69107da2fa75195d61b9a1d0', + makerAssetType: 'erc721' as AssetType, + makerTokenId: '19378573', + takerAssetType: 'erc721' as AssetType, + takerTokenId: '63885673888', +}); + +// tslint:disable:custom-no-magic-numbers +describe('SraOrder and SraOrdersObservedTimeStamp entities', () => { + // Note(albrow): SraOrder and SraOrdersObservedTimeStamp are tightly coupled + // and timestamps have a foreign key constraint such that they have to point + // to an existing SraOrder. For these reasons, we are testing them together + // in the same test. + it('save/find', async () => { + const connection = await createDbConnectionOnceAsync(); + const orderRepository = connection.getRepository(SraOrder); + const timestampRepository = connection.getRepository(SraOrdersObservedTimeStamp); + const orders = [erc20Order, erc721Order]; + for (const order of orders) { + await testOrderWithTimestampAsync(orderRepository, timestampRepository, order); + } + }); +}); + +async function testOrderWithTimestampAsync( + orderRepository: Repository<SraOrder>, + timestampRepository: Repository<SraOrdersObservedTimeStamp>, + order: SraOrder, +): Promise<void> { + await testSaveAndFindEntityAsync(orderRepository, order); + const timestamp = new SraOrdersObservedTimeStamp(); + timestamp.exchangeAddress = order.exchangeAddress; + timestamp.orderHashHex = order.orderHashHex; + timestamp.sourceUrl = order.sourceUrl; + timestamp.observedTimestamp = 1543377376153; + await testSaveAndFindEntityAsync(timestampRepository, timestamp); +} diff --git a/packages/pipeline/test/entities/token_metadata_test.ts b/packages/pipeline/test/entities/token_metadata_test.ts new file mode 100644 index 000000000..48e656644 --- /dev/null +++ b/packages/pipeline/test/entities/token_metadata_test.ts @@ -0,0 +1,39 @@ +import { BigNumber } from '@0x/utils'; +import 'mocha'; +import 'reflect-metadata'; + +import { TokenMetadata } from '../../src/entities'; +import { createDbConnectionOnceAsync } from '../db_setup'; +import { chaiSetup } from '../utils/chai_setup'; + +import { testSaveAndFindEntityAsync } from './util'; + +chaiSetup.configure(); + +const metadataWithoutNullFields: TokenMetadata = { + address: '0xe41d2489571d322189246dafa5ebde1f4699f498', + authority: 'https://website-api.0xproject.com/tokens', + decimals: new BigNumber(18), + symbol: 'ZRX', + name: '0x', +}; + +const metadataWithNullFields: TokenMetadata = { + address: '0xe41d2489571d322189246dafa5ebde1f4699f499', + authority: 'https://website-api.0xproject.com/tokens', + decimals: null, + symbol: null, + name: null, +}; + +// tslint:disable:custom-no-magic-numbers +describe('TokenMetadata entity', () => { + it('save/find', async () => { + const connection = await createDbConnectionOnceAsync(); + const tokenMetadata = [metadataWithoutNullFields, metadataWithNullFields]; + const tokenMetadataRepository = connection.getRepository(TokenMetadata); + for (const tokenMetadatum of tokenMetadata) { + await testSaveAndFindEntityAsync(tokenMetadataRepository, tokenMetadatum); + } + }); +}); diff --git a/packages/pipeline/test/entities/token_order_test.ts b/packages/pipeline/test/entities/token_order_test.ts new file mode 100644 index 000000000..c6057f5aa --- /dev/null +++ b/packages/pipeline/test/entities/token_order_test.ts @@ -0,0 +1,31 @@ +import { BigNumber } from '@0x/utils'; +import 'mocha'; + +import { TokenOrderbookSnapshot } from '../../src/entities'; +import { createDbConnectionOnceAsync } from '../db_setup'; +import { chaiSetup } from '../utils/chai_setup'; + +import { testSaveAndFindEntityAsync } from './util'; + +chaiSetup.configure(); + +const tokenOrderbookSnapshot: TokenOrderbookSnapshot = { + source: 'ddextest', + observedTimestamp: Date.now(), + orderType: 'bid', + price: new BigNumber(10.1), + baseAssetSymbol: 'ETH', + baseAssetAddress: '0x818e6fecd516ecc3849daf6845e3ec868087b755', + baseVolume: new BigNumber(143), + quoteAssetSymbol: 'ABC', + quoteAssetAddress: '0x00923b9a074762b93650716333b3e1473a15048e', + quoteVolume: new BigNumber(12.3234234), +}; + +describe('TokenOrderbookSnapshot entity', () => { + it('save/find', async () => { + const connection = await createDbConnectionOnceAsync(); + const tokenOrderbookSnapshotRepository = connection.getRepository(TokenOrderbookSnapshot); + await testSaveAndFindEntityAsync(tokenOrderbookSnapshotRepository, tokenOrderbookSnapshot); + }); +}); diff --git a/packages/pipeline/test/entities/transaction_test.ts b/packages/pipeline/test/entities/transaction_test.ts new file mode 100644 index 000000000..634844544 --- /dev/null +++ b/packages/pipeline/test/entities/transaction_test.ts @@ -0,0 +1,26 @@ +import { BigNumber } from '@0x/utils'; +import 'mocha'; +import 'reflect-metadata'; + +import { Transaction } from '../../src/entities'; +import { createDbConnectionOnceAsync } from '../db_setup'; +import { chaiSetup } from '../utils/chai_setup'; + +import { testSaveAndFindEntityAsync } from './util'; + +chaiSetup.configure(); + +// tslint:disable:custom-no-magic-numbers +describe('Transaction entity', () => { + it('save/find', async () => { + const connection = await createDbConnectionOnceAsync(); + const transactionRepository = connection.getRepository(Transaction); + const transaction = new Transaction(); + transaction.blockHash = '0x6ff106d00b6c3746072fc06bae140fb2549036ba7bcf9184ae19a42fd33657fd'; + transaction.blockNumber = 6276262; + transaction.gasPrice = new BigNumber(3000000); + transaction.gasUsed = new BigNumber(125000); + transaction.transactionHash = '0x6dd106d002873746072fc5e496dd0fb2541b68c77bcf9184ae19a42fd33657fe'; + await testSaveAndFindEntityAsync(transactionRepository, transaction); + }); +}); diff --git a/packages/pipeline/test/entities/util.ts b/packages/pipeline/test/entities/util.ts new file mode 100644 index 000000000..043a3b15d --- /dev/null +++ b/packages/pipeline/test/entities/util.ts @@ -0,0 +1,25 @@ +import * as chai from 'chai'; +import 'mocha'; + +import { Repository } from 'typeorm'; + +const expect = chai.expect; + +/** + * First saves the given entity to the database, then finds it and makes sure + * that the found entity is exactly equal to the original one. This is a bare + * minimum basic test to make sure that the entity type definition and our + * database schema are aligned and that it is possible to save and find the + * entity. + * @param repository A TypeORM repository corresponding with the type of the entity. + * @param entity An instance of a TypeORM entity which will be saved/retrieved from the database. + */ +export async function testSaveAndFindEntityAsync<T>(repository: Repository<T>, entity: T): Promise<void> { + // Note(albrow): We are forced to use an 'as any' hack here because + // TypeScript complains about stack depth when checking the types. + await repository.save(entity as any); + const gotEntity = await repository.findOneOrFail({ + where: entity, + }); + expect(gotEntity).deep.equal(entity); +} diff --git a/packages/pipeline/test/parsers/bloxy/index_test.ts b/packages/pipeline/test/parsers/bloxy/index_test.ts new file mode 100644 index 000000000..2b8d68f98 --- /dev/null +++ b/packages/pipeline/test/parsers/bloxy/index_test.ts @@ -0,0 +1,99 @@ +// tslint:disable:custom-no-magic-numbers +import { BigNumber } from '@0x/utils'; +import * as chai from 'chai'; +import 'mocha'; +import * as R from 'ramda'; + +import { BLOXY_DEX_TRADES_URL, BloxyTrade } from '../../../src/data_sources/bloxy'; +import { DexTrade } from '../../../src/entities'; +import { _parseBloxyTrade } from '../../../src/parsers/bloxy'; +import { _convertToExchangeFillEvent } from '../../../src/parsers/events'; +import { chaiSetup } from '../../utils/chai_setup'; + +chaiSetup.configure(); +const expect = chai.expect; + +const baseInput: BloxyTrade = { + tx_hash: '0xb93a7faf92efbbb5405c9a73cd4efd99702fe27c03ff22baee1f1b1e37b3a0bf', + tx_time: '2018-11-21T09:06:28.000+00:00', + tx_date: '2018-11-21', + tx_sender: '0x00923b9a074762b93650716333b3e1473a15048e', + smart_contract_id: 7091917, + smart_contract_address: '0x818e6fecd516ecc3849daf6845e3ec868087b755', + contract_type: 'DEX/Kyber Network Proxy', + maker: '0x0000000000000000000000000000000000000001', + taker: '0x0000000000000000000000000000000000000002', + amountBuy: 1.011943163078103, + makerFee: 38.912083, + buyCurrencyId: 1, + buySymbol: 'ETH', + amountSell: 941.4997928436911, + takerFee: 100.39, + sellCurrencyId: 16610, + sellSymbol: 'ELF', + maker_annotation: 'random annotation', + taker_annotation: 'random other annotation', + protocol: 'Kyber Network Proxy', + buyAddress: '0xbf2179859fc6d5bee9bf9158632dc51678a4100d', + sellAddress: '0xbf2179859fc6d5bee9bf9158632dc51678a4100e', +}; + +const baseExpected: DexTrade = { + sourceUrl: BLOXY_DEX_TRADES_URL, + txHash: '0xb93a7faf92efbbb5405c9a73cd4efd99702fe27c03ff22baee1f1b1e37b3a0bf', + txTimestamp: 1542791188000, + txDate: '2018-11-21', + txSender: '0x00923b9a074762b93650716333b3e1473a15048e', + smartContractId: 7091917, + smartContractAddress: '0x818e6fecd516ecc3849daf6845e3ec868087b755', + contractType: 'DEX/Kyber Network Proxy', + maker: '0x0000000000000000000000000000000000000001', + taker: '0x0000000000000000000000000000000000000002', + amountBuy: new BigNumber('1.011943163078103'), + makerFeeAmount: new BigNumber('38.912083'), + buyCurrencyId: 1, + buySymbol: 'ETH', + amountSell: new BigNumber('941.4997928436911'), + takerFeeAmount: new BigNumber('100.39'), + sellCurrencyId: 16610, + sellSymbol: 'ELF', + makerAnnotation: 'random annotation', + takerAnnotation: 'random other annotation', + protocol: 'Kyber Network Proxy', + buyAddress: '0xbf2179859fc6d5bee9bf9158632dc51678a4100d', + sellAddress: '0xbf2179859fc6d5bee9bf9158632dc51678a4100e', +}; + +interface TestCase { + input: BloxyTrade; + expected: DexTrade; +} + +const testCases: TestCase[] = [ + { + input: baseInput, + expected: baseExpected, + }, + { + input: R.merge(baseInput, { buyAddress: null, sellAddress: null }), + expected: R.merge(baseExpected, { buyAddress: null, sellAddress: null }), + }, + { + input: R.merge(baseInput, { + buySymbol: + 'RING\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000', + }), + expected: R.merge(baseExpected, { buySymbol: 'RING' }), + }, +]; + +describe('bloxy', () => { + describe('_parseBloxyTrade', () => { + for (const [i, testCase] of testCases.entries()) { + it(`converts BloxyTrade to DexTrade entity (${i + 1}/${testCases.length})`, () => { + const actual = _parseBloxyTrade(testCase.input); + expect(actual).deep.equal(testCase.expected); + }); + } + }); +}); diff --git a/packages/pipeline/test/parsers/ddex_orders/index_test.ts b/packages/pipeline/test/parsers/ddex_orders/index_test.ts new file mode 100644 index 000000000..213100f44 --- /dev/null +++ b/packages/pipeline/test/parsers/ddex_orders/index_test.ts @@ -0,0 +1,66 @@ +import { BigNumber } from '@0x/utils'; +import * as chai from 'chai'; +import 'mocha'; + +import { DdexMarket } from '../../../src/data_sources/ddex'; +import { TokenOrderbookSnapshot as TokenOrder } from '../../../src/entities'; +import { aggregateOrders, parseDdexOrder } from '../../../src/parsers/ddex_orders'; +import { OrderType } from '../../../src/types'; +import { chaiSetup } from '../../utils/chai_setup'; + +chaiSetup.configure(); +const expect = chai.expect; + +// tslint:disable:custom-no-magic-numbers +describe('ddex_orders', () => { + describe('aggregateOrders', () => { + it('aggregates orders by price point', () => { + const input = [ + { price: '1', amount: '20', orderId: 'testtest' }, + { price: '1', amount: '30', orderId: 'testone' }, + { price: '2', amount: '100', orderId: 'testtwo' }, + ]; + const expected = [['1', new BigNumber(50)], ['2', new BigNumber(100)]]; + const actual = aggregateOrders(input); + expect(actual).deep.equal(expected); + }); + }); + + describe('parseDdexOrder', () => { + it('converts ddexOrder to TokenOrder entity', () => { + const ddexOrder: [string, BigNumber] = ['0.5', new BigNumber(10)]; + const ddexMarket: DdexMarket = { + id: 'ABC-DEF', + quoteToken: 'ABC', + quoteTokenDecimals: 5, + quoteTokenAddress: '0x0000000000000000000000000000000000000000', + baseToken: 'DEF', + baseTokenDecimals: 2, + baseTokenAddress: '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81', + minOrderSize: '0.1', + maxOrderSize: '1000', + pricePrecision: 1, + priceDecimals: 1, + amountDecimals: 0, + }; + const observedTimestamp: number = Date.now(); + const orderType: OrderType = 'bid'; + const source: string = 'ddex'; + + const expected = new TokenOrder(); + expected.source = 'ddex'; + expected.observedTimestamp = observedTimestamp; + expected.orderType = 'bid'; + expected.price = new BigNumber(0.5); + expected.baseAssetSymbol = 'DEF'; + expected.baseAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81'; + expected.baseVolume = new BigNumber(5); + expected.quoteAssetSymbol = 'ABC'; + expected.quoteAssetAddress = '0x0000000000000000000000000000000000000000'; + expected.quoteVolume = new BigNumber(10); + + const actual = parseDdexOrder(ddexMarket, observedTimestamp, orderType, source, ddexOrder); + expect(actual).deep.equal(expected); + }); + }); +}); diff --git a/packages/pipeline/test/parsers/events/index_test.ts b/packages/pipeline/test/parsers/events/index_test.ts new file mode 100644 index 000000000..7e439ce39 --- /dev/null +++ b/packages/pipeline/test/parsers/events/index_test.ts @@ -0,0 +1,78 @@ +import { ExchangeFillEventArgs } from '@0x/contract-wrappers'; +import { BigNumber } from '@0x/utils'; +import * as chai from 'chai'; +import { LogWithDecodedArgs } from 'ethereum-types'; +import 'mocha'; + +import { ExchangeFillEvent } from '../../../src/entities'; +import { _convertToExchangeFillEvent } from '../../../src/parsers/events'; +import { chaiSetup } from '../../utils/chai_setup'; + +chaiSetup.configure(); +const expect = chai.expect; + +// tslint:disable:custom-no-magic-numbers +describe('exchange_events', () => { + describe('_convertToExchangeFillEvent', () => { + it('converts LogWithDecodedArgs to ExchangeFillEvent entity', () => { + const input: LogWithDecodedArgs<ExchangeFillEventArgs> = { + logIndex: 102, + transactionIndex: 38, + transactionHash: '0x6dd106d002873746072fc5e496dd0fb2541b68c77bcf9184ae19a42fd33657fe', + blockHash: '', + blockNumber: 6276262, + address: '0x4f833a24e1f95d70f028921e27040ca56e09ab0b', + data: + '0x000000000000000000000000f6da68519f78b0d0bc93c701e86affcb75c92428000000000000000000000000f6da68519f78b0d0bc93c701e86affcb75c92428000000000000000000000000000000000000000000000000002386f26fc10000000000000000000000000000000000000000000000000000016345785d8a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000001600000000000000000000000000000000000000000000000000000000000000024f47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000024f47261b0000000000000000000000000e41d2489571d322189246dafa5ebde1f4699f49800000000000000000000000000000000000000000000000000000000', + topics: [ + '0x0bcc4c97732e47d9946f229edb95f5b6323f601300e4690de719993f3c371129', + '0x000000000000000000000000f6da68519f78b0d0bc93c701e86affcb75c92428', + '0x000000000000000000000000c370d2a5920344aa6b7d8d11250e3e861434cbdd', + '0xab12ed2cbaa5615ab690b9da75a46e53ddfcf3f1a68655b5fe0d94c75a1aac4a', + ], + event: 'Fill', + args: { + makerAddress: '0xf6da68519f78b0d0bc93c701e86affcb75c92428', + feeRecipientAddress: '0xc370d2a5920344aa6b7d8d11250e3e861434cbdd', + takerAddress: '0xf6da68519f78b0d0bc93c701e86affcb75c92428', + senderAddress: '0xf6da68519f78b0d0bc93c701e86affcb75c92428', + makerAssetFilledAmount: new BigNumber('10000000000000000'), + takerAssetFilledAmount: new BigNumber('100000000000000000'), + makerFeePaid: new BigNumber('0'), + takerFeePaid: new BigNumber('12345'), + orderHash: '0xab12ed2cbaa5615ab690b9da75a46e53ddfcf3f1a68655b5fe0d94c75a1aac4a', + makerAssetData: '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2', + takerAssetData: '0xf47261b0000000000000000000000000e41d2489571d322189246dafa5ebde1f4699f498', + }, + }; + const expected = new ExchangeFillEvent(); + expected.contractAddress = '0x4f833a24e1f95d70f028921e27040ca56e09ab0b'; + expected.blockNumber = 6276262; + expected.logIndex = 102; + expected.rawData = + '0x000000000000000000000000f6da68519f78b0d0bc93c701e86affcb75c92428000000000000000000000000f6da68519f78b0d0bc93c701e86affcb75c92428000000000000000000000000000000000000000000000000002386f26fc10000000000000000000000000000000000000000000000000000016345785d8a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000001600000000000000000000000000000000000000000000000000000000000000024f47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000024f47261b0000000000000000000000000e41d2489571d322189246dafa5ebde1f4699f49800000000000000000000000000000000000000000000000000000000'; + expected.transactionHash = '0x6dd106d002873746072fc5e496dd0fb2541b68c77bcf9184ae19a42fd33657fe'; + expected.makerAddress = '0xf6da68519f78b0d0bc93c701e86affcb75c92428'; + expected.takerAddress = '0xf6da68519f78b0d0bc93c701e86affcb75c92428'; + expected.feeRecipientAddress = '0xc370d2a5920344aa6b7d8d11250e3e861434cbdd'; + expected.senderAddress = '0xf6da68519f78b0d0bc93c701e86affcb75c92428'; + expected.makerAssetFilledAmount = new BigNumber('10000000000000000'); + expected.takerAssetFilledAmount = new BigNumber('100000000000000000'); + expected.makerFeePaid = new BigNumber('0'); + expected.takerFeePaid = new BigNumber('12345'); + expected.orderHash = '0xab12ed2cbaa5615ab690b9da75a46e53ddfcf3f1a68655b5fe0d94c75a1aac4a'; + expected.rawMakerAssetData = '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2'; + expected.makerAssetType = 'erc20'; + expected.makerAssetProxyId = '0xf47261b0'; + expected.makerTokenAddress = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2'; + expected.makerTokenId = null; + expected.rawTakerAssetData = '0xf47261b0000000000000000000000000e41d2489571d322189246dafa5ebde1f4699f498'; + expected.takerAssetType = 'erc20'; + expected.takerAssetProxyId = '0xf47261b0'; + expected.takerTokenAddress = '0xe41d2489571d322189246dafa5ebde1f4699f498'; + expected.takerTokenId = null; + const actual = _convertToExchangeFillEvent(input); + expect(actual).deep.equal(expected); + }); + }); +}); diff --git a/packages/pipeline/test/parsers/ohlcv_external/crypto_compare_test.ts b/packages/pipeline/test/parsers/ohlcv_external/crypto_compare_test.ts new file mode 100644 index 000000000..118cafc5e --- /dev/null +++ b/packages/pipeline/test/parsers/ohlcv_external/crypto_compare_test.ts @@ -0,0 +1,62 @@ +import * as chai from 'chai'; +import 'mocha'; +import * as R from 'ramda'; + +import { CryptoCompareOHLCVRecord } from '../../../src/data_sources/ohlcv_external/crypto_compare'; +import { OHLCVExternal } from '../../../src/entities'; +import { OHLCVMetadata, parseRecords } from '../../../src/parsers/ohlcv_external/crypto_compare'; +import { chaiSetup } from '../../utils/chai_setup'; + +chaiSetup.configure(); +const expect = chai.expect; + +// tslint:disable:custom-no-magic-numbers +describe('ohlcv_external parser (Crypto Compare)', () => { + describe('parseRecords', () => { + const record: CryptoCompareOHLCVRecord = { + time: 200, + close: 100, + high: 101, + low: 99, + open: 98, + volumefrom: 1234, + volumeto: 4321, + }; + + const metadata: OHLCVMetadata = { + fromSymbol: 'ETH', + toSymbol: 'ZRX', + exchange: 'CCCAGG', + source: 'CryptoCompare', + observedTimestamp: new Date().getTime(), + interval: 100000, + }; + + const entity = new OHLCVExternal(); + entity.exchange = metadata.exchange; + entity.fromSymbol = metadata.fromSymbol; + entity.toSymbol = metadata.toSymbol; + entity.startTime = 100000; + entity.endTime = 200000; + entity.open = record.open; + entity.close = record.close; + entity.low = record.low; + entity.high = record.high; + entity.volumeFrom = record.volumefrom; + entity.volumeTo = record.volumeto; + entity.source = metadata.source; + entity.observedTimestamp = metadata.observedTimestamp; + + it('converts Crypto Compare OHLCV records to OHLCVExternal entity', () => { + const input = [record, R.merge(record, { time: 300 }), R.merge(record, { time: 400 })]; + const expected = [ + entity, + R.merge(entity, { startTime: 200000, endTime: 300000 }), + R.merge(entity, { startTime: 300000, endTime: 400000 }), + ]; + + const actual = parseRecords(input, metadata); + expect(actual).deep.equal(expected); + }); + }); +}); diff --git a/packages/pipeline/test/parsers/paradex_orders/index_test.ts b/packages/pipeline/test/parsers/paradex_orders/index_test.ts new file mode 100644 index 000000000..1522806bf --- /dev/null +++ b/packages/pipeline/test/parsers/paradex_orders/index_test.ts @@ -0,0 +1,54 @@ +import { BigNumber } from '@0x/utils'; +import * as chai from 'chai'; +import 'mocha'; + +import { ParadexMarket, ParadexOrder } from '../../../src/data_sources/paradex'; +import { TokenOrderbookSnapshot as TokenOrder } from '../../../src/entities'; +import { parseParadexOrder } from '../../../src/parsers/paradex_orders'; +import { OrderType } from '../../../src/types'; +import { chaiSetup } from '../../utils/chai_setup'; + +chaiSetup.configure(); +const expect = chai.expect; + +// tslint:disable:custom-no-magic-numbers +describe('paradex_orders', () => { + describe('parseParadexOrder', () => { + it('converts ParadexOrder to TokenOrder entity', () => { + const paradexOrder: ParadexOrder = { + amount: '412', + price: '0.1245', + }; + const paradexMarket: ParadexMarket = { + id: '2', + symbol: 'ABC/DEF', + baseToken: 'DEF', + quoteToken: 'ABC', + minOrderSize: '0.1', + maxOrderSize: '1000', + priceMaxDecimals: 5, + amountMaxDecimals: 5, + baseTokenAddress: '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81', + quoteTokenAddress: '0x0000000000000000000000000000000000000000', + }; + const observedTimestamp: number = Date.now(); + const orderType: OrderType = 'bid'; + const source: string = 'paradex'; + + const expected = new TokenOrder(); + expected.source = 'paradex'; + expected.observedTimestamp = observedTimestamp; + expected.orderType = 'bid'; + expected.price = new BigNumber(0.1245); + expected.baseAssetSymbol = 'DEF'; + expected.baseAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81'; + expected.baseVolume = new BigNumber(412 * 0.1245); + expected.quoteAssetSymbol = 'ABC'; + expected.quoteAssetAddress = '0x0000000000000000000000000000000000000000'; + expected.quoteVolume = new BigNumber(412); + + const actual = parseParadexOrder(paradexMarket, observedTimestamp, orderType, source, paradexOrder); + expect(actual).deep.equal(expected); + }); + }); +}); diff --git a/packages/pipeline/test/parsers/sra_orders/index_test.ts b/packages/pipeline/test/parsers/sra_orders/index_test.ts new file mode 100644 index 000000000..ee2842ef3 --- /dev/null +++ b/packages/pipeline/test/parsers/sra_orders/index_test.ts @@ -0,0 +1,68 @@ +import { APIOrder } from '@0x/types'; +import { BigNumber } from '@0x/utils'; +import * as chai from 'chai'; +import 'mocha'; + +import { SraOrder } from '../../../src/entities'; +import { _convertToEntity } from '../../../src/parsers/sra_orders'; +import { chaiSetup } from '../../utils/chai_setup'; + +chaiSetup.configure(); +const expect = chai.expect; + +// tslint:disable:custom-no-magic-numbers +describe('sra_orders', () => { + describe('_convertToEntity', () => { + it('converts ApiOrder to SraOrder entity', () => { + const input: APIOrder = { + order: { + makerAddress: '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81', + takerAddress: '0x0000000000000000000000000000000000000000', + feeRecipientAddress: '0xa258b39954cef5cb142fd567a46cddb31a670124', + senderAddress: '0x0000000000000000000000000000000000000000', + makerAssetAmount: new BigNumber('1619310371000000000'), + takerAssetAmount: new BigNumber('8178335207070707070707'), + makerFee: new BigNumber('0'), + takerFee: new BigNumber('0'), + exchangeAddress: '0x4f833a24e1f95d70f028921e27040ca56e09ab0b', + expirationTimeSeconds: new BigNumber('1538529488'), + signature: + '0x1b5a5d672b0d647b5797387ccbb89d822d5d2e873346b014f4ff816ff0783f2a7a0d2824d2d7042ec8ea375bc7f870963e1cb8248f1db03ddf125e27b5963aa11f03', + salt: new BigNumber('1537924688891'), + makerAssetData: '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2', + takerAssetData: '0xf47261b000000000000000000000000042d6622dece394b54999fbd73d108123806f6a18', + }, + metaData: { isThisArbitraryData: true, powerLevel: 9001 }, + }; + const expected = new SraOrder(); + expected.exchangeAddress = '0x4f833a24e1f95d70f028921e27040ca56e09ab0b'; + expected.orderHashHex = '0x1bdbeb0d088a33da28b9ee6d94e8771452f90f4a69107da2fa75195d61b9a1c9'; + expected.makerAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81'; + expected.takerAddress = '0x0000000000000000000000000000000000000000'; + expected.feeRecipientAddress = '0xa258b39954cef5cb142fd567a46cddb31a670124'; + expected.senderAddress = '0x0000000000000000000000000000000000000000'; + expected.makerAssetAmount = new BigNumber('1619310371000000000'); + expected.takerAssetAmount = new BigNumber('8178335207070707070707'); + expected.makerFee = new BigNumber('0'); + expected.takerFee = new BigNumber('0'); + expected.expirationTimeSeconds = new BigNumber('1538529488'); + expected.salt = new BigNumber('1537924688891'); + expected.signature = + '0x1b5a5d672b0d647b5797387ccbb89d822d5d2e873346b014f4ff816ff0783f2a7a0d2824d2d7042ec8ea375bc7f870963e1cb8248f1db03ddf125e27b5963aa11f03'; + expected.rawMakerAssetData = '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2'; + expected.makerAssetType = 'erc20'; + expected.makerAssetProxyId = '0xf47261b0'; + expected.makerTokenAddress = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2'; + expected.makerTokenId = null; + expected.rawTakerAssetData = '0xf47261b000000000000000000000000042d6622dece394b54999fbd73d108123806f6a18'; + expected.takerAssetType = 'erc20'; + expected.takerAssetProxyId = '0xf47261b0'; + expected.takerTokenAddress = '0x42d6622dece394b54999fbd73d108123806f6a18'; + expected.takerTokenId = null; + expected.metadataJson = '{"isThisArbitraryData":true,"powerLevel":9001}'; + + const actual = _convertToEntity(input); + expect(actual).deep.equal(expected); + }); + }); +}); diff --git a/packages/pipeline/test/utils/chai_setup.ts b/packages/pipeline/test/utils/chai_setup.ts new file mode 100644 index 000000000..1a8733093 --- /dev/null +++ b/packages/pipeline/test/utils/chai_setup.ts @@ -0,0 +1,13 @@ +import * as chai from 'chai'; +import chaiAsPromised = require('chai-as-promised'); +import ChaiBigNumber = require('chai-bignumber'); +import * as dirtyChai from 'dirty-chai'; + +export const chaiSetup = { + configure(): void { + chai.config.includeStack = true; + chai.use(ChaiBigNumber()); + chai.use(dirtyChai); + chai.use(chaiAsPromised); + }, +}; diff --git a/packages/pipeline/tsconfig.json b/packages/pipeline/tsconfig.json new file mode 100644 index 000000000..6f138f260 --- /dev/null +++ b/packages/pipeline/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig", + "compilerOptions": { + "outDir": "lib", + "rootDir": ".", + "emitDecoratorMetadata": true, + "experimentalDecorators": true + }, + "include": ["./src/**/*", "./test/**/*", "./migrations/**/*"] +} diff --git a/packages/pipeline/tslint.json b/packages/pipeline/tslint.json new file mode 100644 index 000000000..dd9053357 --- /dev/null +++ b/packages/pipeline/tslint.json @@ -0,0 +1,3 @@ +{ + "extends": ["@0x/tslint-config"] +} diff --git a/packages/pipeline/typedoc-tsconfig.json b/packages/pipeline/typedoc-tsconfig.json new file mode 100644 index 000000000..8b0ff51c1 --- /dev/null +++ b/packages/pipeline/typedoc-tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../typedoc-tsconfig", + "compilerOptions": { + "outDir": "lib", + "rootDir": ".", + "emitDecoratorMetadata": true, + "experimentalDecorators": true + }, + "include": ["./src/**/*", "./test/**/*", "./migrations/**/*"] +} diff --git a/packages/sol-compiler/CHANGELOG.json b/packages/sol-compiler/CHANGELOG.json index e9274f64e..fe077b6cc 100644 --- a/packages/sol-compiler/CHANGELOG.json +++ b/packages/sol-compiler/CHANGELOG.json @@ -1,5 +1,14 @@ [ { + "version": "1.1.15", + "changes": [ + { + "note": "Fix bug where we were appending base path to absolute imports (e.g NPM imports)", + "pr": 1311 + } + ] + }, + { "timestamp": 1543401373, "version": "1.1.14", "changes": [ diff --git a/packages/sol-compiler/src/compiler.ts b/packages/sol-compiler/src/compiler.ts index 8ee7fa4a9..cba67f292 100644 --- a/packages/sol-compiler/src/compiler.ts +++ b/packages/sol-compiler/src/compiler.ts @@ -394,7 +394,14 @@ export class Compiler { // const lastPathSeparatorPos = contractPath.lastIndexOf('/'); const contractFolder = lastPathSeparatorPos === -1 ? '' : contractPath.slice(0, lastPathSeparatorPos + 1); - importPath = path.resolve('/' + contractFolder, importPath).replace('/', ''); + if (importPath.startsWith('.')) { + /** + * Some imports path are relative ("../Token.sol", "./Wallet.sol") + * while others are absolute ("Token.sol", "@0x/contracts/Wallet.sol") + * And we need to append the base path for relative imports. + */ + importPath = path.resolve('/' + contractFolder, importPath).replace('/', ''); + } if (_.isUndefined(sourcesToAppendTo[importPath])) { sourcesToAppendTo[importPath] = { id: fullSources[importPath].id }; diff --git a/packages/sol-resolver/CHANGELOG.json b/packages/sol-resolver/CHANGELOG.json index fdfb4009b..4c9e612d7 100644 --- a/packages/sol-resolver/CHANGELOG.json +++ b/packages/sol-resolver/CHANGELOG.json @@ -1,5 +1,14 @@ [ { + "version": "1.1.0", + "changes": [ + { + "note": "NPMResolver now supports scoped packages", + "pr": 1311 + } + ] + }, + { "timestamp": 1542821676, "version": "1.0.17", "changes": [ diff --git a/packages/sol-resolver/src/resolvers/npm_resolver.ts b/packages/sol-resolver/src/resolvers/npm_resolver.ts index a2df0dcad..eeb2b5493 100644 --- a/packages/sol-resolver/src/resolvers/npm_resolver.ts +++ b/packages/sol-resolver/src/resolvers/npm_resolver.ts @@ -1,4 +1,5 @@ import * as fs from 'fs'; +import * as _ from 'lodash'; import * as path from 'path'; import { ContractSource } from '../types'; @@ -13,12 +14,22 @@ export class NPMResolver extends Resolver { } public resolveIfExists(importPath: string): ContractSource | undefined { if (!importPath.startsWith('/')) { - const [packageName, ...other] = importPath.split('/'); + let packageName; + let packageScopeIfExists; + let other; + if (_.startsWith(importPath, '@')) { + [packageScopeIfExists, packageName, ...other] = importPath.split('/'); + } else { + [packageName, ...other] = importPath.split('/'); + } const pathWithinPackage = path.join(...other); let currentPath = this._packagePath; const ROOT_PATH = '/'; while (currentPath !== ROOT_PATH) { - const lookupPath = path.join(currentPath, 'node_modules', packageName, pathWithinPackage); + const packagePath = _.isUndefined(packageScopeIfExists) + ? packageName + : path.join(packageScopeIfExists, packageName); + const lookupPath = path.join(currentPath, 'node_modules', packagePath, pathWithinPackage); if (fs.existsSync(lookupPath) && fs.lstatSync(lookupPath).isFile()) { const fileContent = fs.readFileSync(lookupPath).toString(); return { diff --git a/packages/website/translations/chinese.json b/packages/website/translations/chinese.json index eb88b43d0..2b1f2a3f5 100644 --- a/packages/website/translations/chinese.json +++ b/packages/website/translations/chinese.json @@ -76,9 +76,9 @@ "STANDARD_RELAYER_API": "中继方标准API", "PORTAL_DAPP": "去中心化应用门户", "WEBSITE": "网站", - "DEVELOPERS": "首页", - "HOME": "Rocket.chat", - "ROCKETCHAT": "开发人员", + "DEVELOPERS": "开发人员", + "HOME": "首页", + "DISCORD": "discord chat", "BUILD_A_RELAYER": "build a relayer", "BUILD_A_RELAYER_DESCRIPTION": "Learn how to build your own 0x relayer from scratch", "DEVELOP_ON_ETHEREUM": "develop on Ethereum", diff --git a/packages/website/translations/english.json b/packages/website/translations/english.json index 5fba7a0ff..b2799e264 100644 --- a/packages/website/translations/english.json +++ b/packages/website/translations/english.json @@ -81,7 +81,7 @@ "WEBSITE": "website", "DEVELOPERS": "developers", "HOME": "home", - "ROCKETCHAT": "rocket.chat", + "DISCORD": "discord chat", "TRADE_CALL_TO_ACTION": "trade on 0x", "BUILD_A_RELAYER": "build a relayer", "BUILD_A_RELAYER_DESCRIPTION": "Learn how to build your own 0x relayer from scratch", diff --git a/packages/website/translations/korean.json b/packages/website/translations/korean.json index e3ce74676..61c431a5a 100644 --- a/packages/website/translations/korean.json +++ b/packages/website/translations/korean.json @@ -77,7 +77,7 @@ "PORTAL_DAPP": "포털 dApp", "WEBSITE": "Website", "HOME": "홈", - "ROCKETCHAT": "Rocket.chat", + "DISCORD": "discord chat", "DEVELOPERS": "개발자", "BUILD_A_RELAYER": "build a relayer", "BUILD_A_RELAYER_DESCRIPTION": "Learn how to build your own 0x relayer from scratch", diff --git a/packages/website/translations/russian.json b/packages/website/translations/russian.json index c74fb5e32..3180d4a8e 100644 --- a/packages/website/translations/russian.json +++ b/packages/website/translations/russian.json @@ -76,9 +76,9 @@ "STANDARD_RELAYER_API": "standard relayer API", "PORTAL_DAPP": "DApp-портал", "WEBSITE": "Веб-сайт", - "DEVELOPERS": "Домашняя страница", - "HOME": "Rocket.chat", - "ROCKETCHAT": "Для разработчиков", + "DEVELOPERS": "Для разработчиков", + "HOME": "Домашняя страница", + "DISCORD": "discord chat", "BUILD_A_RELAYER": "build a relayer", "BUILD_A_RELAYER_DESCRIPTION": "Learn how to build your own 0x relayer from scratch", "DEVELOP_ON_ETHEREUM": "develop on Ethereum", diff --git a/packages/website/translations/spanish.json b/packages/website/translations/spanish.json index e29db711b..c23bd609b 100644 --- a/packages/website/translations/spanish.json +++ b/packages/website/translations/spanish.json @@ -77,9 +77,9 @@ "STANDARD_RELAYER_API": "API de transmisión estándar", "PORTAL_DAPP": "portal dApp", "WEBSITE": "website", - "DEVELOPERS": "inicio", - "HOME": "rocket.chat", - "ROCKETCHAT": "desarrolladores", + "DEVELOPERS": "desarrolladores", + "HOME": "inicio", + "DISCORD": "discord chat", "BUILD_A_RELAYER": "build a relayer", "BUILD_A_RELAYER_DESCRIPTION": "Learn how to build your own 0x relayer from scratch", "DEVELOP_ON_ETHEREUM": "develop on Ethereum", diff --git a/packages/website/ts/components/footer.tsx b/packages/website/ts/components/footer.tsx index e10005a0a..57071ee29 100644 --- a/packages/website/ts/components/footer.tsx +++ b/packages/website/ts/components/footer.tsx @@ -71,7 +71,7 @@ export class Footer extends React.Component<FooterProps, FooterState> { ], [Key.Community]: [ { - title: this.props.translate.get(Key.RocketChat, Deco.Cap), + title: this.props.translate.get(Key.Discord, Deco.Cap), to: constants.URL_ZEROEX_CHAT, shouldOpenInNewTab: true, }, @@ -177,7 +177,7 @@ export class Footer extends React.Component<FooterProps, FooterState> { } private _renderMenuItem(link: ALink): React.ReactNode { const titleToIcon: { [title: string]: string } = { - [this.props.translate.get(Key.RocketChat, Deco.Cap)]: 'rocketchat.png', + [this.props.translate.get(Key.Discord, Deco.Cap)]: 'rocketchat.png', [this.props.translate.get(Key.Blog, Deco.Cap)]: 'medium.png', Twitter: 'twitter.png', Reddit: 'reddit.png', diff --git a/packages/website/ts/pages/faq/faq.tsx b/packages/website/ts/pages/faq/faq.tsx index 10d91bae8..c4965e61c 100644 --- a/packages/website/ts/pages/faq/faq.tsx +++ b/packages/website/ts/pages/faq/faq.tsx @@ -379,7 +379,7 @@ const sections: FAQSection[] = [ <div> Join our{' '} <a href={constants.URL_ZEROEX_CHAT} target="_blank"> - Rocket.chat + Discord </a>! As an open source project, 0x will rely on a worldwide community of passionate developers to contribute proposals, ideas and code. </div> diff --git a/packages/website/ts/types.ts b/packages/website/ts/types.ts index 9c4b8a018..d2914dbfe 100644 --- a/packages/website/ts/types.ts +++ b/packages/website/ts/types.ts @@ -463,7 +463,7 @@ export enum Key { Website = 'WEBSITE', Developers = 'DEVELOPERS', Home = 'HOME', - RocketChat = 'ROCKETCHAT', + Discord = 'DISCORD', TradeCallToAction = 'TRADE_CALL_TO_ACTION', OurMissionAndValues = 'OUR_MISSION_AND_VALUES', BuildARelayer = 'BUILD_A_RELAYER', diff --git a/packages/website/ts/utils/constants.ts b/packages/website/ts/utils/constants.ts index e9afc8763..6597d51b4 100644 --- a/packages/website/ts/utils/constants.ts +++ b/packages/website/ts/utils/constants.ts @@ -3,7 +3,7 @@ import { BigNumber } from '@0x/utils'; import { Key, WebsitePaths } from 'ts/types'; const URL_FORUM = 'https://forum.0xproject.com'; -const URL_ZEROEX_CHAT = 'https://chat.0xproject.com'; +const URL_ZEROEX_CHAT = 'https://discord.gg/d3FTX3M'; export const constants = { DECIMAL_PLACES_ETH: 18, diff --git a/tsconfig.json b/tsconfig.json index 3d2dc6da7..b8b795aab 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -40,6 +40,7 @@ { "path": "./packages/monorepo-scripts" }, { "path": "./packages/order-utils" }, { "path": "./packages/order-watcher" }, + { "path": "./packages/pipeline" }, { "path": "./packages/react-docs" }, { "path": "./packages/react-shared" }, { "path": "./packages/sol-compiler" }, @@ -472,6 +472,62 @@ npmlog "^4.1.2" write-file-atomic "^2.3.0" +"@0x/abi-gen-wrappers@^1.0.2": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@0x/abi-gen-wrappers/-/abi-gen-wrappers-1.1.0.tgz#d4e4f10699b5da6bcfadc3842f165fc59f686e09" + dependencies: + "@0x/base-contract" "^3.0.7" + +"@0x/contract-addresses@^1.1.0": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@0x/contract-addresses/-/contract-addresses-1.2.0.tgz#aa5001d73adb1ec9cc58ab4f6e0cac0acc4ad0a8" + dependencies: + lodash "^4.17.5" + +"@0x/contract-wrappers@^3.0.0": + version "3.0.1" + resolved "https://registry.yarnpkg.com/@0x/contract-wrappers/-/contract-wrappers-3.0.1.tgz#e25f5812595d994ff66e36aa2aa99eb53becfc7c" + dependencies: + "@0x/abi-gen-wrappers" "^1.0.2" + "@0x/assert" "^1.0.15" + "@0x/contract-addresses" "^1.1.0" + "@0x/contract-artifacts" "^1.1.0" + "@0x/fill-scenarios" "^1.0.9" + "@0x/json-schemas" "^2.0.1" + "@0x/order-utils" "^2.0.1" + "@0x/types" "^1.2.1" + "@0x/typescript-typings" "^3.0.4" + "@0x/utils" "^2.0.4" + "@0x/web3-wrapper" "^3.1.1" + ethereum-types "^1.1.2" + ethereumjs-blockstream "6.0.0" + ethereumjs-util "^5.1.1" + ethers "~4.0.4" + js-sha3 "^0.7.0" + lodash "^4.17.5" + uuid "^3.1.0" + +"@0x/order-utils@^2.0.0", "@0x/order-utils@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@0x/order-utils/-/order-utils-2.0.1.tgz#8c46d7aeb9e2cce54a0822824c12427cbe5f7eb4" + dependencies: + "@0x/abi-gen-wrappers" "^1.0.2" + "@0x/assert" "^1.0.15" + "@0x/base-contract" "^3.0.3" + "@0x/contract-artifacts" "^1.1.0" + "@0x/json-schemas" "^2.0.1" + "@0x/types" "^1.2.1" + "@0x/typescript-typings" "^3.0.4" + "@0x/utils" "^2.0.4" + "@0x/web3-wrapper" "^3.1.1" + "@types/node" "*" + bn.js "^4.11.8" + ethereum-types "^1.1.2" + ethereumjs-abi "0.6.5" + ethereumjs-util "^5.1.1" + ethers "~4.0.4" + lodash "^4.17.5" + "@0xproject/npm-cli-login@^0.0.11": version "0.0.11" resolved "https://registry.yarnpkg.com/@0xproject/npm-cli-login/-/npm-cli-login-0.0.11.tgz#3f1ec06112ce62aad300ff0575358f68aeecde2e" @@ -1175,6 +1231,12 @@ version "0.4.1" resolved "https://registry.yarnpkg.com/@types/accounting/-/accounting-0.4.1.tgz#865d9f5694fd7c438fba34eb4bc82eec6f34cdd5" +"@types/axios@^0.14.0": + version "0.14.0" + resolved "https://registry.yarnpkg.com/@types/axios/-/axios-0.14.0.tgz#ec2300fbe7d7dddd7eb9d3abf87999964cafce46" + dependencies: + axios "*" + "@types/bintrees@^1.0.2": version "1.0.2" resolved "https://registry.yarnpkg.com/@types/bintrees/-/bintrees-1.0.2.tgz#0dfdce4eeebdf90427bd35b0e79dc248b3d157a6" @@ -1222,6 +1284,12 @@ version "2.0.0" resolved "https://registry.npmjs.org/@types/detect-node/-/detect-node-2.0.0.tgz#696e024ddd105c72bbc6a2e3f97902a2886f2c3f" +"@types/dockerode@^2.5.9": + version "2.5.9" + resolved "https://registry.yarnpkg.com/@types/dockerode/-/dockerode-2.5.9.tgz#de702ef63a0db7d025211e2e8af7bc90f617b7d8" + dependencies: + "@types/node" "*" + "@types/enzyme-adapter-react-16@^1.0.3": version "1.0.3" resolved "https://registry.npmjs.org/@types/enzyme-adapter-react-16/-/enzyme-adapter-react-16-1.0.3.tgz#0cf7025b036694ca8d596fe38f24162e7117acf1" @@ -1447,6 +1515,10 @@ dependencies: "@types/node" "*" +"@types/p-limit@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@types/p-limit/-/p-limit-2.0.0.tgz#c076b7daa9163108a35899ea6a9d927526943ea2" + "@types/prop-types@*": version "15.5.5" resolved "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.5.5.tgz#17038dd322c2325f5da650a94d5f9974943625e3" @@ -1457,6 +1529,10 @@ version "5.1.0" resolved "https://registry.yarnpkg.com/@types/query-string/-/query-string-5.1.0.tgz#7f40cdea49ddafa0ea4f3db35fb6c24d3bfd4dcc" +"@types/ramda@^0.25.38": + version "0.25.38" + resolved "https://registry.yarnpkg.com/@types/ramda/-/ramda-0.25.38.tgz#6c945fcc91a5fa470cc3c5a4cd4a62d7f8d03576" + "@types/rc-slider@^8.6.0": version "8.6.0" resolved "https://registry.npmjs.org/@types/rc-slider/-/rc-slider-8.6.0.tgz#7f061a920b067825c8455cf481c57b0927889c72" @@ -1803,6 +1879,13 @@ version "4.2.1" resolved "https://registry.npmjs.org/@xtuc/long/-/long-4.2.1.tgz#5c85d662f76fa1d34575766c5dcd6615abcd30d8" +JSONStream@1.3.2: + version "1.3.2" + resolved "https://registry.yarnpkg.com/JSONStream/-/JSONStream-1.3.2.tgz#c102371b6ec3a7cf3b847ca00c20bb0fce4c6dea" + dependencies: + jsonparse "^1.2.0" + through ">=2.2.7 <3" + JSONStream@^1.0.4: version "1.3.3" resolved "http://registry.yarnpkg.com/JSONStream/-/JSONStream-1.3.3.tgz#27b4b8fbbfeab4e71bcf551e7f27be8d952239bf" @@ -1925,10 +2008,6 @@ aes-js@^0.2.3: version "0.2.4" resolved "https://registry.yarnpkg.com/aes-js/-/aes-js-0.2.4.tgz#94b881ab717286d015fa219e08fb66709dda5a3d" -aes-js@^3.1.1: - version "3.1.2" - resolved "https://registry.yarnpkg.com/aes-js/-/aes-js-3.1.2.tgz#db9aabde85d5caabbfc0d4f2a4446960f627146a" - agent-base@4, agent-base@^4.1.0, agent-base@~4.2.0: version "4.2.1" resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-4.2.1.tgz#d89e5999f797875674c07d87f260fc41e83e8ca9" @@ -2083,6 +2162,10 @@ anymatch@^2.0.0: micromatch "^3.1.4" normalize-path "^2.1.1" +app-root-path@^2.0.1: + version "2.1.0" + resolved "https://registry.yarnpkg.com/app-root-path/-/app-root-path-2.1.0.tgz#98bf6599327ecea199309866e8140368fd2e646a" + append-buffer@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/append-buffer/-/append-buffer-1.0.2.tgz#d8220cf466081525efea50614f3de6514dfa58f1" @@ -2284,6 +2367,10 @@ async-limiter@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/async-limiter/-/async-limiter-1.0.0.tgz#78faed8c3d074ab81f22b4e985d79e8738f720f8" +async-parallel@^1.2.3: + version "1.2.3" + resolved "https://registry.yarnpkg.com/async-parallel/-/async-parallel-1.2.3.tgz#0b90550aeffb7a365d8cee881eb0618f656a3450" + async@1.x, async@^1.4.0, async@^1.4.2, async@^1.5.2: version "1.5.2" resolved "https://registry.yarnpkg.com/async/-/async-1.5.2.tgz#ec6a61ae56480c0c3cb241c95618e20892f9672a" @@ -2392,7 +2479,7 @@ aws4@^1.8.0: version "1.8.0" resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.8.0.tgz#f0e003d9ca9e7f59c7a508945d7b2ef9a04a542f" -axios@^0.18.0: +axios@*, axios@^0.18.0: version "0.18.0" resolved "http://registry.npmjs.org/axios/-/axios-0.18.0.tgz#32d53e4851efdc0a11993b6cd000789d70c05102" dependencies: @@ -3379,7 +3466,7 @@ bs-logger@0.x: dependencies: fast-json-stable-stringify "^2.0.0" -bs58@=4.0.1, bs58@^4.0.0: +bs58@=4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/bs58/-/bs58-4.0.1.tgz#be161e76c354f6f788ae4071f63f34e8c4f0a42a" dependencies: @@ -3402,14 +3489,6 @@ bs58check@^1.0.8: bs58 "^3.1.0" create-hash "^1.1.0" -bs58check@^2.1.2: - version "2.1.2" - resolved "https://registry.yarnpkg.com/bs58check/-/bs58check-2.1.2.tgz#53b018291228d82a5aa08e7d796fdafda54aebfc" - dependencies: - bs58 "^4.0.0" - create-hash "^1.1.0" - safe-buffer "^5.1.2" - bser@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/bser/-/bser-2.0.0.tgz#9ac78d3ed5d915804fd87acb158bc797147a1719" @@ -3448,6 +3527,10 @@ buffer-to-arraybuffer@^0.0.5: version "0.0.5" resolved "https://registry.yarnpkg.com/buffer-to-arraybuffer/-/buffer-to-arraybuffer-0.0.5.tgz#6064a40fa76eb43c723aba9ef8f6e1216d10511a" +buffer-writer@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/buffer-writer/-/buffer-writer-1.0.1.tgz#22a936901e3029afcd7547eb4487ceb697a3bf08" + buffer-xor@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/buffer-xor/-/buffer-xor-1.0.3.tgz#26e61ed1422fb70dd42e6e36729ed51d855fe8d9" @@ -3475,7 +3558,7 @@ buffer@^5.0.3, buffer@^5.0.5: base64-js "^1.0.2" ieee754 "^1.1.4" -buffer@^5.2.1: +buffer@^5.1.0, buffer@^5.2.1: version "5.2.1" resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.2.1.tgz#dd57fa0f109ac59c602479044dca7b8b3d0b71d6" dependencies: @@ -3693,7 +3776,7 @@ center-align@^0.1.1: align-text "^0.1.3" lazy-cache "^1.0.3" -chai-as-promised@^7.1.0: +chai-as-promised@^7.1.0, chai-as-promised@^7.1.1: version "7.1.1" resolved "https://registry.yarnpkg.com/chai-as-promised/-/chai-as-promised-7.1.1.tgz#08645d825deb8696ee61725dbf590c012eb00ca0" dependencies: @@ -3928,6 +4011,16 @@ cli-cursor@^2.1.0: dependencies: restore-cursor "^2.0.0" +cli-highlight@^1.2.3: + version "1.2.3" + resolved "https://registry.yarnpkg.com/cli-highlight/-/cli-highlight-1.2.3.tgz#b200f97ed0e43d24633e89de0f489a48bb87d2bf" + dependencies: + chalk "^2.3.0" + highlight.js "^9.6.0" + mz "^2.4.0" + parse5 "^3.0.3" + yargs "^10.0.3" + cli-spinners@^0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/cli-spinners/-/cli-spinners-0.1.2.tgz#bb764d88e185fb9e1e6a2a1f19772318f605e31c" @@ -4202,7 +4295,7 @@ concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" -concat-stream@^1.4.6, concat-stream@^1.5.1, concat-stream@^1.6.0: +concat-stream@^1.4.6, concat-stream@^1.5.1, concat-stream@^1.6.0, concat-stream@~1.6.2: version "1.6.2" resolved "https://registry.yarnpkg.com/concat-stream/-/concat-stream-1.6.2.tgz#904bdf194cd3122fc675c77fc4ac3d4ff0fd1a34" dependencies: @@ -4875,6 +4968,12 @@ debug@3.1.0, debug@=3.1.0, debug@^3.1.0: dependencies: ms "2.0.0" +debug@^3.2.5: + version "3.2.6" + resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.6.tgz#e83d17de16d8a7efb7717edbe5fb10135eee629b" + dependencies: + ms "^2.1.1" + debug@^4.0.1: version "4.1.0" resolved "https://registry.yarnpkg.com/debug/-/debug-4.1.0.tgz#373687bffa678b38b1cd91f861b63850035ddc87" @@ -5239,6 +5338,23 @@ dns-txt@^2.0.2: dependencies: buffer-indexof "^1.0.0" +docker-modem@1.0.x: + version "1.0.7" + resolved "https://registry.yarnpkg.com/docker-modem/-/docker-modem-1.0.7.tgz#69702a95c060eeb6775f79ccdcc734e5946972a4" + dependencies: + JSONStream "1.3.2" + debug "^3.2.5" + readable-stream "~1.0.26-4" + split-ca "^1.0.0" + +dockerode@^2.5.7: + version "2.5.7" + resolved "https://registry.yarnpkg.com/dockerode/-/dockerode-2.5.7.tgz#13dc9ec0f7f353ac0e512249e77f32d1aaa1199e" + dependencies: + concat-stream "~1.6.2" + docker-modem "1.0.x" + tar-fs "~1.16.3" + doctrine@0.7.2: version "0.7.2" resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-0.7.2.tgz#7cb860359ba3be90e040b26b729ce4bfa654c523" @@ -5343,6 +5459,10 @@ dotenv@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-4.0.0.tgz#864ef1379aced55ce6f95debecdce179f7a0cd1d" +dotenv@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-5.0.1.tgz#a5317459bd3d79ab88cff6e44057a6a3fbb1fcef" + drbg.js@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/drbg.js/-/drbg.js-1.0.1.tgz#3e36b6c42b37043823cdbc332d58f31e2445480b" @@ -6043,19 +6163,6 @@ ethereumjs-wallet@0.6.0: utf8 "^2.1.1" uuid "^2.0.1" -ethereumjs-wallet@~0.6.0: - version "0.6.2" - resolved "https://registry.yarnpkg.com/ethereumjs-wallet/-/ethereumjs-wallet-0.6.2.tgz#67244b6af3e8113b53d709124b25477b64aeccda" - dependencies: - aes-js "^3.1.1" - bs58check "^2.1.2" - ethereumjs-util "^5.2.0" - hdkey "^1.0.0" - safe-buffer "^5.1.2" - scrypt.js "^0.2.0" - utf8 "^3.0.0" - uuid "^3.3.2" - ethers@~4.0.4: version "4.0.4" resolved "https://registry.yarnpkg.com/ethers/-/ethers-4.0.4.tgz#d3f85e8b27f4b59537e06526439b0fb15b44dc65" @@ -6479,6 +6586,10 @@ figgy-pudding@^3.1.0, figgy-pudding@^3.2.1, figgy-pudding@^3.4.1, figgy-pudding@ version "3.5.1" resolved "https://registry.yarnpkg.com/figgy-pudding/-/figgy-pudding-3.5.1.tgz#862470112901c727a0e495a80744bd5baa1d6790" +figlet@^1.1.1: + version "1.2.0" + resolved "https://registry.yarnpkg.com/figlet/-/figlet-1.2.0.tgz#6c46537378fab649146b5a6143dda019b430b410" + figures@^1.3.5, figures@^1.7.0: version "1.7.0" resolved "https://registry.yarnpkg.com/figures/-/figures-1.7.0.tgz#cbe1e3affcf1cd44b80cadfed28dc793a9701d2e" @@ -7616,14 +7727,6 @@ hdkey@^0.7.0, hdkey@^0.7.1: coinstring "^2.0.0" secp256k1 "^3.0.1" -hdkey@^1.0.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/hdkey/-/hdkey-1.1.0.tgz#e74e7b01d2c47f797fa65d1d839adb7a44639f29" - dependencies: - coinstring "^2.0.0" - safe-buffer "^5.1.1" - secp256k1 "^3.0.1" - he@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/he/-/he-1.1.1.tgz#93410fd21b009735151f8868c2f271f3427e23fd" @@ -7639,7 +7742,7 @@ heap@~0.2.6: version "0.2.6" resolved "https://registry.yarnpkg.com/heap/-/heap-0.2.6.tgz#087e1f10b046932fc8594dd9e6d378afc9d1e5ac" -highlight.js@^9.0.0, highlight.js@^9.11.0, highlight.js@~9.12.0: +highlight.js@^9.0.0, highlight.js@^9.11.0, highlight.js@^9.6.0, highlight.js@~9.12.0: version "9.12.0" resolved "https://registry.yarnpkg.com/highlight.js/-/highlight.js-9.12.0.tgz#e6d9dbe57cbefe60751f02af336195870c90c01e" @@ -9031,7 +9134,7 @@ js-yaml@3.x, js-yaml@^3.4.2, js-yaml@^3.6.1, js-yaml@^3.7.0: argparse "^1.0.7" esprima "^4.0.0" -js-yaml@^3.12.0, js-yaml@^3.9.0: +js-yaml@^3.11.0, js-yaml@^3.12.0, js-yaml@^3.9.0: version "3.12.0" resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.12.0.tgz#eaed656ec8344f10f527c6bfa1b6e2244de167d1" dependencies: @@ -10630,7 +10733,7 @@ mute-stream@0.0.7, mute-stream@~0.0.4: version "0.0.7" resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.7.tgz#3075ce93bc21b8fab43e1bc4da7e8115ed1e7bab" -mz@^2.6.0: +mz@^2.4.0, mz@^2.6.0: version "2.7.0" resolved "https://registry.yarnpkg.com/mz/-/mz-2.7.0.tgz#95008057a56cafadc2bc63dde7f9ff6955948e32" dependencies: @@ -10638,7 +10741,7 @@ mz@^2.6.0: object-assign "^4.0.1" thenify-all "^1.0.0" -nan@2.10.0, nan@>=2.5.1, nan@^2.0.8, nan@^2.2.1, nan@^2.3.0, nan@^2.3.3, nan@^2.6.2, nan@^2.9.2: +nan@2.10.0, nan@>=2.5.1, nan@^2.0.8, nan@^2.2.1, nan@^2.3.0, nan@^2.3.3, nan@^2.6.2, nan@^2.9.2, nan@~2.10.0: version "2.10.0" resolved "https://registry.yarnpkg.com/nan/-/nan-2.10.0.tgz#96d0cd610ebd58d4b4de9cc0c6828cda99c7548f" @@ -10820,7 +10923,7 @@ node-notifier@^5.2.1: shellwords "^0.1.1" which "^1.3.0" -node-pre-gyp@^0.10.0: +node-pre-gyp@^0.10.0, node-pre-gyp@^0.10.3: version "0.10.3" resolved "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.10.3.tgz#3070040716afdc778747b61b6887bf78880b80fc" dependencies: @@ -11441,7 +11544,7 @@ p-limit@^1.1.0: p-limit@^2.0.0: version "2.0.0" - resolved "https://registry.npmjs.org/p-limit/-/p-limit-2.0.0.tgz#e624ed54ee8c460a778b3c9f3670496ff8a57aec" + resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-2.0.0.tgz#e624ed54ee8c460a778b3c9f3670496ff8a57aec" dependencies: p-try "^2.0.0" @@ -11504,6 +11607,10 @@ package-json@^4.0.0, package-json@^4.0.1: registry-url "^3.0.3" semver "^5.1.0" +packet-reader@0.3.1: + version "0.3.1" + resolved "https://registry.yarnpkg.com/packet-reader/-/packet-reader-0.3.1.tgz#cd62e60af8d7fea8a705ec4ff990871c46871f27" + pacote@^9.1.0: version "9.1.0" resolved "https://registry.yarnpkg.com/pacote/-/pacote-9.1.0.tgz#59810859bbd72984dcb267269259375d32f391e5" @@ -11554,6 +11661,10 @@ param-case@^2.1.0: dependencies: no-case "^2.2.0" +parent-require@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/parent-require/-/parent-require-1.0.0.tgz#746a167638083a860b0eef6732cb27ed46c32977" + parse-asn1@^5.0.0: version "5.1.1" resolved "https://registry.yarnpkg.com/parse-asn1/-/parse-asn1-5.1.1.tgz#f6bf293818332bd0dab54efb16087724745e6ca8" @@ -11639,7 +11750,7 @@ parse5@4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/parse5/-/parse5-4.0.0.tgz#6d78656e3da8d78b4ec0b906f7c08ef1dfe3f608" -parse5@^3.0.1: +parse5@^3.0.1, parse5@^3.0.3: version "3.0.3" resolved "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz#042f792ffdd36851551cf4e9e066b3874ab45b5c" dependencies: @@ -11776,6 +11887,41 @@ performance-now@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" +pg-connection-string@0.1.3: + version "0.1.3" + resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-0.1.3.tgz#da1847b20940e42ee1492beaf65d49d91b245df7" + +pg-pool@~2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-2.0.3.tgz#c022032c8949f312a4f91fb6409ce04076be3257" + +pg-types@~1.12.1: + version "1.12.1" + resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-1.12.1.tgz#d64087e3903b58ffaad279e7595c52208a14c3d2" + dependencies: + postgres-array "~1.0.0" + postgres-bytea "~1.0.0" + postgres-date "~1.0.0" + postgres-interval "^1.1.0" + +pg@^7.5.0: + version "7.5.0" + resolved "https://registry.yarnpkg.com/pg/-/pg-7.5.0.tgz#c2853bef2fcb91424ba2f649fd951ce866a84760" + dependencies: + buffer-writer "1.0.1" + packet-reader "0.3.1" + pg-connection-string "0.1.3" + pg-pool "~2.0.3" + pg-types "~1.12.1" + pgpass "1.x" + semver "4.3.2" + +pgpass@1.x: + version "1.0.2" + resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.2.tgz#2a7bb41b6065b67907e91da1b07c1847c877b306" + dependencies: + split "^1.0.0" + pify@^2.0.0, pify@^2.2.0, pify@^2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c" @@ -12104,6 +12250,24 @@ postcss@^6.0.1: source-map "^0.6.1" supports-color "^5.3.0" +postgres-array@~1.0.0: + version "1.0.3" + resolved "https://registry.yarnpkg.com/postgres-array/-/postgres-array-1.0.3.tgz#c561fc3b266b21451fc6555384f4986d78ec80f5" + +postgres-bytea@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/postgres-bytea/-/postgres-bytea-1.0.0.tgz#027b533c0aa890e26d172d47cf9ccecc521acd35" + +postgres-date@~1.0.0: + version "1.0.3" + resolved "https://registry.yarnpkg.com/postgres-date/-/postgres-date-1.0.3.tgz#e2d89702efdb258ff9d9cee0fe91bd06975257a8" + +postgres-interval@^1.1.0: + version "1.1.2" + resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.1.2.tgz#bf71ff902635f21cb241a013fc421d81d1db15a9" + dependencies: + xtend "^4.0.0" + prebuild-install@^2.2.2: version "2.5.3" resolved "https://registry.yarnpkg.com/prebuild-install/-/prebuild-install-2.5.3.tgz#9f65f242782d370296353710e9bc843490c19f69" @@ -12168,6 +12332,10 @@ prettier@^1.14.2, prettier@^1.14.3: version "1.15.2" resolved "https://registry.yarnpkg.com/prettier/-/prettier-1.15.2.tgz#d31abe22afa4351efa14c7f8b94b58bb7452205e" +prettier@^1.15.3: + version "1.15.3" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-1.15.3.tgz#1feaac5bdd181237b54dbe65d874e02a1472786a" + pretty-bytes@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/pretty-bytes/-/pretty-bytes-1.0.4.tgz#0a22e8210609ad35542f8c8d5d2159aff0751c84" @@ -12523,6 +12691,10 @@ ramda@0.21.0: version "0.21.0" resolved "https://registry.yarnpkg.com/ramda/-/ramda-0.21.0.tgz#a001abedb3ff61077d4ff1d577d44de77e8d0a35" +ramda@^0.25.0: + version "0.25.0" + resolved "https://registry.yarnpkg.com/ramda/-/ramda-0.25.0.tgz#8fdf68231cffa90bc2f9460390a0cb74a29b29a9" + randexp@0.4.6: version "0.4.6" resolved "https://registry.npmjs.org/randexp/-/randexp-0.4.6.tgz#e986ad5e5e31dae13ddd6f7b3019aa7c87f60ca3" @@ -12981,7 +13153,7 @@ read@1, read@1.0.x, read@~1.0.1, read@~1.0.5: string_decoder "~1.1.1" util-deprecate "~1.0.1" -"readable-stream@>=1.0.33-1 <1.1.0-0", readable-stream@~1.0.15, readable-stream@~1.0.26, readable-stream@~1.0.31: +"readable-stream@>=1.0.33-1 <1.1.0-0", readable-stream@~1.0.15, readable-stream@~1.0.26, readable-stream@~1.0.26-4, readable-stream@~1.0.31: version "1.0.34" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.0.34.tgz#125820e34bc842d2f2aaafafe4c2916ee32c157c" dependencies: @@ -13150,6 +13322,10 @@ redux@^3.6.0: loose-envify "^1.1.0" symbol-observable "^1.0.3" +reflect-metadata@^0.1.12: + version "0.1.12" + resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.1.12.tgz#311bf0c6b63cd782f228a81abe146a2bfa9c56f2" + refractor@^2.4.1: version "2.6.2" resolved "https://registry.npmjs.org/refractor/-/refractor-2.6.2.tgz#8e0877ab8864165275aafeea5d9c8eebe871552f" @@ -13872,6 +14048,10 @@ semver-sort@0.0.4: version "5.5.0" resolved "https://registry.yarnpkg.com/semver/-/semver-5.5.0.tgz#dc4bbc7a6ca9d916dee5d43516f0092b58f7b8ab" +semver@4.3.2: + version "4.3.2" + resolved "http://registry.npmjs.org/semver/-/semver-4.3.2.tgz#c7a07158a80bedd052355b770d82d6640f803be7" + semver@^4.1.0: version "4.3.6" resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.6.tgz#300bc6e0e86374f7ba61068b5b1ecd57fc6532da" @@ -14456,6 +14636,10 @@ speedometer@~0.1.2: version "0.1.4" resolved "https://registry.yarnpkg.com/speedometer/-/speedometer-0.1.4.tgz#9876dbd2a169d3115402d48e6ea6329c8816a50d" +split-ca@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/split-ca/-/split-ca-1.0.1.tgz#6c83aff3692fa61256e0cd197e05e9de157691a6" + split-string@^3.0.1, split-string@^3.0.2: version "3.1.0" resolved "https://registry.yarnpkg.com/split-string/-/split-string-3.1.0.tgz#7cb09dda3a86585705c64b39a6466038682e8fe2" @@ -14484,6 +14668,14 @@ sprintf-js@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" +sqlite3@^4.0.2: + version "4.0.2" + resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-4.0.2.tgz#1bbeb68b03ead5d499e42a3a1b140064791c5a64" + dependencies: + nan "~2.10.0" + node-pre-gyp "^0.10.3" + request "^2.87.0" + sshpk@^1.7.0: version "1.14.1" resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.14.1.tgz#130f5975eddad963f1d56f92b9ac6c51fa9f83eb" @@ -14972,6 +15164,15 @@ tar-fs@^1.13.0: pump "^1.0.0" tar-stream "^1.1.2" +tar-fs@~1.16.3: + version "1.16.3" + resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-1.16.3.tgz#966a628841da2c4010406a82167cbd5e0c72d509" + dependencies: + chownr "^1.0.1" + mkdirp "^0.5.1" + pump "^1.0.0" + tar-stream "^1.1.2" + tar-pack@^3.4.0: version "3.4.1" resolved "https://registry.yarnpkg.com/tar-pack/-/tar-pack-3.4.1.tgz#e1dbc03a9b9d3ba07e896ad027317eb679a10a1f" @@ -15580,6 +15781,24 @@ typedoc@0.13.0: typedoc-default-themes "^0.5.0" typescript "3.1.x" +typeorm@^0.2.7: + version "0.2.7" + resolved "https://registry.yarnpkg.com/typeorm/-/typeorm-0.2.7.tgz#4bbbace80dc91b1303be13f42d44ebf01d1b2558" + dependencies: + app-root-path "^2.0.1" + buffer "^5.1.0" + chalk "^2.3.2" + cli-highlight "^1.2.3" + debug "^3.1.0" + dotenv "^5.0.1" + glob "^7.1.2" + js-yaml "^3.11.0" + mkdirp "^0.5.1" + reflect-metadata "^0.1.12" + xml2js "^0.4.17" + yargonaut "^1.1.2" + yargs "^11.1.0" + types-bn@^0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/types-bn/-/types-bn-0.0.1.tgz#4253c7c7251b14e1d77cdca6f58800e5e2b82c4b" @@ -15704,7 +15923,7 @@ underscore@1.8.3: underscore@~1.4.4: version "1.4.4" - resolved "https://registry.npmjs.org/underscore/-/underscore-1.4.4.tgz#61a6a32010622afa07963bf325203cf12239d604" + resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.4.4.tgz#61a6a32010622afa07963bf325203cf12239d604" unherit@^1.0.4: version "1.1.0" @@ -15925,10 +16144,6 @@ utf8@^2.1.1: version "2.1.2" resolved "https://registry.yarnpkg.com/utf8/-/utf8-2.1.2.tgz#1fa0d9270e9be850d9b05027f63519bf46457d96" -utf8@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/utf8/-/utf8-3.0.0.tgz#f052eed1364d696e769ef058b183df88c87f69d1" - util-deprecate@~1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" @@ -16872,7 +17087,7 @@ xml2js@0.4.17: sax ">=0.6.0" xmlbuilder "^4.1.0" -xml2js@0.4.19: +xml2js@0.4.19, xml2js@^0.4.17: version "0.4.19" resolved "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" dependencies: @@ -16931,6 +17146,14 @@ yallist@^3.0.0, yallist@^3.0.2: version "3.0.2" resolved "https://registry.npmjs.org/yallist/-/yallist-3.0.2.tgz#8452b4bb7e83c7c188d8041c1a837c773d6d8bb9" +yargonaut@^1.1.2: + version "1.1.4" + resolved "https://registry.yarnpkg.com/yargonaut/-/yargonaut-1.1.4.tgz#c64f56432c7465271221f53f5cc517890c3d6e0c" + dependencies: + chalk "^1.1.1" + figlet "^1.1.1" + parent-require "^1.0.0" + yargs-parser@10.x, yargs-parser@^10.0.0, yargs-parser@^10.1.0: version "10.1.0" resolved "https://registry.npmjs.org/yargs-parser/-/yargs-parser-10.1.0.tgz#7202265b89f7e9e9f2e5765e0fe735a905edbaa8" @@ -16968,7 +17191,7 @@ yargs-parser@^9.0.2: dependencies: camelcase "^4.1.0" -yargs@11.1.0: +yargs@11.1.0, yargs@^11.1.0: version "11.1.0" resolved "https://registry.yarnpkg.com/yargs/-/yargs-11.1.0.tgz#90b869934ed6e871115ea2ff58b03f4724ed2d77" dependencies: |