diff options
author | Fabio Berger <me@fabioberger.com> | 2019-01-11 07:47:57 +0800 |
---|---|---|
committer | Fabio Berger <me@fabioberger.com> | 2019-01-11 07:47:57 +0800 |
commit | 583e690b7a028c85809a8be2c7afe3b65264adbe (patch) | |
tree | bc7ba2fe96833633455daaebf44d7382d0e20b7d /packages/pipeline | |
parent | a516b00a0397a567fd233bd955206d46321cc178 (diff) | |
parent | a5b7a351609a5e6689bb97990216153f64302462 (diff) | |
download | dexon-0x-contracts-583e690b7a028c85809a8be2c7afe3b65264adbe.tar dexon-0x-contracts-583e690b7a028c85809a8be2c7afe3b65264adbe.tar.gz dexon-0x-contracts-583e690b7a028c85809a8be2c7afe3b65264adbe.tar.bz2 dexon-0x-contracts-583e690b7a028c85809a8be2c7afe3b65264adbe.tar.lz dexon-0x-contracts-583e690b7a028c85809a8be2c7afe3b65264adbe.tar.xz dexon-0x-contracts-583e690b7a028c85809a8be2c7afe3b65264adbe.tar.zst dexon-0x-contracts-583e690b7a028c85809a8be2c7afe3b65264adbe.zip |
Merge branch 'development' into fix/dev-tools-pages/finalTouches
* development: (49 commits)
upgrade to ddex api v3 (#1507)
Add new deployment related instructions to pipeline README (#1499)
feature: implement logging and friendly wallet name for Opera
Fix prettier
Add missing dependencies
Remove second yarn call
Add .gitkeep
Rename sol-trace-based-tools-common to sol-tracing-utils
Update packages/sol-trace/README.md
Update packages/sol-trace/CHANGELOG.json
Update packages/sol-trace-based-tools-common/src/source_maps.ts
Update packages/sol-trace-based-tools-common/CHANGELOG.json
Update packages/sol-profiler/CHANGELOG.json
Update packages/sol-coverage/CHANGELOG.json
Update packages/monorepo-scripts/src/test_installation.ts
chore: changelog update
fix: account for undefined errors in promisify, not only null
Clean up docs before publish
In README's, don't mention non-existent tests (#1497)
fix: use getBackendBaseUrl instead of hardcoded string
...
Diffstat (limited to 'packages/pipeline')
-rw-r--r-- | packages/pipeline/README.md | 17 | ||||
-rw-r--r-- | packages/pipeline/migrations/1547153875669-UpdateDDexAPIToV3.ts | 21 | ||||
-rw-r--r-- | packages/pipeline/src/data_sources/ddex/index.ts | 3 | ||||
-rw-r--r-- | packages/pipeline/src/parsers/ddex_orders/index.ts | 10 | ||||
-rw-r--r-- | packages/pipeline/test/parsers/ddex_orders/index_test.ts | 13 |
5 files changed, 48 insertions, 16 deletions
diff --git a/packages/pipeline/README.md b/packages/pipeline/README.md index 794488cac..40a64cdd1 100644 --- a/packages/pipeline/README.md +++ b/packages/pipeline/README.md @@ -147,6 +147,20 @@ set the`ZEROEX_DATA_PIPELINE_DB_URL` environment variable to a valid starting from that block number. 7. Run the migrations and then run your new script locally and verify it works as expected. +8. After all tests pass and you can run the script locally, open a new PR to + the monorepo. Don't merge this yet! +9. If you added any new scripts or dependencies between scripts, you will need + to make changes to https://github.com/0xProject/0x-pipeline-orchestration + and make a separate PR there. Don't merge this yet! +10. After your PR passes code review, ask @feuGeneA or @xianny to deploy your + changes to the QA environment. Check the [QA Airflow dashboard](http://airflow-qa.0x.org:8080) + to make sure everything works correctly in the QA environment. +11. Merge your PR to 0x-monorepo (and + https://github.com/0xProject/0x-pipeline-orchestration if needed). Then ask + @feuGeneA or @xianny to deploy to production. +12. Monitor the [production Airflow dashboard](http://airflow.0x.org:8080) to + make sure everything still works. +13. Celebrate! :tada: #### Additional guidelines and tips: @@ -164,3 +178,6 @@ set the`ZEROEX_DATA_PIPELINE_DB_URL` environment variable to a valid floating point numbers. * [TypeORM documentation](http://typeorm.io/#/) is pretty robust and can be a helpful resource. +* Scripts/parsers should perform minimum data transformation/normalization. + The idea here is to have a raw data feed that will be cleaned up and + synthesized in a separate step. diff --git a/packages/pipeline/migrations/1547153875669-UpdateDDexAPIToV3.ts b/packages/pipeline/migrations/1547153875669-UpdateDDexAPIToV3.ts new file mode 100644 index 000000000..957af4941 --- /dev/null +++ b/packages/pipeline/migrations/1547153875669-UpdateDDexAPIToV3.ts @@ -0,0 +1,21 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class UpdateDDexAPIToV31547153875669 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise<any> { + await queryRunner.query(` + UPDATE raw.token_orderbook_snapshots + SET quote_asset_symbol='WETH' + WHERE quote_asset_symbol='ETH' AND + source='ddex'; + `); + } + + public async down(queryRunner: QueryRunner): Promise<any> { + await queryRunner.query(` + UPDATE raw.token_orderbook_snapshots + SET quote_asset_symbol='ETH' + WHERE quote_asset_symbol='WETH' AND + source='ddex'; + `); + } +} diff --git a/packages/pipeline/src/data_sources/ddex/index.ts b/packages/pipeline/src/data_sources/ddex/index.ts index 2bbd8c29b..7ef92b90f 100644 --- a/packages/pipeline/src/data_sources/ddex/index.ts +++ b/packages/pipeline/src/data_sources/ddex/index.ts @@ -1,6 +1,6 @@ import { fetchAsync, logUtils } from '@0x/utils'; -const DDEX_BASE_URL = 'https://api.ddex.io/v2'; +const DDEX_BASE_URL = 'https://api.ddex.io/v3'; const ACTIVE_MARKETS_URL = `${DDEX_BASE_URL}/markets`; const NO_AGGREGATION_LEVEL = 3; // See https://docs.ddex.io/#get-orderbook const ORDERBOOK_ENDPOINT = `/orderbook?level=${NO_AGGREGATION_LEVEL}`; @@ -23,7 +23,6 @@ export interface DdexMarket { baseTokenDecimals: number; baseTokenAddress: string; minOrderSize: string; - maxOrderSize: string; pricePrecision: number; priceDecimals: number; amountDecimals: number; diff --git a/packages/pipeline/src/parsers/ddex_orders/index.ts b/packages/pipeline/src/parsers/ddex_orders/index.ts index eeb9c9d5b..562f894ab 100644 --- a/packages/pipeline/src/parsers/ddex_orders/index.ts +++ b/packages/pipeline/src/parsers/ddex_orders/index.ts @@ -58,14 +58,12 @@ export function parseDdexOrder( tokenOrder.orderType = orderType; tokenOrder.price = price; - // ddex currently confuses quote and base assets. - // We switch them here to maintain our internal consistency. - tokenOrder.baseAssetSymbol = ddexMarket.quoteToken; - tokenOrder.baseAssetAddress = ddexMarket.quoteTokenAddress; + tokenOrder.baseAssetSymbol = ddexMarket.baseToken; + tokenOrder.baseAssetAddress = ddexMarket.baseTokenAddress; tokenOrder.baseVolume = amount; - tokenOrder.quoteAssetSymbol = ddexMarket.baseToken; - tokenOrder.quoteAssetAddress = ddexMarket.baseTokenAddress; + tokenOrder.quoteAssetSymbol = ddexMarket.quoteToken; + tokenOrder.quoteAssetAddress = ddexMarket.quoteTokenAddress; tokenOrder.quoteVolume = price.times(amount); return tokenOrder; } diff --git a/packages/pipeline/test/parsers/ddex_orders/index_test.ts b/packages/pipeline/test/parsers/ddex_orders/index_test.ts index f30e86b02..d6f69e090 100644 --- a/packages/pipeline/test/parsers/ddex_orders/index_test.ts +++ b/packages/pipeline/test/parsers/ddex_orders/index_test.ts @@ -25,7 +25,6 @@ describe('ddex_orders', () => { baseTokenDecimals: 2, baseTokenAddress: '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81', minOrderSize: '0.1', - maxOrderSize: '1000', pricePrecision: 1, priceDecimals: 1, amountDecimals: 0, @@ -39,14 +38,12 @@ describe('ddex_orders', () => { expected.observedTimestamp = observedTimestamp; expected.orderType = OrderType.Bid; expected.price = new BigNumber(0.5); - // ddex currently confuses base and quote assets. - // Switch them to maintain our internal consistency. - expected.baseAssetSymbol = 'ABC'; - expected.baseAssetAddress = '0x0000000000000000000000000000000000000000'; - expected.baseVolume = new BigNumber(10); - expected.quoteAssetSymbol = 'DEF'; - expected.quoteAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81'; + expected.quoteAssetSymbol = 'ABC'; + expected.quoteAssetAddress = '0x0000000000000000000000000000000000000000'; expected.quoteVolume = new BigNumber(5); + expected.baseAssetSymbol = 'DEF'; + expected.baseAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81'; + expected.baseVolume = new BigNumber(10); const actual = parseDdexOrder(ddexMarket, observedTimestamp, orderType, source, ddexOrder); expect(actual).deep.equal(expected); |