diff options
Diffstat (limited to 'packages/pipeline')
7 files changed, 80 insertions, 46 deletions
diff --git a/packages/pipeline/README.md b/packages/pipeline/README.md index 794488cac..4fc8e0ff9 100644 --- a/packages/pipeline/README.md +++ b/packages/pipeline/README.md @@ -147,20 +147,38 @@ set the`ZEROEX_DATA_PIPELINE_DB_URL` environment variable to a valid starting from that block number. 7. Run the migrations and then run your new script locally and verify it works as expected. +8. After all tests pass and you can run the script locally, open a new PR to + the monorepo. Don't merge this yet! +9. If you added any new scripts or dependencies between scripts, you will need + to make changes to https://github.com/0xProject/0x-pipeline-orchestration + and make a separate PR there. Don't merge this yet! +10. After your PR passes code review, ask @feuGeneA or @xianny to deploy your + changes to the QA environment. Check the [QA Airflow dashboard](http://airflow-qa.0x.org:8080) + to make sure everything works correctly in the QA environment. +11. Merge your PR to 0x-monorepo (and + https://github.com/0xProject/0x-pipeline-orchestration if needed). Then ask + @feuGeneA or @xianny to deploy to production. +12. Monitor the [production Airflow dashboard](http://airflow.0x.org:8080) to + make sure everything still works. +13. Celebrate! :tada: #### Additional guidelines and tips: -* Table names should be plural and separated by underscores (e.g., +- Table names should be plural and separated by underscores (e.g., `exchange_fill_events`). -* Any table which contains data which comes directly from a third-party source +- Any table which contains data which comes directly from a third-party source should be namespaced in the `raw` PostgreSQL schema. -* Column names in the database should be separated by underscores (e.g., +- Column names in the database should be separated by underscores (e.g., `maker_asset_type`). -* Field names in entity classes (like any other fields in TypeScript) should +- Field names in entity classes (like any other fields in TypeScript) should be camel-cased (e.g., `makerAssetType`). -* All timestamps should be stored as milliseconds since the Unix Epoch. -* Use the `BigNumber` type for TypeScript code which deals with 256-bit +- All timestamps should be stored as milliseconds since the Unix Epoch. +- Use the `BigNumber` type for TypeScript code which deals with 256-bit numbers from smart contracts or for any case where we are dealing with large floating point numbers. -* [TypeORM documentation](http://typeorm.io/#/) is pretty robust and can be a +- [TypeORM documentation](http://typeorm.io/#/) is pretty robust and can be a helpful resource. + +* Scripts/parsers should perform minimum data transformation/normalization. + The idea here is to have a raw data feed that will be cleaned up and + synthesized in a separate step. diff --git a/packages/pipeline/migrations/1547153875669-UpdateDDexAPIToV3.ts b/packages/pipeline/migrations/1547153875669-UpdateDDexAPIToV3.ts new file mode 100644 index 000000000..957af4941 --- /dev/null +++ b/packages/pipeline/migrations/1547153875669-UpdateDDexAPIToV3.ts @@ -0,0 +1,21 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class UpdateDDexAPIToV31547153875669 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise<any> { + await queryRunner.query(` + UPDATE raw.token_orderbook_snapshots + SET quote_asset_symbol='WETH' + WHERE quote_asset_symbol='ETH' AND + source='ddex'; + `); + } + + public async down(queryRunner: QueryRunner): Promise<any> { + await queryRunner.query(` + UPDATE raw.token_orderbook_snapshots + SET quote_asset_symbol='ETH' + WHERE quote_asset_symbol='WETH' AND + source='ddex'; + `); + } +} diff --git a/packages/pipeline/src/data_sources/ddex/index.ts b/packages/pipeline/src/data_sources/ddex/index.ts index 2bbd8c29b..7ef92b90f 100644 --- a/packages/pipeline/src/data_sources/ddex/index.ts +++ b/packages/pipeline/src/data_sources/ddex/index.ts @@ -1,6 +1,6 @@ import { fetchAsync, logUtils } from '@0x/utils'; -const DDEX_BASE_URL = 'https://api.ddex.io/v2'; +const DDEX_BASE_URL = 'https://api.ddex.io/v3'; const ACTIVE_MARKETS_URL = `${DDEX_BASE_URL}/markets`; const NO_AGGREGATION_LEVEL = 3; // See https://docs.ddex.io/#get-orderbook const ORDERBOOK_ENDPOINT = `/orderbook?level=${NO_AGGREGATION_LEVEL}`; @@ -23,7 +23,6 @@ export interface DdexMarket { baseTokenDecimals: number; baseTokenAddress: string; minOrderSize: string; - maxOrderSize: string; pricePrecision: number; priceDecimals: number; amountDecimals: number; diff --git a/packages/pipeline/src/parsers/ddex_orders/index.ts b/packages/pipeline/src/parsers/ddex_orders/index.ts index eeb9c9d5b..562f894ab 100644 --- a/packages/pipeline/src/parsers/ddex_orders/index.ts +++ b/packages/pipeline/src/parsers/ddex_orders/index.ts @@ -58,14 +58,12 @@ export function parseDdexOrder( tokenOrder.orderType = orderType; tokenOrder.price = price; - // ddex currently confuses quote and base assets. - // We switch them here to maintain our internal consistency. - tokenOrder.baseAssetSymbol = ddexMarket.quoteToken; - tokenOrder.baseAssetAddress = ddexMarket.quoteTokenAddress; + tokenOrder.baseAssetSymbol = ddexMarket.baseToken; + tokenOrder.baseAssetAddress = ddexMarket.baseTokenAddress; tokenOrder.baseVolume = amount; - tokenOrder.quoteAssetSymbol = ddexMarket.baseToken; - tokenOrder.quoteAssetAddress = ddexMarket.baseTokenAddress; + tokenOrder.quoteAssetSymbol = ddexMarket.quoteToken; + tokenOrder.quoteAssetAddress = ddexMarket.quoteTokenAddress; tokenOrder.quoteVolume = price.times(amount); return tokenOrder; } diff --git a/packages/pipeline/src/scripts/pull_radar_relay_orders.ts b/packages/pipeline/src/scripts/pull_radar_relay_orders.ts index 40bb6fc97..03fc764f2 100644 --- a/packages/pipeline/src/scripts/pull_radar_relay_orders.ts +++ b/packages/pipeline/src/scripts/pull_radar_relay_orders.ts @@ -29,18 +29,23 @@ async function getOrderbookAsync(): Promise<void> { console.log(`Got ${rawOrders.records.length} orders.`); console.log('Parsing orders...'); // Parse the sra orders, then add source url to each. - const orders = R.pipe(parseSraOrders, R.map(setSourceUrl(RADAR_RELAY_URL)))(rawOrders); + const orders = R.pipe( + parseSraOrders, + R.map(setSourceUrl(RADAR_RELAY_URL)), + )(rawOrders); // Save all the orders and update the observed time stamps in a single // transaction. console.log('Saving orders and updating timestamps...'); const observedTimestamp = Date.now(); - await connection.transaction(async (manager: EntityManager): Promise<void> => { - for (const order of orders) { - await manager.save(SraOrder, order); - const orderObservation = createObservedTimestampForOrder(order, observedTimestamp); - await manager.save(orderObservation); - } - }); + await connection.transaction( + async (manager: EntityManager): Promise<void> => { + for (const order of orders) { + await manager.save(SraOrder, order); + const orderObservation = createObservedTimestampForOrder(order, observedTimestamp); + await manager.save(orderObservation); + } + }, + ); } const sourceUrlProp = R.lensProp('sourceUrl'); @@ -49,6 +54,8 @@ const sourceUrlProp = R.lensProp('sourceUrl'); * Sets the source url for a single order. Returns a new order instead of * mutating the given one. */ -const setSourceUrl = R.curry((sourceURL: string, order: SraOrder): SraOrder => { - return R.set(sourceUrlProp, sourceURL, order); -}); +const setSourceUrl = R.curry( + (sourceURL: string, order: SraOrder): SraOrder => { + return R.set(sourceUrlProp, sourceURL, order); + }, +); diff --git a/packages/pipeline/test/fixtures/copper/api_v1_list_leads.json b/packages/pipeline/test/fixtures/copper/api_v1_list_leads.json index 5223976f9..e7161085d 100644 --- a/packages/pipeline/test/fixtures/copper/api_v1_list_leads.json +++ b/packages/pipeline/test/fixtures/copper/api_v1_list_leads.json @@ -305,8 +305,7 @@ "custom_fields": [ { "custom_field_definition_id": 100764, - "value": - "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------" + "value": "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------" }, { "custom_field_definition_id": 103481, @@ -345,8 +344,7 @@ "custom_fields": [ { "custom_field_definition_id": 100764, - "value": - "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----" + "value": "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----" }, { "custom_field_definition_id": 103481, @@ -385,13 +383,11 @@ "custom_fields": [ { "custom_field_definition_id": 100764, - "value": - "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----" + "value": "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----" }, { "custom_field_definition_id": 103481, - "value": - "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------" + "value": "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------" } ], "date_created": 1489791470, @@ -426,13 +422,11 @@ "custom_fields": [ { "custom_field_definition_id": 100764, - "value": - "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----" + "value": "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----" }, { "custom_field_definition_id": 103481, - "value": - "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------" + "value": "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------" } ], "date_created": 1489791672, diff --git a/packages/pipeline/test/parsers/ddex_orders/index_test.ts b/packages/pipeline/test/parsers/ddex_orders/index_test.ts index f30e86b02..d6f69e090 100644 --- a/packages/pipeline/test/parsers/ddex_orders/index_test.ts +++ b/packages/pipeline/test/parsers/ddex_orders/index_test.ts @@ -25,7 +25,6 @@ describe('ddex_orders', () => { baseTokenDecimals: 2, baseTokenAddress: '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81', minOrderSize: '0.1', - maxOrderSize: '1000', pricePrecision: 1, priceDecimals: 1, amountDecimals: 0, @@ -39,14 +38,12 @@ describe('ddex_orders', () => { expected.observedTimestamp = observedTimestamp; expected.orderType = OrderType.Bid; expected.price = new BigNumber(0.5); - // ddex currently confuses base and quote assets. - // Switch them to maintain our internal consistency. - expected.baseAssetSymbol = 'ABC'; - expected.baseAssetAddress = '0x0000000000000000000000000000000000000000'; - expected.baseVolume = new BigNumber(10); - expected.quoteAssetSymbol = 'DEF'; - expected.quoteAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81'; + expected.quoteAssetSymbol = 'ABC'; + expected.quoteAssetAddress = '0x0000000000000000000000000000000000000000'; expected.quoteVolume = new BigNumber(5); + expected.baseAssetSymbol = 'DEF'; + expected.baseAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81'; + expected.baseVolume = new BigNumber(10); const actual = parseDdexOrder(ddexMarket, observedTimestamp, orderType, source, ddexOrder); expect(actual).deep.equal(expected); |