diff options
Diffstat (limited to 'packages/pipeline')
| -rw-r--r-- | packages/pipeline/README.md | 15 | ||||
| -rw-r--r-- | packages/pipeline/src/scripts/pull_radar_relay_orders.ts | 29 | ||||
| -rw-r--r-- | packages/pipeline/test/fixtures/copper/api_v1_list_leads.json | 18 |
3 files changed, 32 insertions, 30 deletions
diff --git a/packages/pipeline/README.md b/packages/pipeline/README.md index 40a64cdd1..4fc8e0ff9 100644 --- a/packages/pipeline/README.md +++ b/packages/pipeline/README.md @@ -164,20 +164,21 @@ set the`ZEROEX_DATA_PIPELINE_DB_URL` environment variable to a valid #### Additional guidelines and tips: -* Table names should be plural and separated by underscores (e.g., +- Table names should be plural and separated by underscores (e.g., `exchange_fill_events`). -* Any table which contains data which comes directly from a third-party source +- Any table which contains data which comes directly from a third-party source should be namespaced in the `raw` PostgreSQL schema. -* Column names in the database should be separated by underscores (e.g., +- Column names in the database should be separated by underscores (e.g., `maker_asset_type`). -* Field names in entity classes (like any other fields in TypeScript) should +- Field names in entity classes (like any other fields in TypeScript) should be camel-cased (e.g., `makerAssetType`). -* All timestamps should be stored as milliseconds since the Unix Epoch. -* Use the `BigNumber` type for TypeScript code which deals with 256-bit +- All timestamps should be stored as milliseconds since the Unix Epoch. +- Use the `BigNumber` type for TypeScript code which deals with 256-bit numbers from smart contracts or for any case where we are dealing with large floating point numbers. -* [TypeORM documentation](http://typeorm.io/#/) is pretty robust and can be a +- [TypeORM documentation](http://typeorm.io/#/) is pretty robust and can be a helpful resource. + * Scripts/parsers should perform minimum data transformation/normalization. The idea here is to have a raw data feed that will be cleaned up and synthesized in a separate step. diff --git a/packages/pipeline/src/scripts/pull_radar_relay_orders.ts b/packages/pipeline/src/scripts/pull_radar_relay_orders.ts index 40bb6fc97..03fc764f2 100644 --- a/packages/pipeline/src/scripts/pull_radar_relay_orders.ts +++ b/packages/pipeline/src/scripts/pull_radar_relay_orders.ts @@ -29,18 +29,23 @@ async function getOrderbookAsync(): Promise<void> { console.log(`Got ${rawOrders.records.length} orders.`); console.log('Parsing orders...'); // Parse the sra orders, then add source url to each. - const orders = R.pipe(parseSraOrders, R.map(setSourceUrl(RADAR_RELAY_URL)))(rawOrders); + const orders = R.pipe( + parseSraOrders, + R.map(setSourceUrl(RADAR_RELAY_URL)), + )(rawOrders); // Save all the orders and update the observed time stamps in a single // transaction. console.log('Saving orders and updating timestamps...'); const observedTimestamp = Date.now(); - await connection.transaction(async (manager: EntityManager): Promise<void> => { - for (const order of orders) { - await manager.save(SraOrder, order); - const orderObservation = createObservedTimestampForOrder(order, observedTimestamp); - await manager.save(orderObservation); - } - }); + await connection.transaction( + async (manager: EntityManager): Promise<void> => { + for (const order of orders) { + await manager.save(SraOrder, order); + const orderObservation = createObservedTimestampForOrder(order, observedTimestamp); + await manager.save(orderObservation); + } + }, + ); } const sourceUrlProp = R.lensProp('sourceUrl'); @@ -49,6 +54,8 @@ const sourceUrlProp = R.lensProp('sourceUrl'); * Sets the source url for a single order. Returns a new order instead of * mutating the given one. */ -const setSourceUrl = R.curry((sourceURL: string, order: SraOrder): SraOrder => { - return R.set(sourceUrlProp, sourceURL, order); -}); +const setSourceUrl = R.curry( + (sourceURL: string, order: SraOrder): SraOrder => { + return R.set(sourceUrlProp, sourceURL, order); + }, +); diff --git a/packages/pipeline/test/fixtures/copper/api_v1_list_leads.json b/packages/pipeline/test/fixtures/copper/api_v1_list_leads.json index 5223976f9..e7161085d 100644 --- a/packages/pipeline/test/fixtures/copper/api_v1_list_leads.json +++ b/packages/pipeline/test/fixtures/copper/api_v1_list_leads.json @@ -305,8 +305,7 @@ "custom_fields": [ { "custom_field_definition_id": 100764, - "value": - "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------" + "value": "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------" }, { "custom_field_definition_id": 103481, @@ -345,8 +344,7 @@ "custom_fields": [ { "custom_field_definition_id": 100764, - "value": - "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----" + "value": "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----" }, { "custom_field_definition_id": 103481, @@ -385,13 +383,11 @@ "custom_fields": [ { "custom_field_definition_id": 100764, - "value": - "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----" + "value": "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----" }, { "custom_field_definition_id": 103481, - "value": - "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------" + "value": "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------" } ], "date_created": 1489791470, @@ -426,13 +422,11 @@ "custom_fields": [ { "custom_field_definition_id": 100764, - "value": - "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----" + "value": "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----" }, { "custom_field_definition_id": 103481, - "value": - "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------" + "value": "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------" } ], "date_created": 1489791672, |