aboutsummaryrefslogtreecommitdiffstats
path: root/packages/pipeline
diff options
context:
space:
mode:
authorFabio B <kandinsky454@protonmail.ch>2019-01-11 18:18:18 +0800
committerGitHub <noreply@github.com>2019-01-11 18:18:18 +0800
commit2cf57a48dd2857dd5cf2f31f4c60dd47ae4d34a5 (patch)
tree23187637b8db2927bdcc11b1b42a2e93787ccaaa /packages/pipeline
parenta5b7a351609a5e6689bb97990216153f64302462 (diff)
parenta091b2c19d3dae1eacd9b36f71bca9683001ea33 (diff)
downloaddexon-sol-tools-2cf57a48dd2857dd5cf2f31f4c60dd47ae4d34a5.tar
dexon-sol-tools-2cf57a48dd2857dd5cf2f31f4c60dd47ae4d34a5.tar.gz
dexon-sol-tools-2cf57a48dd2857dd5cf2f31f4c60dd47ae4d34a5.tar.bz2
dexon-sol-tools-2cf57a48dd2857dd5cf2f31f4c60dd47ae4d34a5.tar.lz
dexon-sol-tools-2cf57a48dd2857dd5cf2f31f4c60dd47ae4d34a5.tar.xz
dexon-sol-tools-2cf57a48dd2857dd5cf2f31f4c60dd47ae4d34a5.tar.zst
dexon-sol-tools-2cf57a48dd2857dd5cf2f31f4c60dd47ae4d34a5.zip
Merge pull request #1500 from 0xProject/update-prettier-version
Update prettier to version ^1.15.3
Diffstat (limited to 'packages/pipeline')
-rw-r--r--packages/pipeline/README.md15
-rw-r--r--packages/pipeline/src/scripts/pull_radar_relay_orders.ts29
-rw-r--r--packages/pipeline/test/fixtures/copper/api_v1_list_leads.json18
3 files changed, 32 insertions, 30 deletions
diff --git a/packages/pipeline/README.md b/packages/pipeline/README.md
index 40a64cdd1..4fc8e0ff9 100644
--- a/packages/pipeline/README.md
+++ b/packages/pipeline/README.md
@@ -164,20 +164,21 @@ set the`ZEROEX_DATA_PIPELINE_DB_URL` environment variable to a valid
#### Additional guidelines and tips:
-* Table names should be plural and separated by underscores (e.g.,
+- Table names should be plural and separated by underscores (e.g.,
`exchange_fill_events`).
-* Any table which contains data which comes directly from a third-party source
+- Any table which contains data which comes directly from a third-party source
should be namespaced in the `raw` PostgreSQL schema.
-* Column names in the database should be separated by underscores (e.g.,
+- Column names in the database should be separated by underscores (e.g.,
`maker_asset_type`).
-* Field names in entity classes (like any other fields in TypeScript) should
+- Field names in entity classes (like any other fields in TypeScript) should
be camel-cased (e.g., `makerAssetType`).
-* All timestamps should be stored as milliseconds since the Unix Epoch.
-* Use the `BigNumber` type for TypeScript code which deals with 256-bit
+- All timestamps should be stored as milliseconds since the Unix Epoch.
+- Use the `BigNumber` type for TypeScript code which deals with 256-bit
numbers from smart contracts or for any case where we are dealing with large
floating point numbers.
-* [TypeORM documentation](http://typeorm.io/#/) is pretty robust and can be a
+- [TypeORM documentation](http://typeorm.io/#/) is pretty robust and can be a
helpful resource.
+
* Scripts/parsers should perform minimum data transformation/normalization.
The idea here is to have a raw data feed that will be cleaned up and
synthesized in a separate step.
diff --git a/packages/pipeline/src/scripts/pull_radar_relay_orders.ts b/packages/pipeline/src/scripts/pull_radar_relay_orders.ts
index 40bb6fc97..03fc764f2 100644
--- a/packages/pipeline/src/scripts/pull_radar_relay_orders.ts
+++ b/packages/pipeline/src/scripts/pull_radar_relay_orders.ts
@@ -29,18 +29,23 @@ async function getOrderbookAsync(): Promise<void> {
console.log(`Got ${rawOrders.records.length} orders.`);
console.log('Parsing orders...');
// Parse the sra orders, then add source url to each.
- const orders = R.pipe(parseSraOrders, R.map(setSourceUrl(RADAR_RELAY_URL)))(rawOrders);
+ const orders = R.pipe(
+ parseSraOrders,
+ R.map(setSourceUrl(RADAR_RELAY_URL)),
+ )(rawOrders);
// Save all the orders and update the observed time stamps in a single
// transaction.
console.log('Saving orders and updating timestamps...');
const observedTimestamp = Date.now();
- await connection.transaction(async (manager: EntityManager): Promise<void> => {
- for (const order of orders) {
- await manager.save(SraOrder, order);
- const orderObservation = createObservedTimestampForOrder(order, observedTimestamp);
- await manager.save(orderObservation);
- }
- });
+ await connection.transaction(
+ async (manager: EntityManager): Promise<void> => {
+ for (const order of orders) {
+ await manager.save(SraOrder, order);
+ const orderObservation = createObservedTimestampForOrder(order, observedTimestamp);
+ await manager.save(orderObservation);
+ }
+ },
+ );
}
const sourceUrlProp = R.lensProp('sourceUrl');
@@ -49,6 +54,8 @@ const sourceUrlProp = R.lensProp('sourceUrl');
* Sets the source url for a single order. Returns a new order instead of
* mutating the given one.
*/
-const setSourceUrl = R.curry((sourceURL: string, order: SraOrder): SraOrder => {
- return R.set(sourceUrlProp, sourceURL, order);
-});
+const setSourceUrl = R.curry(
+ (sourceURL: string, order: SraOrder): SraOrder => {
+ return R.set(sourceUrlProp, sourceURL, order);
+ },
+);
diff --git a/packages/pipeline/test/fixtures/copper/api_v1_list_leads.json b/packages/pipeline/test/fixtures/copper/api_v1_list_leads.json
index 5223976f9..e7161085d 100644
--- a/packages/pipeline/test/fixtures/copper/api_v1_list_leads.json
+++ b/packages/pipeline/test/fixtures/copper/api_v1_list_leads.json
@@ -305,8 +305,7 @@
"custom_fields": [
{
"custom_field_definition_id": 100764,
- "value":
- "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------"
+ "value": "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------"
},
{
"custom_field_definition_id": 103481,
@@ -345,8 +344,7 @@
"custom_fields": [
{
"custom_field_definition_id": 100764,
- "value":
- "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----"
+ "value": "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----"
},
{
"custom_field_definition_id": 103481,
@@ -385,13 +383,11 @@
"custom_fields": [
{
"custom_field_definition_id": 100764,
- "value":
- "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----"
+ "value": "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----"
},
{
"custom_field_definition_id": 103481,
- "value":
- "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------"
+ "value": "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------"
}
],
"date_created": 1489791470,
@@ -426,13 +422,11 @@
"custom_fields": [
{
"custom_field_definition_id": 100764,
- "value":
- "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----"
+ "value": "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----"
},
{
"custom_field_definition_id": 103481,
- "value":
- "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------"
+ "value": "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------"
}
],
"date_created": 1489791672,