aboutsummaryrefslogtreecommitdiffstats
path: root/packages/pipeline
diff options
context:
space:
mode:
Diffstat (limited to 'packages/pipeline')
-rw-r--r--packages/pipeline/README.md15
-rw-r--r--packages/pipeline/package.json20
-rw-r--r--packages/pipeline/src/parsers/copper/index.ts6
-rw-r--r--packages/pipeline/src/scripts/pull_radar_relay_orders.ts29
-rw-r--r--packages/pipeline/test/fixtures/copper/api_v1_list_leads.json18
5 files changed, 45 insertions, 43 deletions
diff --git a/packages/pipeline/README.md b/packages/pipeline/README.md
index 40a64cdd1..4fc8e0ff9 100644
--- a/packages/pipeline/README.md
+++ b/packages/pipeline/README.md
@@ -164,20 +164,21 @@ set the`ZEROEX_DATA_PIPELINE_DB_URL` environment variable to a valid
#### Additional guidelines and tips:
-* Table names should be plural and separated by underscores (e.g.,
+- Table names should be plural and separated by underscores (e.g.,
`exchange_fill_events`).
-* Any table which contains data which comes directly from a third-party source
+- Any table which contains data which comes directly from a third-party source
should be namespaced in the `raw` PostgreSQL schema.
-* Column names in the database should be separated by underscores (e.g.,
+- Column names in the database should be separated by underscores (e.g.,
`maker_asset_type`).
-* Field names in entity classes (like any other fields in TypeScript) should
+- Field names in entity classes (like any other fields in TypeScript) should
be camel-cased (e.g., `makerAssetType`).
-* All timestamps should be stored as milliseconds since the Unix Epoch.
-* Use the `BigNumber` type for TypeScript code which deals with 256-bit
+- All timestamps should be stored as milliseconds since the Unix Epoch.
+- Use the `BigNumber` type for TypeScript code which deals with 256-bit
numbers from smart contracts or for any case where we are dealing with large
floating point numbers.
-* [TypeORM documentation](http://typeorm.io/#/) is pretty robust and can be a
+- [TypeORM documentation](http://typeorm.io/#/) is pretty robust and can be a
helpful resource.
+
* Scripts/parsers should perform minimum data transformation/normalization.
The idea here is to have a raw data feed that will be cleaned up and
synthesized in a separate step.
diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json
index 47150300d..56d4ecc98 100644
--- a/packages/pipeline/package.json
+++ b/packages/pipeline/package.json
@@ -1,6 +1,6 @@
{
"name": "@0x/pipeline",
- "version": "1.0.3",
+ "version": "1.0.4",
"private": true,
"description": "Data pipeline for offline analysis",
"scripts": {
@@ -27,7 +27,7 @@
},
"license": "Apache-2.0",
"devDependencies": {
- "@0x/tslint-config": "^2.0.0",
+ "@0x/tslint-config": "^2.0.1",
"@types/axios": "^0.14.0",
"@types/ramda": "^0.25.38",
"chai": "^4.1.2",
@@ -39,23 +39,23 @@
"typescript": "3.0.1"
},
"dependencies": {
- "@0x/connect": "^3.0.11",
+ "@0x/connect": "^3.0.12",
"@0x/contract-addresses": "^2.1.0",
"@0x/contract-artifacts": "^1.2.0",
"@0x/contract-wrappers": "^3.0.0",
- "@0x/dev-utils": "^1.0.22",
- "@0x/order-utils": "^3.1.0",
- "@0x/subproviders": "^2.1.9",
- "@0x/types": "^1.5.0",
- "@0x/utils": "^2.1.1",
- "@0x/web3-wrapper": "^3.2.2",
+ "@0x/dev-utils": "^1.0.23",
+ "@0x/order-utils": "^3.1.1",
+ "@0x/subproviders": "^2.1.10",
+ "@0x/types": "^1.5.1",
+ "@0x/utils": "^3.0.0",
+ "@0x/web3-wrapper": "^3.2.3",
"@types/dockerode": "^2.5.9",
"@types/p-limit": "^2.0.0",
"async-parallel": "^1.2.3",
"axios": "^0.18.0",
"bottleneck": "^2.13.2",
"dockerode": "^2.5.7",
- "ethereum-types": "^1.1.4",
+ "ethereum-types": "^1.1.5",
"pg": "^7.5.0",
"prettier": "^1.15.3",
"ramda": "^0.25.0",
diff --git a/packages/pipeline/src/parsers/copper/index.ts b/packages/pipeline/src/parsers/copper/index.ts
index 6c0c5abd5..07da66d10 100644
--- a/packages/pipeline/src/parsers/copper/index.ts
+++ b/packages/pipeline/src/parsers/copper/index.ts
@@ -44,8 +44,8 @@ export interface CopperActivityParentResponse {
// custom activity types
export enum CopperActivityTypeCategory {
- user = 'user',
- system = 'system',
+ User = 'user',
+ System = 'system',
}
export interface CopperActivityTypeResponse {
id: number;
@@ -90,7 +90,7 @@ export enum CopperCustomFieldType {
Date = 'Date',
Checkbox = 'Checkbox',
Float = 'Float',
- URL = 'URL',
+ URL = 'URL', // tslint:disable-line:enum-naming
Percentage = 'Percentage',
Currency = 'Currency',
Connect = 'Connect',
diff --git a/packages/pipeline/src/scripts/pull_radar_relay_orders.ts b/packages/pipeline/src/scripts/pull_radar_relay_orders.ts
index 40bb6fc97..03fc764f2 100644
--- a/packages/pipeline/src/scripts/pull_radar_relay_orders.ts
+++ b/packages/pipeline/src/scripts/pull_radar_relay_orders.ts
@@ -29,18 +29,23 @@ async function getOrderbookAsync(): Promise<void> {
console.log(`Got ${rawOrders.records.length} orders.`);
console.log('Parsing orders...');
// Parse the sra orders, then add source url to each.
- const orders = R.pipe(parseSraOrders, R.map(setSourceUrl(RADAR_RELAY_URL)))(rawOrders);
+ const orders = R.pipe(
+ parseSraOrders,
+ R.map(setSourceUrl(RADAR_RELAY_URL)),
+ )(rawOrders);
// Save all the orders and update the observed time stamps in a single
// transaction.
console.log('Saving orders and updating timestamps...');
const observedTimestamp = Date.now();
- await connection.transaction(async (manager: EntityManager): Promise<void> => {
- for (const order of orders) {
- await manager.save(SraOrder, order);
- const orderObservation = createObservedTimestampForOrder(order, observedTimestamp);
- await manager.save(orderObservation);
- }
- });
+ await connection.transaction(
+ async (manager: EntityManager): Promise<void> => {
+ for (const order of orders) {
+ await manager.save(SraOrder, order);
+ const orderObservation = createObservedTimestampForOrder(order, observedTimestamp);
+ await manager.save(orderObservation);
+ }
+ },
+ );
}
const sourceUrlProp = R.lensProp('sourceUrl');
@@ -49,6 +54,8 @@ const sourceUrlProp = R.lensProp('sourceUrl');
* Sets the source url for a single order. Returns a new order instead of
* mutating the given one.
*/
-const setSourceUrl = R.curry((sourceURL: string, order: SraOrder): SraOrder => {
- return R.set(sourceUrlProp, sourceURL, order);
-});
+const setSourceUrl = R.curry(
+ (sourceURL: string, order: SraOrder): SraOrder => {
+ return R.set(sourceUrlProp, sourceURL, order);
+ },
+);
diff --git a/packages/pipeline/test/fixtures/copper/api_v1_list_leads.json b/packages/pipeline/test/fixtures/copper/api_v1_list_leads.json
index 5223976f9..e7161085d 100644
--- a/packages/pipeline/test/fixtures/copper/api_v1_list_leads.json
+++ b/packages/pipeline/test/fixtures/copper/api_v1_list_leads.json
@@ -305,8 +305,7 @@
"custom_fields": [
{
"custom_field_definition_id": 100764,
- "value":
- "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------"
+ "value": "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------"
},
{
"custom_field_definition_id": 103481,
@@ -345,8 +344,7 @@
"custom_fields": [
{
"custom_field_definition_id": 100764,
- "value":
- "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----"
+ "value": "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----"
},
{
"custom_field_definition_id": 103481,
@@ -385,13 +383,11 @@
"custom_fields": [
{
"custom_field_definition_id": 100764,
- "value":
- "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----"
+ "value": "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----"
},
{
"custom_field_definition_id": 103481,
- "value":
- "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------"
+ "value": "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------"
}
],
"date_created": 1489791470,
@@ -426,13 +422,11 @@
"custom_fields": [
{
"custom_field_definition_id": 100764,
- "value":
- "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----"
+ "value": "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----"
},
{
"custom_field_definition_id": 103481,
- "value":
- "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------"
+ "value": "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------"
}
],
"date_created": 1489791672,
generated by cgit v1.2.3 (git 2.25.1) at 2024-10-28 13:24:01 +0800