aboutsummaryrefslogtreecommitdiffstats
path: root/packages/pipeline
diff options
context:
space:
mode:
authorSteve Klebanoff <steve.klebanoff@gmail.com>2019-01-10 00:54:55 +0800
committerSteve Klebanoff <steve.klebanoff@gmail.com>2019-01-10 00:54:55 +0800
commitfb3605026ef63ef6897010a52bf3f4c116cdf271 (patch)
treeaf3887e5f785e049daeaf795e0f3e4a9ad370d93 /packages/pipeline
parent76dde294f10bb3d5c1074cd6599668bcb1cdd8ec (diff)
parent5b8c9122a292f6558c45440b053ceae52d36d495 (diff)
downloaddexon-sol-tools-fb3605026ef63ef6897010a52bf3f4c116cdf271.tar
dexon-sol-tools-fb3605026ef63ef6897010a52bf3f4c116cdf271.tar.gz
dexon-sol-tools-fb3605026ef63ef6897010a52bf3f4c116cdf271.tar.bz2
dexon-sol-tools-fb3605026ef63ef6897010a52bf3f4c116cdf271.tar.lz
dexon-sol-tools-fb3605026ef63ef6897010a52bf3f4c116cdf271.tar.xz
dexon-sol-tools-fb3605026ef63ef6897010a52bf3f4c116cdf271.tar.zst
dexon-sol-tools-fb3605026ef63ef6897010a52bf3f4c116cdf271.zip
Merge branch 'development' into feature/instant/tell-amount-available
Diffstat (limited to 'packages/pipeline')
-rw-r--r--packages/pipeline/migrations/1545440485644-CreateCopperTables.ts103
-rw-r--r--packages/pipeline/package.json22
-rw-r--r--packages/pipeline/src/data_sources/copper/index.ts126
-rw-r--r--packages/pipeline/src/entities/copper_activity.ts41
-rw-r--r--packages/pipeline/src/entities/copper_activity_type.ts17
-rw-r--r--packages/pipeline/src/entities/copper_custom_field.ts15
-rw-r--r--packages/pipeline/src/entities/copper_lead.ts38
-rw-r--r--packages/pipeline/src/entities/copper_opportunity.ts45
-rw-r--r--packages/pipeline/src/entities/index.ts6
-rw-r--r--packages/pipeline/src/entities/token_order.ts3
-rw-r--r--packages/pipeline/src/ormconfig.ts10
-rw-r--r--packages/pipeline/src/parsers/copper/index.ts259
-rw-r--r--packages/pipeline/src/parsers/ddex_orders/index.ts8
-rw-r--r--packages/pipeline/src/parsers/events/exchange_events.ts38
-rw-r--r--packages/pipeline/src/parsers/idex_orders/index.ts12
-rw-r--r--packages/pipeline/src/parsers/oasis_orders/index.ts8
-rw-r--r--packages/pipeline/src/parsers/paradex_orders/index.ts4
-rw-r--r--packages/pipeline/src/parsers/sra_orders/index.ts20
-rw-r--r--packages/pipeline/src/scripts/pull_copper.ts129
-rw-r--r--packages/pipeline/src/scripts/pull_missing_blocks.ts39
-rw-r--r--packages/pipeline/src/types.ts11
-rw-r--r--packages/pipeline/src/utils/transformers/asset_proxy_id_types.ts20
-rw-r--r--packages/pipeline/src/utils/transformers/index.ts1
-rw-r--r--packages/pipeline/src/utils/transformers/number_to_bigint.ts8
-rw-r--r--packages/pipeline/test/entities/copper_test.ts54
-rw-r--r--packages/pipeline/test/entities/util.ts4
-rw-r--r--packages/pipeline/test/fixtures/copper/api_v1_activity_types.json24
-rw-r--r--packages/pipeline/test/fixtures/copper/api_v1_activity_types.ts16
-rw-r--r--packages/pipeline/test/fixtures/copper/api_v1_custom_field_definitions.json38
-rw-r--r--packages/pipeline/test/fixtures/copper/api_v1_custom_field_definitions.ts39
-rw-r--r--packages/pipeline/test/fixtures/copper/api_v1_list_activities.json242
-rw-r--r--packages/pipeline/test/fixtures/copper/api_v1_list_activities.ts305
-rw-r--r--packages/pipeline/test/fixtures/copper/api_v1_list_leads.json583
-rw-r--r--packages/pipeline/test/fixtures/copper/api_v1_list_leads.ts229
-rw-r--r--packages/pipeline/test/fixtures/copper/api_v1_list_opportunities.json662
-rw-r--r--packages/pipeline/test/fixtures/copper/api_v1_list_opportunities.ts425
-rw-r--r--packages/pipeline/test/fixtures/copper/parsed_entities.ts5
-rw-r--r--packages/pipeline/test/parsers/copper/index_test.ts87
-rw-r--r--packages/pipeline/test/parsers/ddex_orders/index_test.ts4
-rw-r--r--packages/pipeline/test/parsers/events/exchange_events_test.ts5
-rw-r--r--packages/pipeline/test/parsers/idex_orders/index_test.ts8
-rw-r--r--packages/pipeline/test/parsers/oasis_orders/index_test.ts4
-rw-r--r--packages/pipeline/test/parsers/paradex_orders/index_test.ts4
-rw-r--r--packages/pipeline/test/parsers/sra_orders/index_test.ts5
-rw-r--r--packages/pipeline/tsconfig.json12
45 files changed, 3652 insertions, 86 deletions
diff --git a/packages/pipeline/migrations/1545440485644-CreateCopperTables.ts b/packages/pipeline/migrations/1545440485644-CreateCopperTables.ts
new file mode 100644
index 000000000..64bf70af4
--- /dev/null
+++ b/packages/pipeline/migrations/1545440485644-CreateCopperTables.ts
@@ -0,0 +1,103 @@
+import { MigrationInterface, QueryRunner, Table } from 'typeorm';
+
+const leads = new Table({
+ name: 'raw.copper_leads',
+ columns: [
+ { name: 'id', type: 'bigint', isPrimary: true },
+ { name: 'name', type: 'varchar', isNullable: true },
+ { name: 'first_name', type: 'varchar', isNullable: true },
+ { name: 'last_name', type: 'varchar', isNullable: true },
+ { name: 'middle_name', type: 'varchar', isNullable: true },
+ { name: 'assignee_id', type: 'bigint', isNullable: true },
+ { name: 'company_name', type: 'varchar', isNullable: true },
+ { name: 'customer_source_id', type: 'bigint', isNullable: true },
+ { name: 'monetary_value', type: 'integer', isNullable: true },
+ { name: 'status', type: 'varchar' },
+ { name: 'status_id', type: 'bigint' },
+ { name: 'title', type: 'varchar', isNullable: true },
+ { name: 'date_created', type: 'bigint' },
+ { name: 'date_modified', type: 'bigint', isPrimary: true },
+ ],
+});
+const activities = new Table({
+ name: 'raw.copper_activities',
+ columns: [
+ { name: 'id', type: 'bigint', isPrimary: true },
+ { name: 'parent_id', type: 'bigint' },
+ { name: 'parent_type', type: 'varchar' },
+ { name: 'type_id', type: 'bigint' },
+ { name: 'type_category', type: 'varchar' },
+ { name: 'type_name', type: 'varchar', isNullable: true },
+ { name: 'user_id', type: 'bigint' },
+ { name: 'old_value_id', type: 'bigint', isNullable: true },
+ { name: 'old_value_name', type: 'varchar', isNullable: true },
+ { name: 'new_value_id', type: 'bigint', isNullable: true },
+ { name: 'new_value_name', type: 'varchar', isNullable: true },
+ { name: 'date_created', type: 'bigint' },
+ { name: 'date_modified', type: 'bigint', isPrimary: true },
+ ],
+});
+
+const opportunities = new Table({
+ name: 'raw.copper_opportunities',
+ columns: [
+ { name: 'id', type: 'bigint', isPrimary: true },
+ { name: 'name', type: 'varchar' },
+ { name: 'assignee_id', isNullable: true, type: 'bigint' },
+ { name: 'close_date', isNullable: true, type: 'varchar' },
+ { name: 'company_id', isNullable: true, type: 'bigint' },
+ { name: 'company_name', isNullable: true, type: 'varchar' },
+ { name: 'customer_source_id', isNullable: true, type: 'bigint' },
+ { name: 'loss_reason_id', isNullable: true, type: 'bigint' },
+ { name: 'pipeline_id', type: 'bigint' },
+ { name: 'pipeline_stage_id', type: 'bigint' },
+ { name: 'primary_contact_id', isNullable: true, type: 'bigint' },
+ { name: 'priority', isNullable: true, type: 'varchar' },
+ { name: 'status', type: 'varchar' },
+ { name: 'interaction_count', type: 'bigint' },
+ { name: 'monetary_value', isNullable: true, type: 'integer' },
+ { name: 'win_probability', isNullable: true, type: 'integer' },
+ { name: 'date_created', type: 'bigint' },
+ { name: 'date_modified', type: 'bigint', isPrimary: true },
+ { name: 'custom_fields', type: 'jsonb' },
+ ],
+});
+
+const activityTypes = new Table({
+ name: 'raw.copper_activity_types',
+ columns: [
+ { name: 'id', type: 'bigint', isPrimary: true },
+ { name: 'category', type: 'varchar' },
+ { name: 'name', type: 'varchar' },
+ { name: 'is_disabled', type: 'boolean', isNullable: true },
+ { name: 'count_as_interaction', type: 'boolean', isNullable: true },
+ ],
+});
+
+const customFields = new Table({
+ name: 'raw.copper_custom_fields',
+ columns: [
+ { name: 'id', type: 'bigint', isPrimary: true },
+ { name: 'name', type: 'varchar' },
+ { name: 'data_type', type: 'varchar' },
+ { name: 'field_type', type: 'varchar', isNullable: true },
+ ],
+});
+
+export class CreateCopperTables1544055699284 implements MigrationInterface {
+ public async up(queryRunner: QueryRunner): Promise<any> {
+ await queryRunner.createTable(leads);
+ await queryRunner.createTable(activities);
+ await queryRunner.createTable(opportunities);
+ await queryRunner.createTable(activityTypes);
+ await queryRunner.createTable(customFields);
+ }
+
+ public async down(queryRunner: QueryRunner): Promise<any> {
+ await queryRunner.dropTable(leads.name);
+ await queryRunner.dropTable(activities.name);
+ await queryRunner.dropTable(opportunities.name);
+ await queryRunner.dropTable(activityTypes.name);
+ await queryRunner.dropTable(customFields.name);
+ }
+}
diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json
index a40f3d21c..47150300d 100644
--- a/packages/pipeline/package.json
+++ b/packages/pipeline/package.json
@@ -1,6 +1,6 @@
{
"name": "@0x/pipeline",
- "version": "1.0.2",
+ "version": "1.0.3",
"private": true,
"description": "Data pipeline for offline analysis",
"scripts": {
@@ -16,7 +16,7 @@
"test:coverage": "nyc npm run test:all --all && yarn coverage:report:lcov",
"coverage:report:lcov": "nyc report --reporter=text-lcov > coverage/lcov.info",
"clean": "shx rm -rf lib",
- "lint": "tslint --project . --format stylish --exclude ./migrations/**/*",
+ "lint": "tslint --project . --format stylish --exclude ./migrations/**/* --exclude ./test/fixtures/**/**/*.json",
"migrate:run": "yarn typeorm migration:run --config ./lib/src/ormconfig",
"migrate:revert": "yarn typeorm migration:revert --config ./lib/src/ormconfig",
"migrate:create": "yarn typeorm migration:create --config ./lib/src/ormconfig --dir migrations"
@@ -39,16 +39,16 @@
"typescript": "3.0.1"
},
"dependencies": {
- "@0x/connect": "^3.0.10",
- "@0x/contract-addresses": "^2.0.0",
- "@0x/contract-artifacts": "^1.0.1",
+ "@0x/connect": "^3.0.11",
+ "@0x/contract-addresses": "^2.1.0",
+ "@0x/contract-artifacts": "^1.2.0",
"@0x/contract-wrappers": "^3.0.0",
- "@0x/dev-utils": "^1.0.21",
- "@0x/order-utils": "^2.0.0",
- "@0x/subproviders": "^2.1.8",
- "@0x/types": "^1.4.1",
- "@0x/utils": "^2.0.8",
- "@0x/web3-wrapper": "^3.2.1",
+ "@0x/dev-utils": "^1.0.22",
+ "@0x/order-utils": "^3.1.0",
+ "@0x/subproviders": "^2.1.9",
+ "@0x/types": "^1.5.0",
+ "@0x/utils": "^2.1.1",
+ "@0x/web3-wrapper": "^3.2.2",
"@types/dockerode": "^2.5.9",
"@types/p-limit": "^2.0.0",
"async-parallel": "^1.2.3",
diff --git a/packages/pipeline/src/data_sources/copper/index.ts b/packages/pipeline/src/data_sources/copper/index.ts
new file mode 100644
index 000000000..15df2fd7d
--- /dev/null
+++ b/packages/pipeline/src/data_sources/copper/index.ts
@@ -0,0 +1,126 @@
+import { fetchAsync } from '@0x/utils';
+import Bottleneck from 'bottleneck';
+
+import {
+ CopperActivityTypeCategory,
+ CopperActivityTypeResponse,
+ CopperCustomFieldResponse,
+ CopperSearchResponse,
+} from '../../parsers/copper';
+
+const HTTP_OK_STATUS = 200;
+const COPPER_URI = 'https://api.prosperworks.com/developer_api/v1';
+
+const DEFAULT_PAGINATION_PARAMS = {
+ page_size: 200,
+ sort_by: 'date_modified',
+ sort_direction: 'desc',
+};
+
+export type CopperSearchParams = CopperLeadSearchParams | CopperActivitySearchParams | CopperOpportunitySearchParams;
+export interface CopperLeadSearchParams {
+ page_number?: number;
+}
+
+export interface CopperActivitySearchParams {
+ minimum_activity_date: number;
+ page_number?: number;
+}
+
+export interface CopperOpportunitySearchParams {
+ sort_by: string; // must override the default 'date_modified' for this endpoint
+ page_number?: number;
+}
+export enum CopperEndpoint {
+ Leads = '/leads/search',
+ Opportunities = '/opportunities/search',
+ Activities = '/activities/search',
+}
+const ONE_SECOND = 1000;
+
+function httpErrorCheck(response: Response): void {
+ if (response.status !== HTTP_OK_STATUS) {
+ throw new Error(`HTTP error while scraping Copper: [${JSON.stringify(response)}]`);
+ }
+}
+export class CopperSource {
+ private readonly _accessToken: string;
+ private readonly _userEmail: string;
+ private readonly _defaultHeaders: any;
+ private readonly _limiter: Bottleneck;
+
+ constructor(maxConcurrentRequests: number, accessToken: string, userEmail: string) {
+ this._accessToken = accessToken;
+ this._userEmail = userEmail;
+ this._defaultHeaders = {
+ 'Content-Type': 'application/json',
+ 'X-PW-AccessToken': this._accessToken,
+ 'X-PW-Application': 'developer_api',
+ 'X-PW-UserEmail': this._userEmail,
+ };
+ this._limiter = new Bottleneck({
+ minTime: ONE_SECOND / maxConcurrentRequests,
+ reservoir: 30,
+ reservoirRefreshAmount: 30,
+ reservoirRefreshInterval: maxConcurrentRequests,
+ });
+ }
+
+ public async fetchNumberOfPagesAsync(endpoint: CopperEndpoint, searchParams?: CopperSearchParams): Promise<number> {
+ const resp = await this._limiter.schedule(() =>
+ fetchAsync(COPPER_URI + endpoint, {
+ method: 'POST',
+ body: JSON.stringify({ ...DEFAULT_PAGINATION_PARAMS, ...searchParams }),
+ headers: this._defaultHeaders,
+ }),
+ );
+
+ httpErrorCheck(resp);
+
+ // total number of records that match the request parameters
+ if (resp.headers.has('X-Pw-Total')) {
+ const totalRecords: number = parseInt(resp.headers.get('X-Pw-Total') as string, 10); // tslint:disable-line:custom-no-magic-numbers
+ return Math.ceil(totalRecords / DEFAULT_PAGINATION_PARAMS.page_size);
+ } else {
+ return 1;
+ }
+ }
+ public async fetchSearchResultsAsync<T extends CopperSearchResponse>(
+ endpoint: CopperEndpoint,
+ searchParams?: CopperSearchParams,
+ ): Promise<T[]> {
+ const request = { ...DEFAULT_PAGINATION_PARAMS, ...searchParams };
+ const response = await this._limiter.schedule(() =>
+ fetchAsync(COPPER_URI + endpoint, {
+ method: 'POST',
+ body: JSON.stringify(request),
+ headers: this._defaultHeaders,
+ }),
+ );
+ httpErrorCheck(response);
+ const json: T[] = await response.json();
+ return json;
+ }
+
+ public async fetchActivityTypesAsync(): Promise<Map<CopperActivityTypeCategory, CopperActivityTypeResponse[]>> {
+ const response = await this._limiter.schedule(() =>
+ fetchAsync(`${COPPER_URI}/activity_types`, {
+ method: 'GET',
+ headers: this._defaultHeaders,
+ }),
+ );
+ httpErrorCheck(response);
+ return response.json();
+ }
+
+ public async fetchCustomFieldsAsync(): Promise<CopperCustomFieldResponse[]> {
+ const response = await this._limiter.schedule(() =>
+ fetchAsync(`${COPPER_URI}/custom_field_definitions`, {
+ method: 'GET',
+ headers: this._defaultHeaders,
+ }),
+ );
+ httpErrorCheck(response);
+ return response.json();
+ }
+}
diff --git a/packages/pipeline/src/entities/copper_activity.ts b/packages/pipeline/src/entities/copper_activity.ts
new file mode 100644
index 000000000..cbc034285
--- /dev/null
+++ b/packages/pipeline/src/entities/copper_activity.ts
@@ -0,0 +1,41 @@
+import { Column, Entity, Index, PrimaryColumn } from 'typeorm';
+
+import { numberToBigIntTransformer } from '../utils';
+
+@Entity({ name: 'copper_activities', schema: 'raw' })
+export class CopperActivity {
+ @PrimaryColumn({ type: 'bigint', transformer: numberToBigIntTransformer })
+ public id!: number;
+
+ @Index()
+ @Column({ name: 'parent_id', type: 'bigint', transformer: numberToBigIntTransformer })
+ public parentId!: number;
+ @Column({ name: 'parent_type', type: 'varchar' })
+ public parentType!: string;
+
+ // join with CopperActivityType
+ @Index()
+ @Column({ name: 'type_id', type: 'bigint', transformer: numberToBigIntTransformer })
+ public typeId!: number;
+ @Column({ name: 'type_category', type: 'varchar' })
+ public typeCategory!: string;
+ @Column({ name: 'type_name', type: 'varchar', nullable: true })
+ public typeName?: string;
+
+ @Column({ name: 'user_id', type: 'bigint', transformer: numberToBigIntTransformer })
+ public userId!: number;
+ @Column({ name: 'old_value_id', type: 'bigint', nullable: true, transformer: numberToBigIntTransformer })
+ public oldValueId?: number;
+ @Column({ name: 'old_value_name', type: 'varchar', nullable: true })
+ public oldValueName?: string;
+ @Column({ name: 'new_value_id', type: 'bigint', nullable: true, transformer: numberToBigIntTransformer })
+ public newValueId?: number;
+ @Column({ name: 'new_value_name', type: 'varchar', nullable: true })
+ public newValueName?: string;
+
+ @Index()
+ @Column({ name: 'date_created', type: 'bigint', transformer: numberToBigIntTransformer })
+ public dateCreated!: number;
+ @PrimaryColumn({ name: 'date_modified', type: 'bigint', transformer: numberToBigIntTransformer })
+ public dateModified!: number;
+}
diff --git a/packages/pipeline/src/entities/copper_activity_type.ts b/packages/pipeline/src/entities/copper_activity_type.ts
new file mode 100644
index 000000000..8fb2dcf70
--- /dev/null
+++ b/packages/pipeline/src/entities/copper_activity_type.ts
@@ -0,0 +1,17 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+import { numberToBigIntTransformer } from '../utils';
+
+@Entity({ name: 'copper_activity_types', schema: 'raw' })
+export class CopperActivityType {
+ @PrimaryColumn({ type: 'bigint', transformer: numberToBigIntTransformer })
+ public id!: number;
+ @Column({ name: 'category', type: 'varchar' })
+ public category!: string;
+ @Column({ name: 'name', type: 'varchar' })
+ public name!: string;
+ @Column({ name: 'is_disabled', type: 'boolean', nullable: true })
+ public isDisabled?: boolean;
+ @Column({ name: 'count_as_interaction', type: 'boolean', nullable: true })
+ public countAsInteraction?: boolean;
+}
diff --git a/packages/pipeline/src/entities/copper_custom_field.ts b/packages/pipeline/src/entities/copper_custom_field.ts
new file mode 100644
index 000000000..f23f6ab22
--- /dev/null
+++ b/packages/pipeline/src/entities/copper_custom_field.ts
@@ -0,0 +1,15 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+import { numberToBigIntTransformer } from '../utils';
+
+@Entity({ name: 'copper_custom_fields', schema: 'raw' })
+export class CopperCustomField {
+ @PrimaryColumn({ type: 'bigint', transformer: numberToBigIntTransformer })
+ public id!: number;
+ @Column({ name: 'data_type', type: 'varchar' })
+ public dataType!: string;
+ @Column({ name: 'field_type', type: 'varchar', nullable: true })
+ public fieldType?: string;
+ @Column({ name: 'name', type: 'varchar' })
+ public name!: string;
+}
diff --git a/packages/pipeline/src/entities/copper_lead.ts b/packages/pipeline/src/entities/copper_lead.ts
new file mode 100644
index 000000000..c51ccd761
--- /dev/null
+++ b/packages/pipeline/src/entities/copper_lead.ts
@@ -0,0 +1,38 @@
+import { Column, Entity, Index, PrimaryColumn } from 'typeorm';
+
+import { numberToBigIntTransformer } from '../utils';
+
+@Entity({ name: 'copper_leads', schema: 'raw' })
+export class CopperLead {
+ @PrimaryColumn({ type: 'bigint', transformer: numberToBigIntTransformer })
+ public id!: number;
+
+ @Column({ name: 'name', type: 'varchar', nullable: true })
+ public name?: string;
+ @Column({ name: 'first_name', type: 'varchar', nullable: true })
+ public firstName?: string;
+ @Column({ name: 'last_name', type: 'varchar', nullable: true })
+ public lastName?: string;
+ @Column({ name: 'middle_name', type: 'varchar', nullable: true })
+ public middleName?: string;
+ @Column({ name: 'assignee_id', type: 'bigint', transformer: numberToBigIntTransformer, nullable: true })
+ public assigneeId?: number;
+ @Column({ name: 'company_name', type: 'varchar', nullable: true })
+ public companyName?: string;
+ @Column({ name: 'customer_source_id', type: 'bigint', transformer: numberToBigIntTransformer, nullable: true })
+ public customerSourceId?: number;
+ @Column({ name: 'monetary_value', type: 'integer', nullable: true })
+ public monetaryValue?: number;
+ @Column({ name: 'status', type: 'varchar' })
+ public status!: string;
+ @Column({ name: 'status_id', type: 'bigint', transformer: numberToBigIntTransformer })
+ public statusId!: number;
+ @Column({ name: 'title', type: 'varchar', nullable: true })
+ public title?: string;
+
+ @Index()
+ @Column({ name: 'date_created', type: 'bigint', transformer: numberToBigIntTransformer })
+ public dateCreated!: number;
+ @PrimaryColumn({ name: 'date_modified', type: 'bigint', transformer: numberToBigIntTransformer })
+ public dateModified!: number;
+}
diff --git a/packages/pipeline/src/entities/copper_opportunity.ts b/packages/pipeline/src/entities/copper_opportunity.ts
new file mode 100644
index 000000000..e12bd69ce
--- /dev/null
+++ b/packages/pipeline/src/entities/copper_opportunity.ts
@@ -0,0 +1,45 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+import { numberToBigIntTransformer } from '../utils';
+
+@Entity({ name: 'copper_opportunities', schema: 'raw' })
+export class CopperOpportunity {
+ @PrimaryColumn({ name: 'id', type: 'bigint', transformer: numberToBigIntTransformer })
+ public id!: number;
+ @Column({ name: 'name', type: 'varchar' })
+ public name!: string;
+ @Column({ name: 'assignee_id', nullable: true, type: 'bigint', transformer: numberToBigIntTransformer })
+ public assigneeId?: number;
+ @Column({ name: 'close_date', nullable: true, type: 'varchar' })
+ public closeDate?: string;
+ @Column({ name: 'company_id', nullable: true, type: 'bigint', transformer: numberToBigIntTransformer })
+ public companyId?: number;
+ @Column({ name: 'company_name', nullable: true, type: 'varchar' })
+ public companyName?: string;
+ @Column({ name: 'customer_source_id', nullable: true, type: 'bigint', transformer: numberToBigIntTransformer })
+ public customerSourceId?: number;
+ @Column({ name: 'loss_reason_id', nullable: true, type: 'bigint', transformer: numberToBigIntTransformer })
+ public lossReasonId?: number;
+ @Column({ name: 'pipeline_id', type: 'bigint', transformer: numberToBigIntTransformer })
+ public pipelineId!: number;
+ @Column({ name: 'pipeline_stage_id', type: 'bigint', transformer: numberToBigIntTransformer })
+ public pipelineStageId!: number;
+ @Column({ name: 'primary_contact_id', nullable: true, type: 'bigint', transformer: numberToBigIntTransformer })
+ public primaryContactId?: number;
+ @Column({ name: 'priority', nullable: true, type: 'varchar' })
+ public priority?: string;
+ @Column({ name: 'status', type: 'varchar' })
+ public status!: string;
+ @Column({ name: 'interaction_count', type: 'bigint', transformer: numberToBigIntTransformer })
+ public interactionCount!: number;
+ @Column({ name: 'monetary_value', nullable: true, type: 'integer' })
+ public monetaryValue?: number;
+ @Column({ name: 'win_probability', nullable: true, type: 'integer' })
+ public winProbability?: number;
+ @Column({ name: 'date_created', type: 'bigint', transformer: numberToBigIntTransformer })
+ public dateCreated!: number;
+ @PrimaryColumn({ name: 'date_modified', type: 'bigint', transformer: numberToBigIntTransformer })
+ public dateModified!: number;
+ @Column({ name: 'custom_fields', type: 'jsonb' })
+ public customFields!: { [key: number]: number };
+}
diff --git a/packages/pipeline/src/entities/index.ts b/packages/pipeline/src/entities/index.ts
index cc3de78bb..27c153c07 100644
--- a/packages/pipeline/src/entities/index.ts
+++ b/packages/pipeline/src/entities/index.ts
@@ -16,4 +16,10 @@ export { TokenOrderbookSnapshot } from './token_order';
export { Transaction } from './transaction';
export { ERC20ApprovalEvent } from './erc20_approval_event';
+export { CopperLead } from './copper_lead';
+export { CopperActivity } from './copper_activity';
+export { CopperOpportunity } from './copper_opportunity';
+export { CopperActivityType } from './copper_activity_type';
+export { CopperCustomField } from './copper_custom_field';
+
export type ExchangeEvent = ExchangeFillEvent | ExchangeCancelEvent | ExchangeCancelUpToEvent;
diff --git a/packages/pipeline/src/entities/token_order.ts b/packages/pipeline/src/entities/token_order.ts
index 4b8f0abc3..2709747cb 100644
--- a/packages/pipeline/src/entities/token_order.ts
+++ b/packages/pipeline/src/entities/token_order.ts
@@ -1,7 +1,6 @@
import { BigNumber } from '@0x/utils';
import { Column, Entity, PrimaryColumn } from 'typeorm';
-import { OrderType } from '../types';
import { bigNumberTransformer, numberToBigIntTransformer } from '../utils';
@Entity({ name: 'token_orderbook_snapshots', schema: 'raw' })
@@ -11,7 +10,7 @@ export class TokenOrderbookSnapshot {
@PrimaryColumn({ name: 'source' })
public source!: string;
@PrimaryColumn({ name: 'order_type' })
- public orderType!: OrderType;
+ public orderType!: string;
@PrimaryColumn({ name: 'price', type: 'numeric', transformer: bigNumberTransformer })
public price!: BigNumber;
@PrimaryColumn({ name: 'base_asset_symbol' })
diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts
index fe11d81d5..2700714cd 100644
--- a/packages/pipeline/src/ormconfig.ts
+++ b/packages/pipeline/src/ormconfig.ts
@@ -2,6 +2,11 @@ import { ConnectionOptions } from 'typeorm';
import {
Block,
+ CopperActivity,
+ CopperActivityType,
+ CopperCustomField,
+ CopperLead,
+ CopperOpportunity,
DexTrade,
ERC20ApprovalEvent,
ExchangeCancelEvent,
@@ -18,6 +23,11 @@ import {
const entities = [
Block,
+ CopperOpportunity,
+ CopperActivity,
+ CopperActivityType,
+ CopperCustomField,
+ CopperLead,
DexTrade,
ExchangeCancelEvent,
ExchangeCancelUpToEvent,
diff --git a/packages/pipeline/src/parsers/copper/index.ts b/packages/pipeline/src/parsers/copper/index.ts
new file mode 100644
index 000000000..6c0c5abd5
--- /dev/null
+++ b/packages/pipeline/src/parsers/copper/index.ts
@@ -0,0 +1,259 @@
+import * as R from 'ramda';
+
+import { CopperActivity, CopperActivityType, CopperCustomField, CopperLead, CopperOpportunity } from '../../entities';
+
+const ONE_SECOND = 1000;
+export type CopperSearchResponse = CopperLeadResponse | CopperActivityResponse | CopperOpportunityResponse;
+export interface CopperLeadResponse {
+ id: number;
+ name?: string;
+ first_name?: string;
+ last_name?: string;
+ middle_name?: string;
+ assignee_id?: number;
+ company_name?: string;
+ customer_source_id?: number;
+ monetary_value?: number;
+ status: string;
+ status_id: number;
+ title?: string;
+ date_created: number; // in seconds
+ date_modified: number; // in seconds
+}
+
+export interface CopperActivityResponse {
+ id: number;
+ parent: CopperActivityParentResponse;
+ type: CopperActivityTypeResponse;
+ user_id: number;
+ activity_date: number;
+ old_value: CopperActivityValueResponse;
+ new_value: CopperActivityValueResponse;
+ date_created: number; // in seconds
+ date_modified: number; // in seconds
+}
+
+export interface CopperActivityValueResponse {
+ id: number;
+ name: string;
+}
+export interface CopperActivityParentResponse {
+ id: number;
+ type: string;
+}
+
+// custom activity types
+export enum CopperActivityTypeCategory {
+ user = 'user',
+ system = 'system',
+}
+export interface CopperActivityTypeResponse {
+ id: number;
+ category: CopperActivityTypeCategory;
+ name: string;
+ is_disabled?: boolean;
+ count_as_interaction?: boolean;
+}
+
+export interface CopperOpportunityResponse {
+ id: number;
+ name: string;
+ assignee_id?: number;
+ close_date?: string;
+ company_id?: number;
+ company_name?: string;
+ customer_source_id?: number;
+ loss_reason_id?: number;
+ pipeline_id: number;
+ pipeline_stage_id: number;
+ primary_contact_id?: number;
+ priority?: string;
+ status: string;
+ tags: string[];
+ interaction_count: number;
+ monetary_value?: number;
+ win_probability?: number;
+ date_created: number; // in seconds
+ date_modified: number; // in seconds
+ custom_fields: CopperNestedCustomFieldResponse[];
+}
+interface CopperNestedCustomFieldResponse {
+ custom_field_definition_id: number;
+ value: number | number[] | null;
+}
+// custom fields
+export enum CopperCustomFieldType {
+ String = 'String',
+ Text = 'Text',
+ Dropdown = 'Dropdown',
+ MultiSelect = 'MultiSelect', // not in API documentation but shows up in results
+ Date = 'Date',
+ Checkbox = 'Checkbox',
+ Float = 'Float',
+ URL = 'URL',
+ Percentage = 'Percentage',
+ Currency = 'Currency',
+ Connect = 'Connect',
+}
+export interface CopperCustomFieldOptionResponse {
+ id: number;
+ name: string;
+}
+export interface CopperCustomFieldResponse {
+ id: number;
+ name: string;
+ data_type: CopperCustomFieldType;
+ options?: CopperCustomFieldOptionResponse[];
+}
+/**
+ * Parse response from Copper API /search/leads/
+ *
+ * @param leads - The array of leads returned from the API
+ * @returns Returns an array of Copper Lead entities
+ */
+export function parseLeads(leads: CopperLeadResponse[]): CopperLead[] {
+ return leads.map(lead => {
+ const entity = new CopperLead();
+ entity.id = lead.id;
+ entity.name = lead.name || undefined;
+ entity.firstName = lead.first_name || undefined;
+ entity.lastName = lead.last_name || undefined;
+ entity.middleName = lead.middle_name || undefined;
+ entity.assigneeId = lead.assignee_id || undefined;
+ entity.companyName = lead.company_name || undefined;
+ entity.customerSourceId = lead.customer_source_id || undefined;
+ entity.monetaryValue = lead.monetary_value || undefined;
+ entity.status = lead.status;
+ entity.statusId = lead.status_id;
+ entity.title = lead.title || undefined;
+ entity.dateCreated = lead.date_created * ONE_SECOND;
+ entity.dateModified = lead.date_modified * ONE_SECOND;
+ return entity;
+ });
+}
+
+/**
+ * Parse response from Copper API /search/activities/
+ *
+ * @param activities - The array of activities returned from the API
+ * @returns Returns an array of Copper Activity entities
+ */
+export function parseActivities(activities: CopperActivityResponse[]): CopperActivity[] {
+ return activities.map(activity => {
+ const entity = new CopperActivity();
+ entity.id = activity.id;
+
+ entity.parentId = activity.parent.id;
+ entity.parentType = activity.parent.type;
+
+ entity.typeId = activity.type.id;
+ entity.typeCategory = activity.type.category.toString();
+ entity.typeName = activity.type.name;
+
+ entity.userId = activity.user_id;
+ entity.dateCreated = activity.date_created * ONE_SECOND;
+ entity.dateModified = activity.date_modified * ONE_SECOND;
+
+ // nested nullable fields
+ entity.oldValueId = R.path(['old_value', 'id'], activity);
+ entity.oldValueName = R.path(['old_value', 'name'], activity);
+ entity.newValueId = R.path(['new_value', 'id'], activity);
+ entity.newValueName = R.path(['new_value', 'name'], activity);
+
+ return entity;
+ });
+}
+
+/**
+ * Parse response from Copper API /search/opportunities/
+ *
+ * @param opportunities - The array of opportunities returned from the API
+ * @returns Returns an array of Copper Opportunity entities
+ */
+export function parseOpportunities(opportunities: CopperOpportunityResponse[]): CopperOpportunity[] {
+ return opportunities.map(opp => {
+ const customFields: { [key: number]: number } = opp.custom_fields
+ .filter(f => f.value !== null)
+ .map(f => ({
+ ...f,
+ value: ([] as number[]).concat(f.value || []), // normalise all values to number[]
+ }))
+ .map(f => f.value.map(val => [f.custom_field_definition_id, val] as [number, number])) // pair each value with the custom_field_definition_id
+ .reduce((acc, pair) => acc.concat(pair)) // flatten
+ .reduce<{ [key: number]: number }>((obj, [key, value]) => {
+ // transform into object literal
+ obj[key] = value;
+ return obj;
+ }, {});
+
+ const entity = new CopperOpportunity();
+ entity.id = opp.id;
+ entity.name = opp.name;
+ entity.assigneeId = opp.assignee_id || undefined;
+ entity.closeDate = opp.close_date || undefined;
+ entity.companyId = opp.company_id || undefined;
+ entity.companyName = opp.company_name || undefined;
+ entity.customerSourceId = opp.customer_source_id || undefined;
+ entity.lossReasonId = opp.loss_reason_id || undefined;
+ entity.pipelineId = opp.pipeline_id;
+ entity.pipelineStageId = opp.pipeline_stage_id;
+ entity.primaryContactId = opp.primary_contact_id || undefined;
+ entity.priority = opp.priority || undefined;
+ entity.status = opp.status;
+ entity.interactionCount = opp.interaction_count;
+ entity.monetaryValue = opp.monetary_value || undefined;
+ entity.winProbability = opp.win_probability === null ? undefined : opp.win_probability;
+ entity.dateCreated = opp.date_created * ONE_SECOND;
+ entity.dateModified = opp.date_modified * ONE_SECOND;
+ entity.customFields = customFields;
+ return entity;
+ });
+}
+
+/**
+ * Parse response from Copper API /activity_types/
+ *
+ * @param activityTypeResponse - Activity Types response from the API, keyed by "user" or "system"
+ * @returns Returns an array of Copper Activity Type entities
+ */
+export function parseActivityTypes(
+ activityTypeResponse: Map<CopperActivityTypeCategory, CopperActivityTypeResponse[]>,
+): CopperActivityType[] {
+ const values: CopperActivityTypeResponse[] = R.flatten(Object.values(activityTypeResponse));
+ return values.map(activityType => ({
+ id: activityType.id,
+ name: activityType.name,
+ category: activityType.category.toString(),
+ isDisabled: activityType.is_disabled,
+ countAsInteraction: activityType.count_as_interaction,
+ }));
+}
+
+/**
+ * Parse response from Copper API /custom_field_definitions/
+ *
+ * @param customFieldResponse - array of custom field definitions returned from the API, consisting of top-level fields and nested fields
+ * @returns Returns an array of Copper Custom Field entities
+ */
+export function parseCustomFields(customFieldResponse: CopperCustomFieldResponse[]): CopperCustomField[] {
+ function parseTopLevelField(field: CopperCustomFieldResponse): CopperCustomField[] {
+ const topLevelField: CopperCustomField = {
+ id: field.id,
+ name: field.name,
+ dataType: field.data_type.toString(),
+ };
+
+ if (field.options !== undefined) {
+ const nestedFields: CopperCustomField[] = field.options.map(option => ({
+ id: option.id,
+ name: option.name,
+ dataType: field.name,
+ fieldType: 'option',
+ }));
+ return nestedFields.concat(topLevelField);
+ } else {
+ return [topLevelField];
+ }
+ }
+ return R.chain(parseTopLevelField, customFieldResponse);
+}
diff --git a/packages/pipeline/src/parsers/ddex_orders/index.ts b/packages/pipeline/src/parsers/ddex_orders/index.ts
index d7b97efbe..eeb9c9d5b 100644
--- a/packages/pipeline/src/parsers/ddex_orders/index.ts
+++ b/packages/pipeline/src/parsers/ddex_orders/index.ts
@@ -23,8 +23,12 @@ export function parseDdexOrders(
): TokenOrder[] {
const aggregatedBids = aggregateOrders(ddexOrderbook.bids);
const aggregatedAsks = aggregateOrders(ddexOrderbook.asks);
- const parsedBids = aggregatedBids.map(order => parseDdexOrder(ddexMarket, observedTimestamp, 'bid', source, order));
- const parsedAsks = aggregatedAsks.map(order => parseDdexOrder(ddexMarket, observedTimestamp, 'ask', source, order));
+ const parsedBids = aggregatedBids.map(order =>
+ parseDdexOrder(ddexMarket, observedTimestamp, OrderType.Bid, source, order),
+ );
+ const parsedAsks = aggregatedAsks.map(order =>
+ parseDdexOrder(ddexMarket, observedTimestamp, OrderType.Ask, source, order),
+ );
return parsedBids.concat(parsedAsks);
}
diff --git a/packages/pipeline/src/parsers/events/exchange_events.ts b/packages/pipeline/src/parsers/events/exchange_events.ts
index e18106c75..9c4a5f89a 100644
--- a/packages/pipeline/src/parsers/events/exchange_events.ts
+++ b/packages/pipeline/src/parsers/events/exchange_events.ts
@@ -5,7 +5,7 @@ import { LogWithDecodedArgs } from 'ethereum-types';
import * as R from 'ramda';
import { ExchangeCancelEvent, ExchangeCancelUpToEvent, ExchangeFillEvent } from '../../entities';
-import { bigNumbertoStringOrNull } from '../../utils';
+import { bigNumbertoStringOrNull, convertAssetProxyIdToType } from '../../utils';
/**
* Parses raw event logs for a fill event and returns an array of
@@ -40,9 +40,7 @@ export const parseExchangeCancelUpToEvents: (
*/
export function _convertToExchangeFillEvent(eventLog: LogWithDecodedArgs<ExchangeFillEventArgs>): ExchangeFillEvent {
const makerAssetData = assetDataUtils.decodeAssetDataOrThrow(eventLog.args.makerAssetData);
- const makerAssetType = makerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721';
const takerAssetData = assetDataUtils.decodeAssetDataOrThrow(eventLog.args.takerAssetData);
- const takerAssetType = takerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721';
const exchangeFillEvent = new ExchangeFillEvent();
exchangeFillEvent.contractAddress = eventLog.address as string;
exchangeFillEvent.blockNumber = eventLog.blockNumber as number;
@@ -59,16 +57,24 @@ export function _convertToExchangeFillEvent(eventLog: LogWithDecodedArgs<Exchang
exchangeFillEvent.takerFeePaid = eventLog.args.takerFeePaid;
exchangeFillEvent.orderHash = eventLog.args.orderHash;
exchangeFillEvent.rawMakerAssetData = eventLog.args.makerAssetData;
- exchangeFillEvent.makerAssetType = makerAssetType;
+ // tslint:disable-next-line:no-unnecessary-type-assertion
+ exchangeFillEvent.makerAssetType = convertAssetProxyIdToType(makerAssetData.assetProxyId as AssetProxyId);
exchangeFillEvent.makerAssetProxyId = makerAssetData.assetProxyId;
- exchangeFillEvent.makerTokenAddress = makerAssetData.tokenAddress;
+ // HACK(abandeali1): this event schema currently does not support multiple maker/taker assets, so we store the first token address from the MultiAssetProxy assetData
+ exchangeFillEvent.makerTokenAddress = assetDataUtils.isMultiAssetData(makerAssetData)
+ ? assetDataUtils.decodeMultiAssetDataRecursively(eventLog.args.makerAssetData).nestedAssetData[0].tokenAddress
+ : makerAssetData.tokenAddress;
// tslint has a false positive here. Type assertion is required.
// tslint:disable-next-line:no-unnecessary-type-assertion
exchangeFillEvent.makerTokenId = bigNumbertoStringOrNull((makerAssetData as ERC721AssetData).tokenId);
exchangeFillEvent.rawTakerAssetData = eventLog.args.takerAssetData;
- exchangeFillEvent.takerAssetType = takerAssetType;
+ // tslint:disable-next-line:no-unnecessary-type-assertion
+ exchangeFillEvent.takerAssetType = convertAssetProxyIdToType(takerAssetData.assetProxyId as AssetProxyId);
exchangeFillEvent.takerAssetProxyId = takerAssetData.assetProxyId;
- exchangeFillEvent.takerTokenAddress = takerAssetData.tokenAddress;
+ // HACK(abandeali1): this event schema currently does not support multiple maker/taker assets, so we store the first token address from the MultiAssetProxy assetData
+ exchangeFillEvent.takerTokenAddress = assetDataUtils.isMultiAssetData(takerAssetData)
+ ? assetDataUtils.decodeMultiAssetDataRecursively(eventLog.args.takerAssetData).nestedAssetData[0].tokenAddress
+ : takerAssetData.tokenAddress;
// tslint:disable-next-line:no-unnecessary-type-assertion
exchangeFillEvent.takerTokenId = bigNumbertoStringOrNull((takerAssetData as ERC721AssetData).tokenId);
return exchangeFillEvent;
@@ -83,9 +89,7 @@ export function _convertToExchangeCancelEvent(
eventLog: LogWithDecodedArgs<ExchangeCancelEventArgs>,
): ExchangeCancelEvent {
const makerAssetData = assetDataUtils.decodeAssetDataOrThrow(eventLog.args.makerAssetData);
- const makerAssetType = makerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721';
const takerAssetData = assetDataUtils.decodeAssetDataOrThrow(eventLog.args.takerAssetData);
- const takerAssetType = takerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721';
const exchangeCancelEvent = new ExchangeCancelEvent();
exchangeCancelEvent.contractAddress = eventLog.address as string;
exchangeCancelEvent.blockNumber = eventLog.blockNumber as number;
@@ -98,15 +102,23 @@ export function _convertToExchangeCancelEvent(
exchangeCancelEvent.senderAddress = eventLog.args.senderAddress;
exchangeCancelEvent.orderHash = eventLog.args.orderHash;
exchangeCancelEvent.rawMakerAssetData = eventLog.args.makerAssetData;
- exchangeCancelEvent.makerAssetType = makerAssetType;
+ // tslint:disable-next-line:no-unnecessary-type-assertion
+ exchangeCancelEvent.makerAssetType = convertAssetProxyIdToType(makerAssetData.assetProxyId as AssetProxyId);
exchangeCancelEvent.makerAssetProxyId = makerAssetData.assetProxyId;
- exchangeCancelEvent.makerTokenAddress = makerAssetData.tokenAddress;
+ // HACK(abandeali1): this event schema currently does not support multiple maker/taker assets, so we store the first token address from the MultiAssetProxy assetData
+ exchangeCancelEvent.makerTokenAddress = assetDataUtils.isMultiAssetData(makerAssetData)
+ ? assetDataUtils.decodeMultiAssetDataRecursively(eventLog.args.makerAssetData).nestedAssetData[0].tokenAddress
+ : makerAssetData.tokenAddress;
// tslint:disable-next-line:no-unnecessary-type-assertion
exchangeCancelEvent.makerTokenId = bigNumbertoStringOrNull((makerAssetData as ERC721AssetData).tokenId);
exchangeCancelEvent.rawTakerAssetData = eventLog.args.takerAssetData;
- exchangeCancelEvent.takerAssetType = takerAssetType;
+ // tslint:disable-next-line:no-unnecessary-type-assertion
+ exchangeCancelEvent.takerAssetType = convertAssetProxyIdToType(takerAssetData.assetProxyId as AssetProxyId);
exchangeCancelEvent.takerAssetProxyId = takerAssetData.assetProxyId;
- exchangeCancelEvent.takerTokenAddress = takerAssetData.tokenAddress;
+ // HACK(abandeali1): this event schema currently does not support multiple maker/taker assets, so we store the first token address from the MultiAssetProxy assetData
+ exchangeCancelEvent.takerTokenAddress = assetDataUtils.isMultiAssetData(takerAssetData)
+ ? assetDataUtils.decodeMultiAssetDataRecursively(eventLog.args.takerAssetData).nestedAssetData[0].tokenAddress
+ : takerAssetData.tokenAddress;
// tslint:disable-next-line:no-unnecessary-type-assertion
exchangeCancelEvent.takerTokenId = bigNumbertoStringOrNull((takerAssetData as ERC721AssetData).tokenId);
return exchangeCancelEvent;
diff --git a/packages/pipeline/src/parsers/idex_orders/index.ts b/packages/pipeline/src/parsers/idex_orders/index.ts
index dfe27455c..14b871195 100644
--- a/packages/pipeline/src/parsers/idex_orders/index.ts
+++ b/packages/pipeline/src/parsers/idex_orders/index.ts
@@ -2,7 +2,7 @@ import { BigNumber } from '@0x/utils';
import { aggregateOrders } from '../utils';
-import { IdexOrder, IdexOrderbook, IdexOrderParam } from '../../data_sources/idex';
+import { IdexOrderbook, IdexOrderParam } from '../../data_sources/idex';
import { TokenOrderbookSnapshot as TokenOrder } from '../../entities';
import { OrderType } from '../../types';
@@ -21,7 +21,9 @@ export function parseIdexOrders(idexOrderbook: IdexOrderbook, observedTimestamp:
const idexBidOrder = idexOrderbook.bids[0];
const parsedBids =
aggregatedBids.length > 0
- ? aggregatedBids.map(order => parseIdexOrder(idexBidOrder.params, observedTimestamp, 'bid', source, order))
+ ? aggregatedBids.map(order =>
+ parseIdexOrder(idexBidOrder.params, observedTimestamp, OrderType.Bid, source, order),
+ )
: [];
const aggregatedAsks = aggregateOrders(idexOrderbook.asks);
@@ -29,7 +31,9 @@ export function parseIdexOrders(idexOrderbook: IdexOrderbook, observedTimestamp:
const idexAskOrder = idexOrderbook.asks[0];
const parsedAsks =
aggregatedAsks.length > 0
- ? aggregatedAsks.map(order => parseIdexOrder(idexAskOrder.params, observedTimestamp, 'ask', source, order))
+ ? aggregatedAsks.map(order =>
+ parseIdexOrder(idexAskOrder.params, observedTimestamp, OrderType.Ask, source, order),
+ )
: [];
return parsedBids.concat(parsedAsks);
}
@@ -62,7 +66,7 @@ export function parseIdexOrder(
tokenOrder.baseVolume = amount;
tokenOrder.quoteVolume = price.times(amount);
- if (orderType === 'bid') {
+ if (orderType === OrderType.Bid) {
tokenOrder.baseAssetSymbol = idexOrderParam.buySymbol;
tokenOrder.baseAssetAddress = idexOrderParam.tokenBuy;
tokenOrder.quoteAssetSymbol = idexOrderParam.sellSymbol;
diff --git a/packages/pipeline/src/parsers/oasis_orders/index.ts b/packages/pipeline/src/parsers/oasis_orders/index.ts
index 13997f31b..b71fb65b9 100644
--- a/packages/pipeline/src/parsers/oasis_orders/index.ts
+++ b/packages/pipeline/src/parsers/oasis_orders/index.ts
@@ -23,13 +23,13 @@ export function parseOasisOrders(
observedTimestamp: number,
source: string,
): TokenOrder[] {
- const aggregatedBids = aggregateOrders(R.filter(R.propEq('act', 'bid'), oasisOrderbook));
- const aggregatedAsks = aggregateOrders(R.filter(R.propEq('act', 'ask'), oasisOrderbook));
+ const aggregatedBids = aggregateOrders(R.filter(R.propEq('act', OrderType.Bid), oasisOrderbook));
+ const aggregatedAsks = aggregateOrders(R.filter(R.propEq('act', OrderType.Ask), oasisOrderbook));
const parsedBids = aggregatedBids.map(order =>
- parseOasisOrder(oasisMarket, observedTimestamp, 'bid', source, order),
+ parseOasisOrder(oasisMarket, observedTimestamp, OrderType.Bid, source, order),
);
const parsedAsks = aggregatedAsks.map(order =>
- parseOasisOrder(oasisMarket, observedTimestamp, 'ask', source, order),
+ parseOasisOrder(oasisMarket, observedTimestamp, OrderType.Ask, source, order),
);
return parsedBids.concat(parsedAsks);
}
diff --git a/packages/pipeline/src/parsers/paradex_orders/index.ts b/packages/pipeline/src/parsers/paradex_orders/index.ts
index 5ceeb64a4..85990dae4 100644
--- a/packages/pipeline/src/parsers/paradex_orders/index.ts
+++ b/packages/pipeline/src/parsers/paradex_orders/index.ts
@@ -21,10 +21,10 @@ export function parseParadexOrders(
source: string,
): TokenOrder[] {
const parsedBids = paradexOrderbookResponse.bids.map(order =>
- parseParadexOrder(paradexMarket, observedTimestamp, 'bid', source, order),
+ parseParadexOrder(paradexMarket, observedTimestamp, OrderType.Bid, source, order),
);
const parsedAsks = paradexOrderbookResponse.asks.map(order =>
- parseParadexOrder(paradexMarket, observedTimestamp, 'ask', source, order),
+ parseParadexOrder(paradexMarket, observedTimestamp, OrderType.Ask, source, order),
);
return parsedBids.concat(parsedAsks);
}
diff --git a/packages/pipeline/src/parsers/sra_orders/index.ts b/packages/pipeline/src/parsers/sra_orders/index.ts
index ef8901e40..13fe632a4 100644
--- a/packages/pipeline/src/parsers/sra_orders/index.ts
+++ b/packages/pipeline/src/parsers/sra_orders/index.ts
@@ -4,7 +4,7 @@ import { AssetProxyId, ERC721AssetData } from '@0x/types';
import * as R from 'ramda';
import { SraOrder } from '../../entities';
-import { bigNumbertoStringOrNull } from '../../utils';
+import { bigNumbertoStringOrNull, convertAssetProxyIdToType } from '../../utils';
/**
* Parses a raw order response from an SRA endpoint and returns an array of
@@ -22,9 +22,7 @@ export function parseSraOrders(rawOrdersResponse: OrdersResponse): SraOrder[] {
export function _convertToEntity(apiOrder: APIOrder): SraOrder {
// TODO(albrow): refactor out common asset data decoding code.
const makerAssetData = assetDataUtils.decodeAssetDataOrThrow(apiOrder.order.makerAssetData);
- const makerAssetType = makerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721';
const takerAssetData = assetDataUtils.decodeAssetDataOrThrow(apiOrder.order.takerAssetData);
- const takerAssetType = takerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721';
const sraOrder = new SraOrder();
sraOrder.exchangeAddress = apiOrder.order.exchangeAddress;
@@ -43,16 +41,24 @@ export function _convertToEntity(apiOrder: APIOrder): SraOrder {
sraOrder.signature = apiOrder.order.signature;
sraOrder.rawMakerAssetData = apiOrder.order.makerAssetData;
- sraOrder.makerAssetType = makerAssetType;
+ // tslint:disable-next-line:no-unnecessary-type-assertion
+ sraOrder.makerAssetType = convertAssetProxyIdToType(makerAssetData.assetProxyId as AssetProxyId);
sraOrder.makerAssetProxyId = makerAssetData.assetProxyId;
- sraOrder.makerTokenAddress = makerAssetData.tokenAddress;
+ // HACK(abandeali1): this event schema currently does not support multiple maker/taker assets, so we store the first token address from the MultiAssetProxy assetData
+ sraOrder.makerTokenAddress = assetDataUtils.isMultiAssetData(makerAssetData)
+ ? assetDataUtils.decodeMultiAssetDataRecursively(apiOrder.order.makerAssetData).nestedAssetData[0].tokenAddress
+ : makerAssetData.tokenAddress;
// tslint has a false positive here. Type assertion is required.
// tslint:disable-next-line:no-unnecessary-type-assertion
sraOrder.makerTokenId = bigNumbertoStringOrNull((makerAssetData as ERC721AssetData).tokenId);
sraOrder.rawTakerAssetData = apiOrder.order.takerAssetData;
- sraOrder.takerAssetType = takerAssetType;
+ // tslint:disable-next-line:no-unnecessary-type-assertion
+ sraOrder.takerAssetType = convertAssetProxyIdToType(takerAssetData.assetProxyId as AssetProxyId);
sraOrder.takerAssetProxyId = takerAssetData.assetProxyId;
- sraOrder.takerTokenAddress = takerAssetData.tokenAddress;
+ // HACK(abandeali1): this event schema currently does not support multiple maker/taker assets, so we store the first token address from the MultiAssetProxy assetData
+ sraOrder.takerTokenAddress = assetDataUtils.isMultiAssetData(takerAssetData)
+ ? assetDataUtils.decodeMultiAssetDataRecursively(apiOrder.order.takerAssetData).nestedAssetData[0].tokenAddress
+ : takerAssetData.tokenAddress;
// tslint:disable-next-line:no-unnecessary-type-assertion
sraOrder.takerTokenId = bigNumbertoStringOrNull((takerAssetData as ERC721AssetData).tokenId);
diff --git a/packages/pipeline/src/scripts/pull_copper.ts b/packages/pipeline/src/scripts/pull_copper.ts
new file mode 100644
index 000000000..69814f209
--- /dev/null
+++ b/packages/pipeline/src/scripts/pull_copper.ts
@@ -0,0 +1,129 @@
+// tslint:disable:no-console
+import * as R from 'ramda';
+import { Connection, ConnectionOptions, createConnection, Repository } from 'typeorm';
+
+import { CopperEndpoint, CopperSearchParams, CopperSource } from '../data_sources/copper';
+import { CopperActivity, CopperActivityType, CopperCustomField, CopperLead, CopperOpportunity } from '../entities';
+import * as ormConfig from '../ormconfig';
+import {
+ CopperSearchResponse,
+ parseActivities,
+ parseActivityTypes,
+ parseCustomFields,
+ parseLeads,
+ parseOpportunities,
+} from '../parsers/copper';
+import { handleError } from '../utils';
+const ONE_SECOND = 1000;
+const COPPER_RATE_LIMIT = 10;
+let connection: Connection;
+
+(async () => {
+ connection = await createConnection(ormConfig as ConnectionOptions);
+
+ const accessToken = process.env.COPPER_ACCESS_TOKEN;
+ const userEmail = process.env.COPPER_USER_EMAIL;
+ if (accessToken === undefined || userEmail === undefined) {
+ throw new Error('Missing required env var: COPPER_ACCESS_TOKEN and/or COPPER_USER_EMAIL');
+ }
+ const source = new CopperSource(COPPER_RATE_LIMIT, accessToken, userEmail);
+
+ const fetchPromises = [
+ fetchAndSaveLeadsAsync(source),
+ fetchAndSaveOpportunitiesAsync(source),
+ fetchAndSaveActivitiesAsync(source),
+ fetchAndSaveCustomFieldsAsync(source),
+ fetchAndSaveActivityTypesAsync(source),
+ ];
+ fetchPromises.forEach(async fn => {
+ await fn;
+ });
+})().catch(handleError);
+
+async function fetchAndSaveLeadsAsync(source: CopperSource): Promise<void> {
+ const repository = connection.getRepository(CopperLead);
+ const startTime = await getMaxAsync(connection, 'date_modified', 'raw.copper_leads');
+ console.log(`Fetching Copper leads starting from ${startTime}...`);
+ await fetchAndSaveAsync(CopperEndpoint.Leads, source, startTime, {}, parseLeads, repository);
+}
+
+async function fetchAndSaveOpportunitiesAsync(source: CopperSource): Promise<void> {
+ const repository = connection.getRepository(CopperOpportunity);
+ const startTime = await getMaxAsync(connection, 'date_modified', 'raw.copper_opportunities');
+ console.log(`Fetching Copper opportunities starting from ${startTime}...`);
+ await fetchAndSaveAsync(
+ CopperEndpoint.Opportunities,
+ source,
+ startTime,
+ { sort_by: 'name' },
+ parseOpportunities,
+ repository,
+ );
+}
+
+async function fetchAndSaveActivitiesAsync(source: CopperSource): Promise<void> {
+ const repository = connection.getRepository(CopperActivity);
+ const startTime = await getMaxAsync(connection, 'date_modified', 'raw.copper_activities');
+ const searchParams = {
+ minimum_activity_date: Math.floor(startTime / ONE_SECOND),
+ };
+ console.log(`Fetching Copper activities starting from ${startTime}...`);
+ await fetchAndSaveAsync(CopperEndpoint.Activities, source, startTime, searchParams, parseActivities, repository);
+}
+
+async function getMaxAsync(conn: Connection, sortColumn: string, tableName: string): Promise<number> {
+ const queryResult = await conn.query(`SELECT MAX(${sortColumn}) as _max from ${tableName};`);
+ if (R.isEmpty(queryResult)) {
+ return 0;
+ } else {
+ return queryResult[0]._max;
+ }
+}
+
+// (Xianny): Copper API doesn't allow queries to filter by date. To ensure that we are filling in ascending chronological
+// order and not missing any records, we are scraping all available pages. If Copper data gets larger,
+// it would make sense to search for and start filling from the first page that contains a new record.
+// This search would increase our network calls and is not efficient to implement with our current small volume
+// of Copper records.
+async function fetchAndSaveAsync<T extends CopperSearchResponse, E>(
+ endpoint: CopperEndpoint,
+ source: CopperSource,
+ startTime: number,
+ searchParams: CopperSearchParams,
+ parseFn: (recs: T[]) => E[],
+ repository: Repository<E>,
+): Promise<void> {
+ let saved = 0;
+ const numPages = await source.fetchNumberOfPagesAsync(endpoint);
+ try {
+ for (let i = numPages; i > 0; i--) {
+ console.log(`Fetching page ${i}/${numPages} of ${endpoint}...`);
+ const raw = await source.fetchSearchResultsAsync<T>(endpoint, {
+ ...searchParams,
+ page_number: i,
+ });
+ const newRecords = raw.filter(rec => rec.date_modified * ONE_SECOND > startTime);
+ const parsed = parseFn(newRecords);
+ await repository.save<any>(parsed);
+ saved += newRecords.length;
+ }
+ } catch (err) {
+ console.log(`Error fetching ${endpoint}, stopping: ${err.stack}`);
+ } finally {
+ console.log(`Saved ${saved} items from ${endpoint}, done.`);
+ }
+}
+
+async function fetchAndSaveActivityTypesAsync(source: CopperSource): Promise<void> {
+ console.log(`Fetching Copper activity types...`);
+ const activityTypes = await source.fetchActivityTypesAsync();
+ const repository = connection.getRepository(CopperActivityType);
+ await repository.save(parseActivityTypes(activityTypes));
+}
+
+async function fetchAndSaveCustomFieldsAsync(source: CopperSource): Promise<void> {
+ console.log(`Fetching Copper custom fields...`);
+ const customFields = await source.fetchCustomFieldsAsync();
+ const repository = connection.getRepository(CopperCustomField);
+ await repository.save(parseCustomFields(customFields));
+}
diff --git a/packages/pipeline/src/scripts/pull_missing_blocks.ts b/packages/pipeline/src/scripts/pull_missing_blocks.ts
index bb5385126..ced9d99eb 100644
--- a/packages/pipeline/src/scripts/pull_missing_blocks.ts
+++ b/packages/pipeline/src/scripts/pull_missing_blocks.ts
@@ -14,20 +14,29 @@ import { handleError, INFURA_ROOT_URL } from '../utils';
// Number of blocks to save at once.
const BATCH_SAVE_SIZE = 1000;
// Maximum number of requests to send at once.
-const MAX_CONCURRENCY = 10;
+const MAX_CONCURRENCY = 20;
// Maximum number of blocks to query for at once. This is also the maximum
// number of blocks we will hold in memory prior to being saved to the database.
const MAX_BLOCKS_PER_QUERY = 1000;
let connection: Connection;
+const tablesWithMissingBlocks = [
+ 'raw.exchange_fill_events',
+ 'raw.exchange_cancel_events',
+ 'raw.exchange_cancel_up_to_events',
+ 'raw.erc20_approval_events',
+];
+
(async () => {
connection = await createConnection(ormConfig as ConnectionOptions);
const provider = web3Factory.getRpcProvider({
rpcUrl: INFURA_ROOT_URL,
});
const web3Source = new Web3Source(provider);
- await getAllMissingBlocksAsync(web3Source);
+ for (const tableName of tablesWithMissingBlocks) {
+ await getAllMissingBlocksAsync(web3Source, tableName);
+ }
process.exit(0);
})().catch(handleError);
@@ -35,10 +44,11 @@ interface MissingBlocksResponse {
block_number: string;
}
-async function getAllMissingBlocksAsync(web3Source: Web3Source): Promise<void> {
+async function getAllMissingBlocksAsync(web3Source: Web3Source, tableName: string): Promise<void> {
const blocksRepository = connection.getRepository(Block);
while (true) {
- const blockNumbers = await getMissingBlockNumbersAsync();
+ console.log(`Checking for missing blocks in ${tableName}...`);
+ const blockNumbers = await getMissingBlockNumbersAsync(tableName);
if (blockNumbers.length === 0) {
// There are no more missing blocks. We're done.
break;
@@ -46,24 +56,14 @@ async function getAllMissingBlocksAsync(web3Source: Web3Source): Promise<void> {
await getAndSaveBlocksAsync(web3Source, blocksRepository, blockNumbers);
}
const totalBlocks = await blocksRepository.count();
- console.log(`Done saving blocks. There are now ${totalBlocks} total blocks.`);
+ console.log(`Done saving blocks for ${tableName}. There are now ${totalBlocks} total blocks.`);
}
-async function getMissingBlockNumbersAsync(): Promise<number[]> {
- // Note(albrow): The easiest way to get all the blocks we need is to
- // consider all the events tables together in a single query. If this query
- // gets too slow, we should consider re-architecting so that we can work on
- // getting the blocks for one type of event at a time.
+async function getMissingBlockNumbersAsync(tableName: string): Promise<number[]> {
+ // This query returns up to `MAX_BLOCKS_PER_QUERY` distinct block numbers
+ // which are present in `tableName` but not in `raw.blocks`.
const response = (await connection.query(
- `WITH all_events AS (
- SELECT block_number FROM raw.exchange_fill_events
- UNION SELECT block_number FROM raw.exchange_cancel_events
- UNION SELECT block_number FROM raw.exchange_cancel_up_to_events
- UNION SELECT block_number FROM raw.erc20_approval_events
- )
- SELECT DISTINCT(block_number) FROM all_events
- WHERE block_number NOT IN (SELECT number FROM raw.blocks)
- ORDER BY block_number ASC LIMIT $1`,
+ `SELECT DISTINCT(block_number) FROM ${tableName} LEFT JOIN raw.blocks ON ${tableName}.block_number = raw.blocks.number WHERE number IS NULL LIMIT $1;`,
[MAX_BLOCKS_PER_QUERY],
)) as MissingBlocksResponse[];
const blockNumberStrings = R.pluck('block_number', response);
@@ -86,4 +86,5 @@ async function getAndSaveBlocksAsync(
const blocks = R.map(parseBlock, rawBlocks);
console.log(`Saving ${blocks.length} blocks...`);
await blocksRepository.save(blocks, { chunk: Math.ceil(blocks.length / BATCH_SAVE_SIZE) });
+ console.log('Done saving this batch of blocks');
}
diff --git a/packages/pipeline/src/types.ts b/packages/pipeline/src/types.ts
index e02b42a40..5f2121807 100644
--- a/packages/pipeline/src/types.ts
+++ b/packages/pipeline/src/types.ts
@@ -1,2 +1,9 @@
-export type AssetType = 'erc20' | 'erc721';
-export type OrderType = 'bid' | 'ask';
+export enum AssetType {
+ ERC20 = 'erc20',
+ ERC721 = 'erc721',
+ MultiAsset = 'multiAsset',
+}
+export enum OrderType {
+ Bid = 'bid',
+ Ask = 'ask',
+}
diff --git a/packages/pipeline/src/utils/transformers/asset_proxy_id_types.ts b/packages/pipeline/src/utils/transformers/asset_proxy_id_types.ts
new file mode 100644
index 000000000..2cd05a616
--- /dev/null
+++ b/packages/pipeline/src/utils/transformers/asset_proxy_id_types.ts
@@ -0,0 +1,20 @@
+import { AssetProxyId } from '@0x/types';
+
+import { AssetType } from '../../types';
+
+/**
+ * Converts an assetProxyId to its string equivalent
+ * @param assetProxyId Id of AssetProxy
+ */
+export function convertAssetProxyIdToType(assetProxyId: AssetProxyId): AssetType {
+ switch (assetProxyId) {
+ case AssetProxyId.ERC20:
+ return AssetType.ERC20;
+ case AssetProxyId.ERC721:
+ return AssetType.ERC721;
+ case AssetProxyId.MultiAsset:
+ return AssetType.MultiAsset;
+ default:
+ throw new Error(`${assetProxyId} not a supported assetProxyId`);
+ }
+}
diff --git a/packages/pipeline/src/utils/transformers/index.ts b/packages/pipeline/src/utils/transformers/index.ts
index 232c1c5de..31a4c9223 100644
--- a/packages/pipeline/src/utils/transformers/index.ts
+++ b/packages/pipeline/src/utils/transformers/index.ts
@@ -1,2 +1,3 @@
export * from './big_number';
export * from './number_to_bigint';
+export * from './asset_proxy_id_types';
diff --git a/packages/pipeline/src/utils/transformers/number_to_bigint.ts b/packages/pipeline/src/utils/transformers/number_to_bigint.ts
index 85560c1f0..9736d7c18 100644
--- a/packages/pipeline/src/utils/transformers/number_to_bigint.ts
+++ b/packages/pipeline/src/utils/transformers/number_to_bigint.ts
@@ -9,8 +9,12 @@ const decimalRadix = 10;
// https://github.com/typeorm/typeorm/issues/2400 for more information.
export class NumberToBigIntTransformer implements ValueTransformer {
// tslint:disable-next-line:prefer-function-over-method
- public to(value: number): string {
- return value.toString();
+ public to(value: number): string | null {
+ if (value === null || value === undefined) {
+ return null;
+ } else {
+ return value.toString();
+ }
}
// tslint:disable-next-line:prefer-function-over-method
diff --git a/packages/pipeline/test/entities/copper_test.ts b/packages/pipeline/test/entities/copper_test.ts
new file mode 100644
index 000000000..2543364e6
--- /dev/null
+++ b/packages/pipeline/test/entities/copper_test.ts
@@ -0,0 +1,54 @@
+import 'mocha';
+import 'reflect-metadata';
+
+import {
+ CopperActivity,
+ CopperActivityType,
+ CopperCustomField,
+ CopperLead,
+ CopperOpportunity,
+} from '../../src/entities';
+import { createDbConnectionOnceAsync } from '../db_setup';
+import {
+ ParsedActivities,
+ ParsedActivityTypes,
+ ParsedCustomFields,
+ ParsedLeads,
+ ParsedOpportunities,
+} from '../fixtures/copper/parsed_entities';
+import { chaiSetup } from '../utils/chai_setup';
+
+import { testSaveAndFindEntityAsync } from './util';
+
+chaiSetup.configure();
+
+describe('Copper entities', () => {
+ describe('save and find', async () => {
+ it('Copper lead', async () => {
+ const connection = await createDbConnectionOnceAsync();
+ const repository = connection.getRepository(CopperLead);
+ ParsedLeads.forEach(async entity => testSaveAndFindEntityAsync(repository, entity));
+ });
+ it('Copper activity', async () => {
+ const connection = await createDbConnectionOnceAsync();
+ const repository = connection.getRepository(CopperActivity);
+ ParsedActivities.forEach(async entity => testSaveAndFindEntityAsync(repository, entity));
+ });
+ // searching on jsonb fields is broken in typeorm
+ it.skip('Copper opportunity', async () => {
+ const connection = await createDbConnectionOnceAsync();
+ const repository = connection.getRepository(CopperOpportunity);
+ ParsedOpportunities.forEach(async entity => testSaveAndFindEntityAsync(repository, entity));
+ });
+ it('Copper activity type', async () => {
+ const connection = await createDbConnectionOnceAsync();
+ const repository = connection.getRepository(CopperActivityType);
+ ParsedActivityTypes.forEach(async entity => testSaveAndFindEntityAsync(repository, entity));
+ });
+ it('Copper custom field', async () => {
+ const connection = await createDbConnectionOnceAsync();
+ const repository = connection.getRepository(CopperCustomField);
+ ParsedCustomFields.forEach(async entity => testSaveAndFindEntityAsync(repository, entity));
+ });
+ });
+});
diff --git a/packages/pipeline/test/entities/util.ts b/packages/pipeline/test/entities/util.ts
index 043a3b15d..42df23a4a 100644
--- a/packages/pipeline/test/entities/util.ts
+++ b/packages/pipeline/test/entities/util.ts
@@ -15,9 +15,9 @@ const expect = chai.expect;
* @param entity An instance of a TypeORM entity which will be saved/retrieved from the database.
*/
export async function testSaveAndFindEntityAsync<T>(repository: Repository<T>, entity: T): Promise<void> {
- // Note(albrow): We are forced to use an 'as any' hack here because
+ // Note(albrow): We are forced to use an 'any' hack here because
// TypeScript complains about stack depth when checking the types.
- await repository.save(entity as any);
+ await repository.save<any>(entity);
const gotEntity = await repository.findOneOrFail({
where: entity,
});
diff --git a/packages/pipeline/test/fixtures/copper/api_v1_activity_types.json b/packages/pipeline/test/fixtures/copper/api_v1_activity_types.json
new file mode 100644
index 000000000..dbd39c31b
--- /dev/null
+++ b/packages/pipeline/test/fixtures/copper/api_v1_activity_types.json
@@ -0,0 +1,24 @@
+{
+ "user": [
+ { "id": 0, "category": "user", "name": "Note", "is_disabled": false, "count_as_interaction": false },
+ { "id": 660496, "category": "user", "name": "To Do", "is_disabled": false, "count_as_interaction": false },
+ { "id": 660495, "category": "user", "name": "Meeting", "is_disabled": false, "count_as_interaction": true },
+ { "id": 660494, "category": "user", "name": "Phone Call", "is_disabled": false, "count_as_interaction": true }
+ ],
+ "system": [
+ {
+ "id": 1,
+ "category": "system",
+ "name": "Property Changed",
+ "is_disabled": false,
+ "count_as_interaction": false
+ },
+ {
+ "id": 3,
+ "category": "system",
+ "name": "Pipeline Stage Changed",
+ "is_disabled": false,
+ "count_as_interaction": false
+ }
+ ]
+}
diff --git a/packages/pipeline/test/fixtures/copper/api_v1_activity_types.ts b/packages/pipeline/test/fixtures/copper/api_v1_activity_types.ts
new file mode 100644
index 000000000..fd2d62a6c
--- /dev/null
+++ b/packages/pipeline/test/fixtures/copper/api_v1_activity_types.ts
@@ -0,0 +1,16 @@
+import { CopperActivityType } from '../../../src/entities';
+const ParsedActivityTypes: CopperActivityType[] = [
+ { id: 0, name: 'Note', category: 'user', isDisabled: false, countAsInteraction: false },
+ { id: 660496, name: 'To Do', category: 'user', isDisabled: false, countAsInteraction: false },
+ { id: 660495, name: 'Meeting', category: 'user', isDisabled: false, countAsInteraction: true },
+ { id: 660494, name: 'Phone Call', category: 'user', isDisabled: false, countAsInteraction: true },
+ { id: 1, name: 'Property Changed', category: 'system', isDisabled: false, countAsInteraction: false },
+ {
+ id: 3,
+ name: 'Pipeline Stage Changed',
+ category: 'system',
+ isDisabled: false,
+ countAsInteraction: false,
+ },
+];
+export { ParsedActivityTypes };
diff --git a/packages/pipeline/test/fixtures/copper/api_v1_custom_field_definitions.json b/packages/pipeline/test/fixtures/copper/api_v1_custom_field_definitions.json
new file mode 100644
index 000000000..c6665cb0f
--- /dev/null
+++ b/packages/pipeline/test/fixtures/copper/api_v1_custom_field_definitions.json
@@ -0,0 +1,38 @@
+[
+ {
+ "id": 261066,
+ "name": "Integration Type",
+ "canonical_name": null,
+ "data_type": "MultiSelect",
+ "available_on": ["opportunity", "company", "person"],
+ "options": [
+ { "id": 394020, "name": "Strategic Relationship", "rank": 7 },
+ { "id": 394013, "name": "ERC-20 Exchange", "rank": 0 },
+ { "id": 394014, "name": "ERC-721 Marketplace", "rank": 1 },
+ { "id": 394015, "name": "Trade Widget", "rank": 2 },
+ { "id": 394016, "name": "Prediction Market Exchange", "rank": 3 },
+ { "id": 394017, "name": "Security Token Exchange", "rank": 4 },
+ { "id": 394018, "name": "Complementary Company", "rank": 5 },
+ { "id": 394019, "name": "Service Provider", "rank": 6 }
+ ]
+ },
+ {
+ "id": 261067,
+ "name": "Company Type",
+ "canonical_name": null,
+ "data_type": "Dropdown",
+ "available_on": ["company", "opportunity", "person"],
+ "options": [
+ { "id": 394129, "name": "Market Maker", "rank": 6 },
+ { "id": 394130, "name": "Events", "rank": 2 },
+ { "id": 394023, "name": "Exchange", "rank": 3 },
+ { "id": 394024, "name": "Investor", "rank": 5 },
+ { "id": 394026, "name": "Service Provider", "rank": 8 },
+ { "id": 394027, "name": "Wallet", "rank": 9 },
+ { "id": 394134, "name": "Game", "rank": 4 },
+ { "id": 394025, "name": "OTC", "rank": 7 },
+ { "id": 394021, "name": "Blockchain/Protocol", "rank": 0 },
+ { "id": 394022, "name": "dApp", "rank": 1 }
+ ]
+ }
+]
diff --git a/packages/pipeline/test/fixtures/copper/api_v1_custom_field_definitions.ts b/packages/pipeline/test/fixtures/copper/api_v1_custom_field_definitions.ts
new file mode 100644
index 000000000..a44bbd2c3
--- /dev/null
+++ b/packages/pipeline/test/fixtures/copper/api_v1_custom_field_definitions.ts
@@ -0,0 +1,39 @@
+import { CopperCustomField } from '../../../src/entities';
+const ParsedCustomFields: CopperCustomField[] = [
+ {
+ id: 394020,
+ name: 'Strategic Relationship',
+ dataType: 'Integration Type',
+ fieldType: 'option',
+ },
+ { id: 394013, name: 'ERC-20 Exchange', dataType: 'Integration Type', fieldType: 'option' },
+ { id: 394014, name: 'ERC-721 Marketplace', dataType: 'Integration Type', fieldType: 'option' },
+ { id: 394015, name: 'Trade Widget', dataType: 'Integration Type', fieldType: 'option' },
+ {
+ id: 394016,
+ name: 'Prediction Market Exchange',
+ dataType: 'Integration Type',
+ fieldType: 'option',
+ },
+ {
+ id: 394017,
+ name: 'Security Token Exchange',
+ dataType: 'Integration Type',
+ fieldType: 'option',
+ },
+ { id: 394018, name: 'Complementary Company', dataType: 'Integration Type', fieldType: 'option' },
+ { id: 394019, name: 'Service Provider', dataType: 'Integration Type', fieldType: 'option' },
+ { id: 261066, name: 'Integration Type', dataType: 'MultiSelect' },
+ { id: 394129, name: 'Market Maker', dataType: 'Company Type', fieldType: 'option' },
+ { id: 394130, name: 'Events', dataType: 'Company Type', fieldType: 'option' },
+ { id: 394023, name: 'Exchange', dataType: 'Company Type', fieldType: 'option' },
+ { id: 394024, name: 'Investor', dataType: 'Company Type', fieldType: 'option' },
+ { id: 394026, name: 'Service Provider', dataType: 'Company Type', fieldType: 'option' },
+ { id: 394027, name: 'Wallet', dataType: 'Company Type', fieldType: 'option' },
+ { id: 394134, name: 'Game', dataType: 'Company Type', fieldType: 'option' },
+ { id: 394025, name: 'OTC', dataType: 'Company Type', fieldType: 'option' },
+ { id: 394021, name: 'Blockchain/Protocol', dataType: 'Company Type', fieldType: 'option' },
+ { id: 394022, name: 'dApp', dataType: 'Company Type', fieldType: 'option' },
+ { id: 261067, name: 'Company Type', dataType: 'Dropdown' },
+];
+export { ParsedCustomFields };
diff --git a/packages/pipeline/test/fixtures/copper/api_v1_list_activities.json b/packages/pipeline/test/fixtures/copper/api_v1_list_activities.json
new file mode 100644
index 000000000..a726111ac
--- /dev/null
+++ b/packages/pipeline/test/fixtures/copper/api_v1_list_activities.json
@@ -0,0 +1,242 @@
+[
+ {
+ "id": 5015299552,
+ "parent": { "id": 14667512, "type": "opportunity" },
+ "type": { "id": 3, "category": "system", "name": "Stage Change" },
+ "user_id": 680302,
+ "details": "blah blah",
+ "activity_date": 1545329595,
+ "old_value": { "id": 2392929, "name": "Evaluation" },
+ "new_value": { "id": 2392931, "name": "Integration Started" },
+ "date_created": 1545329595,
+ "date_modified": 1545329595
+ },
+ {
+ "id": 5010214065,
+ "parent": { "id": 14978865, "type": "opportunity" },
+ "type": { "id": 3, "category": "system", "name": "Stage Change" },
+ "user_id": 680302,
+ "details": "blah blah",
+ "activity_date": 1545245706,
+ "old_value": { "id": 2392928, "name": "Intro" },
+ "new_value": { "id": 2392929, "name": "Evaluation" },
+ "date_created": 1545245706,
+ "date_modified": 1545245706
+ },
+ {
+ "id": 5006149111,
+ "parent": { "id": 70430977, "type": "person" },
+ "type": { "id": 660495, "category": "user" },
+ "user_id": 680302,
+ "details": "blah blah",
+ "activity_date": 1545166908,
+ "old_value": null,
+ "new_value": null,
+ "date_created": 1545168280,
+ "date_modified": 1545166908
+ },
+ {
+ "id": 5005314622,
+ "parent": { "id": 27778968, "type": "company" },
+ "type": { "id": 660495, "category": "user" },
+ "user_id": 680302,
+ "details": "blah blah",
+ "activity_date": 1545080504,
+ "old_value": null,
+ "new_value": null,
+ "date_created": 1545160479,
+ "date_modified": 1545080504
+ },
+ {
+ "id": 5000006802,
+ "parent": { "id": 14956518, "type": "opportunity" },
+ "type": { "id": 660495, "category": "user" },
+ "user_id": 680302,
+ "details": "blah blah",
+ "activity_date": 1545071374,
+ "old_value": null,
+ "new_value": null,
+ "date_created": 1545071500,
+ "date_modified": 1545071374
+ },
+ {
+ "id": 4985504199,
+ "parent": { "id": 14912790, "type": "opportunity" },
+ "type": { "id": 660495, "category": "user" },
+ "user_id": 680302,
+ "details": "blah blah",
+ "activity_date": 1544644058,
+ "old_value": null,
+ "new_value": null,
+ "date_created": 1544644661,
+ "date_modified": 1544644058
+ },
+ {
+ "id": 4985456147,
+ "parent": { "id": 14912790, "type": "opportunity" },
+ "type": { "id": 660495, "category": "user" },
+ "user_id": 680302,
+ "details": "blah blah",
+ "activity_date": 1544644048,
+ "old_value": null,
+ "new_value": null,
+ "date_created": 1544644053,
+ "date_modified": 1544644048
+ },
+ {
+ "id": 4980975996,
+ "parent": { "id": 14902828, "type": "opportunity" },
+ "type": { "id": 660495, "category": "user" },
+ "user_id": 680302,
+ "details": "blah blah",
+ "activity_date": 1544563171,
+ "old_value": null,
+ "new_value": null,
+ "date_created": 1544563224,
+ "date_modified": 1544563171
+ },
+ {
+ "id": 4980910331,
+ "parent": { "id": 14902828, "type": "opportunity" },
+ "type": { "id": 3, "category": "system", "name": "Stage Change" },
+ "user_id": 680302,
+ "details": "blah blah",
+ "activity_date": 1544562495,
+ "old_value": { "id": 2392928, "name": "Intro" },
+ "new_value": { "id": 2392931, "name": "Integration Started" },
+ "date_created": 1544562495,
+ "date_modified": 1544562495
+ },
+ {
+ "id": 4980872220,
+ "parent": { "id": 14888910, "type": "opportunity" },
+ "type": { "id": 660495, "category": "user" },
+ "user_id": 680302,
+ "details": "blah blah",
+ "activity_date": 1544559279,
+ "old_value": null,
+ "new_value": null,
+ "date_created": 1544562118,
+ "date_modified": 1544559279
+ },
+ {
+ "id": 4980508097,
+ "parent": { "id": 14050167, "type": "opportunity" },
+ "type": { "id": 1, "category": "system", "name": "Status Change" },
+ "user_id": 680302,
+ "details": "blah blah",
+ "activity_date": 1544558077,
+ "old_value": "Open",
+ "new_value": "Won",
+ "date_created": 1544558077,
+ "date_modified": 1544558077
+ },
+ {
+ "id": 4980508095,
+ "parent": { "id": 66538237, "type": "person" },
+ "type": { "id": 1, "category": "system" },
+ "user_id": 680302,
+ "details": "blah blah",
+ "activity_date": 1544558077,
+ "old_value": null,
+ "new_value": null,
+ "date_created": 1544558077,
+ "date_modified": 1544558077
+ },
+ {
+ "id": 4980508092,
+ "parent": { "id": 27779020, "type": "company" },
+ "type": { "id": 1, "category": "system" },
+ "user_id": 680302,
+ "details": "blah blah",
+ "activity_date": 1544558077,
+ "old_value": null,
+ "new_value": null,
+ "date_created": 1544558077,
+ "date_modified": 1544558077
+ },
+ {
+ "id": 4980507507,
+ "parent": { "id": 14050167, "type": "opportunity" },
+ "type": { "id": 3, "category": "system", "name": "Stage Change" },
+ "user_id": 680302,
+ "details": "blah blah",
+ "activity_date": 1544558071,
+ "old_value": { "id": 2392931, "name": "Integration Started" },
+ "new_value": { "id": 2405442, "name": "Integration Complete" },
+ "date_created": 1544558071,
+ "date_modified": 1544558071
+ },
+ {
+ "id": 4980479684,
+ "parent": { "id": 14901232, "type": "opportunity" },
+ "type": { "id": 3, "category": "system", "name": "Stage Change" },
+ "user_id": 680302,
+ "details": "blah blah",
+ "activity_date": 1544557777,
+ "old_value": { "id": 2392928, "name": "Intro" },
+ "new_value": { "id": 2392929, "name": "Evaluation" },
+ "date_created": 1544557777,
+ "date_modified": 1544557777
+ },
+ {
+ "id": 4980327164,
+ "parent": { "id": 14901232, "type": "opportunity" },
+ "type": { "id": 660495, "category": "user" },
+ "user_id": 680302,
+ "details": "blah blah",
+ "activity_date": 1544554864,
+ "old_value": null,
+ "new_value": null,
+ "date_created": 1544556132,
+ "date_modified": 1544554864
+ },
+ {
+ "id": 4975270470,
+ "parent": { "id": 14888744, "type": "opportunity" },
+ "type": { "id": 3, "category": "system", "name": "Stage Change" },
+ "user_id": 680302,
+ "details": "blah blah",
+ "activity_date": 1544469501,
+ "old_value": { "id": 2392928, "name": "Intro" },
+ "new_value": { "id": 2392931, "name": "Integration Started" },
+ "date_created": 1544469501,
+ "date_modified": 1544469501
+ },
+ {
+ "id": 4975255523,
+ "parent": { "id": 64713448, "type": "person" },
+ "type": { "id": 1, "category": "system" },
+ "user_id": 680302,
+ "details": "blah blah",
+ "activity_date": 1544469389,
+ "old_value": null,
+ "new_value": null,
+ "date_created": 1544469389,
+ "date_modified": 1544469389
+ },
+ {
+ "id": 4975255519,
+ "parent": { "id": 13735617, "type": "opportunity" },
+ "type": { "id": 1, "category": "system", "name": "Status Change" },
+ "user_id": 680302,
+ "details": "blah blah",
+ "activity_date": 1544469388,
+ "old_value": "Open",
+ "new_value": "Won",
+ "date_created": 1544469388,
+ "date_modified": 1544469388
+ },
+ {
+ "id": 4975255514,
+ "parent": { "id": 27778968, "type": "company" },
+ "type": { "id": 1, "category": "system" },
+ "user_id": 680302,
+ "details": "blah blah",
+ "activity_date": 1544469388,
+ "old_value": null,
+ "new_value": null,
+ "date_created": 1544469388,
+ "date_modified": 1544469388
+ }
+]
diff --git a/packages/pipeline/test/fixtures/copper/api_v1_list_activities.ts b/packages/pipeline/test/fixtures/copper/api_v1_list_activities.ts
new file mode 100644
index 000000000..51ee9ced3
--- /dev/null
+++ b/packages/pipeline/test/fixtures/copper/api_v1_list_activities.ts
@@ -0,0 +1,305 @@
+import { CopperActivity } from '../../../src/entities';
+
+const ParsedActivities: CopperActivity[] = [
+ {
+ id: 5015299552,
+ parentId: 14667512,
+ parentType: 'opportunity',
+ typeId: 3,
+ typeCategory: 'system',
+ typeName: 'Stage Change',
+ userId: 680302,
+ dateCreated: 1545329595000,
+ dateModified: 1545329595000,
+ oldValueId: 2392929,
+ oldValueName: 'Evaluation',
+ newValueId: 2392931,
+ newValueName: 'Integration Started',
+ },
+ {
+ id: 5010214065,
+ parentId: 14978865,
+ parentType: 'opportunity',
+ typeId: 3,
+ typeCategory: 'system',
+ typeName: 'Stage Change',
+ userId: 680302,
+ dateCreated: 1545245706000,
+ dateModified: 1545245706000,
+ oldValueId: 2392928,
+ oldValueName: 'Intro',
+ newValueId: 2392929,
+ newValueName: 'Evaluation',
+ },
+ {
+ id: 5006149111,
+ parentId: 70430977,
+ parentType: 'person',
+ typeId: 660495,
+ typeCategory: 'user',
+ typeName: undefined,
+ userId: 680302,
+ dateCreated: 1545168280000,
+ dateModified: 1545166908000,
+ oldValueId: undefined,
+ oldValueName: undefined,
+ newValueId: undefined,
+ newValueName: undefined,
+ },
+ {
+ id: 5005314622,
+ parentId: 27778968,
+ parentType: 'company',
+ typeId: 660495,
+ typeCategory: 'user',
+ typeName: undefined,
+ userId: 680302,
+ dateCreated: 1545160479000,
+ dateModified: 1545080504000,
+ oldValueId: undefined,
+ oldValueName: undefined,
+ newValueId: undefined,
+ newValueName: undefined,
+ },
+ {
+ id: 5000006802,
+ parentId: 14956518,
+ parentType: 'opportunity',
+ typeId: 660495,
+ typeCategory: 'user',
+ typeName: undefined,
+ userId: 680302,
+ dateCreated: 1545071500000,
+ dateModified: 1545071374000,
+ oldValueId: undefined,
+ oldValueName: undefined,
+ newValueId: undefined,
+ newValueName: undefined,
+ },
+ {
+ id: 4985504199,
+ parentId: 14912790,
+ parentType: 'opportunity',
+ typeId: 660495,
+ typeCategory: 'user',
+ typeName: undefined,
+ userId: 680302,
+ dateCreated: 1544644661000,
+ dateModified: 1544644058000,
+ oldValueId: undefined,
+ oldValueName: undefined,
+ newValueId: undefined,
+ newValueName: undefined,
+ },
+ {
+ id: 4985456147,
+ parentId: 14912790,
+ parentType: 'opportunity',
+ typeId: 660495,
+ typeCategory: 'user',
+ typeName: undefined,
+ userId: 680302,
+ dateCreated: 1544644053000,
+ dateModified: 1544644048000,
+ oldValueId: undefined,
+ oldValueName: undefined,
+ newValueId: undefined,
+ newValueName: undefined,
+ },
+ {
+ id: 4980975996,
+ parentId: 14902828,
+ parentType: 'opportunity',
+ typeId: 660495,
+ typeCategory: 'user',
+ typeName: undefined,
+ userId: 680302,
+ dateCreated: 1544563224000,
+ dateModified: 1544563171000,
+ oldValueId: undefined,
+ oldValueName: undefined,
+ newValueId: undefined,
+ newValueName: undefined,
+ },
+ {
+ id: 4980910331,
+ parentId: 14902828,
+ parentType: 'opportunity',
+ typeId: 3,
+ typeCategory: 'system',
+ typeName: 'Stage Change',
+ userId: 680302,
+ dateCreated: 1544562495000,
+ dateModified: 1544562495000,
+ oldValueId: 2392928,
+ oldValueName: 'Intro',
+ newValueId: 2392931,
+ newValueName: 'Integration Started',
+ },
+ {
+ id: 4980872220,
+ parentId: 14888910,
+ parentType: 'opportunity',
+ typeId: 660495,
+ typeCategory: 'user',
+ typeName: undefined,
+ userId: 680302,
+ dateCreated: 1544562118000,
+ dateModified: 1544559279000,
+ oldValueId: undefined,
+ oldValueName: undefined,
+ newValueId: undefined,
+ newValueName: undefined,
+ },
+ {
+ id: 4980508097,
+ parentId: 14050167,
+ parentType: 'opportunity',
+ typeId: 1,
+ typeCategory: 'system',
+ typeName: 'Status Change',
+ userId: 680302,
+ dateCreated: 1544558077000,
+ dateModified: 1544558077000,
+ oldValueId: undefined,
+ oldValueName: undefined,
+ newValueId: undefined,
+ newValueName: undefined,
+ },
+ {
+ id: 4980508095,
+ parentId: 66538237,
+ parentType: 'person',
+ typeId: 1,
+ typeCategory: 'system',
+ typeName: undefined,
+ userId: 680302,
+ dateCreated: 1544558077000,
+ dateModified: 1544558077000,
+ oldValueId: undefined,
+ oldValueName: undefined,
+ newValueId: undefined,
+ newValueName: undefined,
+ },
+ {
+ id: 4980508092,
+ parentId: 27779020,
+ parentType: 'company',
+ typeId: 1,
+ typeCategory: 'system',
+ typeName: undefined,
+ userId: 680302,
+ dateCreated: 1544558077000,
+ dateModified: 1544558077000,
+ oldValueId: undefined,
+ oldValueName: undefined,
+ newValueId: undefined,
+ newValueName: undefined,
+ },
+ {
+ id: 4980507507,
+ parentId: 14050167,
+ parentType: 'opportunity',
+ typeId: 3,
+ typeCategory: 'system',
+ typeName: 'Stage Change',
+ userId: 680302,
+ dateCreated: 1544558071000,
+ dateModified: 1544558071000,
+ oldValueId: 2392931,
+ oldValueName: 'Integration Started',
+ newValueId: 2405442,
+ newValueName: 'Integration Complete',
+ },
+ {
+ id: 4980479684,
+ parentId: 14901232,
+ parentType: 'opportunity',
+ typeId: 3,
+ typeCategory: 'system',
+ typeName: 'Stage Change',
+ userId: 680302,
+ dateCreated: 1544557777000,
+ dateModified: 1544557777000,
+ oldValueId: 2392928,
+ oldValueName: 'Intro',
+ newValueId: 2392929,
+ newValueName: 'Evaluation',
+ },
+ {
+ id: 4980327164,
+ parentId: 14901232,
+ parentType: 'opportunity',
+ typeId: 660495,
+ typeCategory: 'user',
+ typeName: undefined,
+ userId: 680302,
+ dateCreated: 1544556132000,
+ dateModified: 1544554864000,
+ oldValueId: undefined,
+ oldValueName: undefined,
+ newValueId: undefined,
+ newValueName: undefined,
+ },
+ {
+ id: 4975270470,
+ parentId: 14888744,
+ parentType: 'opportunity',
+ typeId: 3,
+ typeCategory: 'system',
+ typeName: 'Stage Change',
+ userId: 680302,
+ dateCreated: 1544469501000,
+ dateModified: 1544469501000,
+ oldValueId: 2392928,
+ oldValueName: 'Intro',
+ newValueId: 2392931,
+ newValueName: 'Integration Started',
+ },
+ {
+ id: 4975255523,
+ parentId: 64713448,
+ parentType: 'person',
+ typeId: 1,
+ typeCategory: 'system',
+ typeName: undefined,
+ userId: 680302,
+ dateCreated: 1544469389000,
+ dateModified: 1544469389000,
+ oldValueId: undefined,
+ oldValueName: undefined,
+ newValueId: undefined,
+ newValueName: undefined,
+ },
+ {
+ id: 4975255519,
+ parentId: 13735617,
+ parentType: 'opportunity',
+ typeId: 1,
+ typeCategory: 'system',
+ typeName: 'Status Change',
+ userId: 680302,
+ dateCreated: 1544469388000,
+ dateModified: 1544469388000,
+ oldValueId: undefined,
+ oldValueName: undefined,
+ newValueId: undefined,
+ newValueName: undefined,
+ },
+ {
+ id: 4975255514,
+ parentId: 27778968,
+ parentType: 'company',
+ typeId: 1,
+ typeCategory: 'system',
+ typeName: undefined,
+ userId: 680302,
+ dateCreated: 1544469388000,
+ dateModified: 1544469388000,
+ oldValueId: undefined,
+ oldValueName: undefined,
+ newValueId: undefined,
+ newValueName: undefined,
+ },
+];
+export { ParsedActivities };
diff --git a/packages/pipeline/test/fixtures/copper/api_v1_list_leads.json b/packages/pipeline/test/fixtures/copper/api_v1_list_leads.json
new file mode 100644
index 000000000..5223976f9
--- /dev/null
+++ b/packages/pipeline/test/fixtures/copper/api_v1_list_leads.json
@@ -0,0 +1,583 @@
+[
+ {
+ "id": 9150547,
+ "name": "My Contact",
+ "prefix": null,
+ "first_name": "My",
+ "last_name": "Contact",
+ "middle_name": null,
+ "suffix": null,
+ "address": null,
+ "assignee_id": null,
+ "company_name": null,
+ "customer_source_id": null,
+ "details": null,
+ "email": {
+ "email": "mycontact@noemail.com",
+ "category": "work"
+ },
+ "interaction_count": 0,
+ "monetary_value": null,
+ "socials": [],
+ "status": "New",
+ "status_id": 208231,
+ "tags": [],
+ "title": null,
+ "websites": [],
+ "phone_numbers": [],
+ "custom_fields": [
+ {
+ "custom_field_definition_id": 100764,
+ "value": null
+ },
+ {
+ "custom_field_definition_id": 103481,
+ "value": null
+ }
+ ],
+ "date_created": 1490045162,
+ "date_modified": 1490045162
+ },
+ {
+ "id": 9150552,
+ "name": "My Contact",
+ "prefix": null,
+ "first_name": "My",
+ "last_name": "Contact",
+ "middle_name": null,
+ "suffix": null,
+ "address": null,
+ "assignee_id": null,
+ "company_name": null,
+ "customer_source_id": null,
+ "details": null,
+ "email": null,
+ "interaction_count": 0,
+ "monetary_value": null,
+ "socials": [],
+ "status": "New",
+ "status_id": 208231,
+ "tags": [],
+ "title": null,
+ "websites": [],
+ "phone_numbers": [
+ {
+ "number": "415-123-45678",
+ "category": "mobile"
+ }
+ ],
+ "custom_fields": [
+ {
+ "custom_field_definition_id": 100764,
+ "value": null
+ },
+ {
+ "custom_field_definition_id": 103481,
+ "value": null
+ }
+ ],
+ "date_created": 1490045237,
+ "date_modified": 1490045237
+ },
+ {
+ "id": 9150578,
+ "name": "My Contact",
+ "prefix": null,
+ "first_name": "My",
+ "last_name": "Contact",
+ "middle_name": null,
+ "suffix": null,
+ "address": null,
+ "assignee_id": null,
+ "company_name": null,
+ "customer_source_id": null,
+ "details": null,
+ "email": null,
+ "interaction_count": 0,
+ "monetary_value": null,
+ "socials": [],
+ "status": "New",
+ "status_id": 208231,
+ "tags": [],
+ "title": null,
+ "websites": [],
+ "phone_numbers": [
+ {
+ "number": "415-123-45678",
+ "category": "mobile"
+ }
+ ],
+ "custom_fields": [
+ {
+ "custom_field_definition_id": 100764,
+ "value": null
+ },
+ {
+ "custom_field_definition_id": 103481,
+ "value": null
+ }
+ ],
+ "date_created": 1490045279,
+ "date_modified": 1490045279
+ },
+ {
+ "id": 8982554,
+ "name": "My Lead",
+ "prefix": null,
+ "first_name": "My",
+ "last_name": "Lead",
+ "middle_name": null,
+ "suffix": null,
+ "address": null,
+ "assignee_id": null,
+ "company_name": null,
+ "customer_source_id": null,
+ "details": null,
+ "email": {
+ "email": "mylead@noemail.com",
+ "category": "work"
+ },
+ "interaction_count": 0,
+ "monetary_value": null,
+ "socials": [],
+ "status": "New",
+ "status_id": 208231,
+ "tags": [],
+ "title": null,
+ "websites": [],
+ "phone_numbers": [],
+ "custom_fields": [
+ {
+ "custom_field_definition_id": 100764,
+ "value": null
+ },
+ {
+ "custom_field_definition_id": 103481,
+ "value": null
+ }
+ ],
+ "date_created": 1489528899,
+ "date_modified": 1489528899
+ },
+ {
+ "id": 8982702,
+ "name": "My Lead",
+ "prefix": null,
+ "first_name": "My",
+ "last_name": "Lead",
+ "middle_name": null,
+ "suffix": null,
+ "address": null,
+ "assignee_id": null,
+ "company_name": null,
+ "customer_source_id": null,
+ "details": null,
+ "email": {
+ "email": "mylead@gmail.test",
+ "category": "work"
+ },
+ "interaction_count": 0,
+ "monetary_value": null,
+ "socials": [],
+ "status": "New",
+ "status_id": 208231,
+ "tags": [],
+ "title": null,
+ "websites": [],
+ "phone_numbers": [],
+ "custom_fields": [
+ {
+ "custom_field_definition_id": 100764,
+ "value": null
+ },
+ {
+ "custom_field_definition_id": 103481,
+ "value": null
+ }
+ ],
+ "date_created": 1489531171,
+ "date_modified": 1489531171
+ },
+ {
+ "id": 9094361,
+ "name": "My Lead",
+ "prefix": null,
+ "first_name": "My",
+ "last_name": "Lead",
+ "middle_name": null,
+ "suffix": null,
+ "address": null,
+ "assignee_id": null,
+ "company_name": null,
+ "customer_source_id": null,
+ "details": null,
+ "email": {
+ "email": "mylead@noemail.com",
+ "category": "work"
+ },
+ "interaction_count": 0,
+ "monetary_value": null,
+ "socials": [],
+ "status": "New",
+ "status_id": 208231,
+ "tags": [],
+ "title": null,
+ "websites": [],
+ "phone_numbers": [],
+ "custom_fields": [
+ {
+ "custom_field_definition_id": 100764,
+ "value": null
+ },
+ {
+ "custom_field_definition_id": 103481,
+ "value": null
+ }
+ ],
+ "date_created": 1489791225,
+ "date_modified": 1489791225
+ },
+ {
+ "id": 9094364,
+ "name": "My Lead",
+ "prefix": null,
+ "first_name": "My",
+ "last_name": "Lead",
+ "middle_name": null,
+ "suffix": null,
+ "address": null,
+ "assignee_id": null,
+ "company_name": null,
+ "customer_source_id": null,
+ "details": null,
+ "email": {
+ "email": "mylead@noemail.com",
+ "category": "work"
+ },
+ "interaction_count": 0,
+ "monetary_value": null,
+ "socials": [],
+ "status": "New",
+ "status_id": 208231,
+ "tags": [],
+ "title": null,
+ "websites": [],
+ "phone_numbers": [],
+ "custom_fields": [
+ {
+ "custom_field_definition_id": 100764,
+ "value": "123456789012345678901234567890"
+ },
+ {
+ "custom_field_definition_id": 103481,
+ "value": "123456789012345678901234567890"
+ }
+ ],
+ "date_created": 1489791283,
+ "date_modified": 1489791283
+ },
+ {
+ "id": 9094371,
+ "name": "My Lead",
+ "prefix": null,
+ "first_name": "My",
+ "last_name": "Lead",
+ "middle_name": null,
+ "suffix": null,
+ "address": null,
+ "assignee_id": null,
+ "company_name": null,
+ "customer_source_id": null,
+ "details": null,
+ "email": {
+ "email": "mylead@noemail.com",
+ "category": "work"
+ },
+ "interaction_count": 0,
+ "monetary_value": null,
+ "socials": [],
+ "status": "New",
+ "status_id": 208231,
+ "tags": [],
+ "title": null,
+ "websites": [],
+ "phone_numbers": [],
+ "custom_fields": [
+ {
+ "custom_field_definition_id": 100764,
+ "value":
+ "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------"
+ },
+ {
+ "custom_field_definition_id": 103481,
+ "value": "123456789012345678901234567890"
+ }
+ ],
+ "date_created": 1489791417,
+ "date_modified": 1489791417
+ },
+ {
+ "id": 9094372,
+ "name": "My Lead",
+ "prefix": null,
+ "first_name": "My",
+ "last_name": "Lead",
+ "middle_name": null,
+ "suffix": null,
+ "address": null,
+ "assignee_id": null,
+ "company_name": null,
+ "customer_source_id": null,
+ "details": null,
+ "email": {
+ "email": "mylead@noemail.com",
+ "category": "work"
+ },
+ "interaction_count": 0,
+ "monetary_value": null,
+ "socials": [],
+ "status": "New",
+ "status_id": 208231,
+ "tags": [],
+ "title": null,
+ "websites": [],
+ "phone_numbers": [],
+ "custom_fields": [
+ {
+ "custom_field_definition_id": 100764,
+ "value":
+ "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----"
+ },
+ {
+ "custom_field_definition_id": 103481,
+ "value": "123456789012345678901234567890"
+ }
+ ],
+ "date_created": 1489791453,
+ "date_modified": 1489791453
+ },
+ {
+ "id": 9094373,
+ "name": "My Lead",
+ "prefix": null,
+ "first_name": "My",
+ "last_name": "Lead",
+ "middle_name": null,
+ "suffix": null,
+ "address": null,
+ "assignee_id": null,
+ "company_name": null,
+ "customer_source_id": null,
+ "details": null,
+ "email": {
+ "email": "mylead@noemail.com",
+ "category": "work"
+ },
+ "interaction_count": 0,
+ "monetary_value": null,
+ "socials": [],
+ "status": "New",
+ "status_id": 208231,
+ "tags": [],
+ "title": null,
+ "websites": [],
+ "phone_numbers": [],
+ "custom_fields": [
+ {
+ "custom_field_definition_id": 100764,
+ "value":
+ "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----"
+ },
+ {
+ "custom_field_definition_id": 103481,
+ "value":
+ "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------"
+ }
+ ],
+ "date_created": 1489791470,
+ "date_modified": 1489791470
+ },
+ {
+ "id": 9094383,
+ "name": "My Lead",
+ "prefix": null,
+ "first_name": "My",
+ "last_name": "Lead",
+ "middle_name": null,
+ "suffix": null,
+ "address": null,
+ "assignee_id": null,
+ "company_name": null,
+ "customer_source_id": null,
+ "details": null,
+ "email": {
+ "email": "mylead@noemail.com",
+ "category": "work"
+ },
+ "interaction_count": 0,
+ "monetary_value": null,
+ "socials": [],
+ "status": "New",
+ "status_id": 208231,
+ "tags": [],
+ "title": null,
+ "websites": [],
+ "phone_numbers": [],
+ "custom_fields": [
+ {
+ "custom_field_definition_id": 100764,
+ "value":
+ "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----"
+ },
+ {
+ "custom_field_definition_id": 103481,
+ "value":
+ "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------"
+ }
+ ],
+ "date_created": 1489791672,
+ "date_modified": 1489791672
+ },
+ {
+ "id": 9174441,
+ "name": "My Lead",
+ "prefix": null,
+ "first_name": "My",
+ "last_name": "Lead",
+ "middle_name": null,
+ "suffix": null,
+ "address": null,
+ "assignee_id": null,
+ "company_name": null,
+ "customer_source_id": null,
+ "details": null,
+ "email": {
+ "email": "mylead@noemail.com",
+ "category": "work"
+ },
+ "interaction_count": 0,
+ "monetary_value": null,
+ "socials": [],
+ "status": "New",
+ "status_id": 208231,
+ "tags": [],
+ "title": null,
+ "websites": [],
+ "phone_numbers": [],
+ "custom_fields": [
+ {
+ "custom_field_definition_id": 100764,
+ "value": "Text fields are 255 chars or less!"
+ },
+ {
+ "custom_field_definition_id": 103481,
+ "value": "text \n text"
+ }
+ ],
+ "date_created": 1490112942,
+ "date_modified": 1490112942
+ },
+ {
+ "id": 9174443,
+ "name": "My Lead",
+ "prefix": null,
+ "first_name": "My",
+ "last_name": "Lead",
+ "middle_name": null,
+ "suffix": null,
+ "address": null,
+ "assignee_id": null,
+ "company_name": null,
+ "customer_source_id": null,
+ "details": null,
+ "email": {
+ "email": "mylead@noemail.com",
+ "category": "work"
+ },
+ "interaction_count": 0,
+ "monetary_value": null,
+ "socials": [],
+ "status": "New",
+ "status_id": 208231,
+ "tags": [],
+ "title": null,
+ "websites": [],
+ "phone_numbers": [],
+ "custom_fields": [
+ {
+ "custom_field_definition_id": 100764,
+ "value": "Text fields are 255 chars or less!"
+ },
+ {
+ "custom_field_definition_id": 103481,
+ "value": "text /n text"
+ }
+ ],
+ "date_created": 1490112953,
+ "date_modified": 1490112953
+ },
+ {
+ "id": 8894157,
+ "name": "Test Lead",
+ "prefix": null,
+ "first_name": "Test",
+ "last_name": "Lead",
+ "middle_name": null,
+ "suffix": null,
+ "address": {
+ "street": "301 Howard St Ste 600",
+ "city": "San Francisco",
+ "state": "CA",
+ "postal_code": "94105",
+ "country": "US"
+ },
+ "assignee_id": 137658,
+ "company_name": "Lead's Company",
+ "customer_source_id": 331241,
+ "details": "This is an update",
+ "email": {
+ "email": "address@workemail.com",
+ "category": "work"
+ },
+ "interaction_count": 0,
+ "monetary_value": 100,
+ "socials": [
+ {
+ "url": "facebook.com/test_lead",
+ "category": "facebook"
+ }
+ ],
+ "status": "New",
+ "status_id": 208231,
+ "tags": ["tag 1", "tag 2"],
+ "title": "Title",
+ "websites": [
+ {
+ "url": "www.workwebsite.com",
+ "category": "work"
+ }
+ ],
+ "phone_numbers": [
+ {
+ "number": "415-999-4321",
+ "category": "mobile"
+ },
+ {
+ "number": "415-555-1234",
+ "category": "work"
+ }
+ ],
+ "custom_fields": [
+ {
+ "custom_field_definition_id": 100764,
+ "value": null
+ },
+ {
+ "custom_field_definition_id": 103481,
+ "value": null
+ }
+ ],
+ "date_created": 1489018784,
+ "date_modified": 1496692911
+ }
+]
diff --git a/packages/pipeline/test/fixtures/copper/api_v1_list_leads.ts b/packages/pipeline/test/fixtures/copper/api_v1_list_leads.ts
new file mode 100644
index 000000000..b1f00cba7
--- /dev/null
+++ b/packages/pipeline/test/fixtures/copper/api_v1_list_leads.ts
@@ -0,0 +1,229 @@
+import { CopperLead } from '../../../src/entities';
+const ParsedLeads: CopperLead[] = [
+ {
+ id: 9150547,
+ name: 'My Contact',
+ firstName: 'My',
+ lastName: 'Contact',
+ middleName: undefined,
+ assigneeId: undefined,
+ companyName: undefined,
+ customerSourceId: undefined,
+ monetaryValue: undefined,
+ status: 'New',
+ statusId: 208231,
+ title: undefined,
+ dateCreated: 1490045162000,
+ dateModified: 1490045162000,
+ },
+ {
+ id: 9150552,
+ name: 'My Contact',
+ firstName: 'My',
+ lastName: 'Contact',
+ middleName: undefined,
+ assigneeId: undefined,
+ companyName: undefined,
+ customerSourceId: undefined,
+ monetaryValue: undefined,
+ status: 'New',
+ statusId: 208231,
+ title: undefined,
+ dateCreated: 1490045237000,
+ dateModified: 1490045237000,
+ },
+ {
+ id: 9150578,
+ name: 'My Contact',
+ firstName: 'My',
+ lastName: 'Contact',
+ middleName: undefined,
+ assigneeId: undefined,
+ companyName: undefined,
+ customerSourceId: undefined,
+ monetaryValue: undefined,
+ status: 'New',
+ statusId: 208231,
+ title: undefined,
+ dateCreated: 1490045279000,
+ dateModified: 1490045279000,
+ },
+ {
+ id: 8982554,
+ name: 'My Lead',
+ firstName: 'My',
+ lastName: 'Lead',
+ middleName: undefined,
+ assigneeId: undefined,
+ companyName: undefined,
+ customerSourceId: undefined,
+ monetaryValue: undefined,
+ status: 'New',
+ statusId: 208231,
+ title: undefined,
+ dateCreated: 1489528899000,
+ dateModified: 1489528899000,
+ },
+ {
+ id: 8982702,
+ name: 'My Lead',
+ firstName: 'My',
+ lastName: 'Lead',
+ middleName: undefined,
+ assigneeId: undefined,
+ companyName: undefined,
+ customerSourceId: undefined,
+ monetaryValue: undefined,
+ status: 'New',
+ statusId: 208231,
+ title: undefined,
+ dateCreated: 1489531171000,
+ dateModified: 1489531171000,
+ },
+ {
+ id: 9094361,
+ name: 'My Lead',
+ firstName: 'My',
+ lastName: 'Lead',
+ middleName: undefined,
+ assigneeId: undefined,
+ companyName: undefined,
+ customerSourceId: undefined,
+ monetaryValue: undefined,
+ status: 'New',
+ statusId: 208231,
+ title: undefined,
+ dateCreated: 1489791225000,
+ dateModified: 1489791225000,
+ },
+ {
+ id: 9094364,
+ name: 'My Lead',
+ firstName: 'My',
+ lastName: 'Lead',
+ middleName: undefined,
+ assigneeId: undefined,
+ companyName: undefined,
+ customerSourceId: undefined,
+ monetaryValue: undefined,
+ status: 'New',
+ statusId: 208231,
+ title: undefined,
+ dateCreated: 1489791283000,
+ dateModified: 1489791283000,
+ },
+ {
+ id: 9094371,
+ name: 'My Lead',
+ firstName: 'My',
+ lastName: 'Lead',
+ middleName: undefined,
+ assigneeId: undefined,
+ companyName: undefined,
+ customerSourceId: undefined,
+ monetaryValue: undefined,
+ status: 'New',
+ statusId: 208231,
+ title: undefined,
+ dateCreated: 1489791417000,
+ dateModified: 1489791417000,
+ },
+ {
+ id: 9094372,
+ name: 'My Lead',
+ firstName: 'My',
+ lastName: 'Lead',
+ middleName: undefined,
+ assigneeId: undefined,
+ companyName: undefined,
+ customerSourceId: undefined,
+ monetaryValue: undefined,
+ status: 'New',
+ statusId: 208231,
+ title: undefined,
+ dateCreated: 1489791453000,
+ dateModified: 1489791453000,
+ },
+ {
+ id: 9094373,
+ name: 'My Lead',
+ firstName: 'My',
+ lastName: 'Lead',
+ middleName: undefined,
+ assigneeId: undefined,
+ companyName: undefined,
+ customerSourceId: undefined,
+ monetaryValue: undefined,
+ status: 'New',
+ statusId: 208231,
+ title: undefined,
+ dateCreated: 1489791470000,
+ dateModified: 1489791470000,
+ },
+ {
+ id: 9094383,
+ name: 'My Lead',
+ firstName: 'My',
+ lastName: 'Lead',
+ middleName: undefined,
+ assigneeId: undefined,
+ companyName: undefined,
+ customerSourceId: undefined,
+ monetaryValue: undefined,
+ status: 'New',
+ statusId: 208231,
+ title: undefined,
+ dateCreated: 1489791672000,
+ dateModified: 1489791672000,
+ },
+ {
+ id: 9174441,
+ name: 'My Lead',
+ firstName: 'My',
+ lastName: 'Lead',
+ middleName: undefined,
+ assigneeId: undefined,
+ companyName: undefined,
+ customerSourceId: undefined,
+ monetaryValue: undefined,
+ status: 'New',
+ statusId: 208231,
+ title: undefined,
+ dateCreated: 1490112942000,
+ dateModified: 1490112942000,
+ },
+ {
+ id: 9174443,
+ name: 'My Lead',
+ firstName: 'My',
+ lastName: 'Lead',
+ middleName: undefined,
+ assigneeId: undefined,
+ companyName: undefined,
+ customerSourceId: undefined,
+ monetaryValue: undefined,
+ status: 'New',
+ statusId: 208231,
+ title: undefined,
+ dateCreated: 1490112953000,
+ dateModified: 1490112953000,
+ },
+ {
+ id: 8894157,
+ name: 'Test Lead',
+ firstName: 'Test',
+ lastName: 'Lead',
+ middleName: undefined,
+ assigneeId: 137658,
+ companyName: "Lead's Company",
+ customerSourceId: 331241,
+ monetaryValue: 100,
+ status: 'New',
+ statusId: 208231,
+ title: 'Title',
+ dateCreated: 1489018784000,
+ dateModified: 1496692911000,
+ },
+];
+
+export { ParsedLeads };
diff --git a/packages/pipeline/test/fixtures/copper/api_v1_list_opportunities.json b/packages/pipeline/test/fixtures/copper/api_v1_list_opportunities.json
new file mode 100644
index 000000000..34ac58c30
--- /dev/null
+++ b/packages/pipeline/test/fixtures/copper/api_v1_list_opportunities.json
@@ -0,0 +1,662 @@
+[
+ {
+ "id": 14050269,
+ "name": "8Base RaaS",
+ "assignee_id": 680302,
+ "close_date": "11/19/2018",
+ "company_id": 27778962,
+ "company_name": "8base",
+ "customer_source_id": null,
+ "details": "blah blah",
+ "loss_reason_id": null,
+ "pipeline_id": 512676,
+ "pipeline_stage_id": 2405442,
+ "primary_contact_id": 66088850,
+ "priority": "None",
+ "status": "Won",
+ "tags": [],
+ "interaction_count": 81,
+ "monetary_unit": null,
+ "monetary_value": null,
+ "converted_unit": null,
+ "converted_value": null,
+ "win_probability": 0,
+ "date_stage_changed": 1542653860,
+ "date_last_contacted": 1544757550,
+ "leads_converted_from": [],
+ "date_lead_created": null,
+ "date_created": 1538414159,
+ "date_modified": 1544769562,
+ "custom_fields": [
+ { "custom_field_definition_id": 261066, "value": [394013, 394018] },
+ { "custom_field_definition_id": 261067, "value": 394026 }
+ ]
+ },
+ {
+ "id": 14631430,
+ "name": "Alice.si TW + ERC 20 Marketplace",
+ "assignee_id": 680302,
+ "close_date": "12/15/2018",
+ "company_id": 30238847,
+ "company_name": "Alice SI",
+ "customer_source_id": null,
+ "details": "blah blah",
+ "loss_reason_id": null,
+ "pipeline_id": 512676,
+ "pipeline_stage_id": 2392929,
+ "primary_contact_id": 69354024,
+ "priority": "None",
+ "status": "Open",
+ "tags": [],
+ "interaction_count": 4,
+ "monetary_unit": null,
+ "monetary_value": null,
+ "converted_unit": null,
+ "converted_value": null,
+ "win_probability": 0,
+ "date_stage_changed": 1542304481,
+ "date_last_contacted": 1542304800,
+ "leads_converted_from": [],
+ "date_lead_created": null,
+ "date_created": 1542304481,
+ "date_modified": 1542304943,
+ "custom_fields": [
+ { "custom_field_definition_id": 261066, "value": [394013, 394015] },
+ { "custom_field_definition_id": 261067, "value": 394023 }
+ ]
+ },
+ {
+ "id": 14632057,
+ "name": "Altcoin.io Relayer",
+ "assignee_id": 680302,
+ "close_date": "12/15/2018",
+ "company_id": 29936486,
+ "company_name": "Altcoin.io",
+ "customer_source_id": null,
+ "details": "blah blah",
+ "loss_reason_id": null,
+ "pipeline_id": 512676,
+ "pipeline_stage_id": 2392929,
+ "primary_contact_id": 68724646,
+ "priority": "None",
+ "status": "Open",
+ "tags": [],
+ "interaction_count": 22,
+ "monetary_unit": null,
+ "monetary_value": null,
+ "converted_unit": null,
+ "converted_value": null,
+ "win_probability": 0,
+ "date_stage_changed": 1542310909,
+ "date_last_contacted": 1543864597,
+ "leads_converted_from": [],
+ "date_lead_created": null,
+ "date_created": 1542306827,
+ "date_modified": 1543864667,
+ "custom_fields": [
+ { "custom_field_definition_id": 261066, "value": [394013, 394017] },
+ { "custom_field_definition_id": 261067, "value": 394023 }
+ ]
+ },
+ {
+ "id": 14667523,
+ "name": "Altcoin.io Relayer",
+ "assignee_id": 680302,
+ "close_date": "12/19/2018",
+ "company_id": 29936486,
+ "company_name": "Altcoin.io",
+ "customer_source_id": null,
+ "details": "blah blah",
+ "loss_reason_id": null,
+ "pipeline_id": 512676,
+ "pipeline_stage_id": 2392929,
+ "primary_contact_id": 68724646,
+ "priority": "None",
+ "status": "Open",
+ "tags": [],
+ "interaction_count": 21,
+ "monetary_unit": null,
+ "monetary_value": null,
+ "converted_unit": null,
+ "converted_value": null,
+ "win_probability": 0,
+ "date_stage_changed": 1542657437,
+ "date_last_contacted": 1543864597,
+ "leads_converted_from": [],
+ "date_lead_created": null,
+ "date_created": 1542657437,
+ "date_modified": 1543864667,
+ "custom_fields": [
+ { "custom_field_definition_id": 261066, "value": [394013, 394017] },
+ { "custom_field_definition_id": 261067, "value": 394023 }
+ ]
+ },
+ {
+ "id": 14666706,
+ "name": "Amadeus Relayer",
+ "assignee_id": 680302,
+ "close_date": "11/19/2018",
+ "company_id": 29243209,
+ "company_name": "Amadeus",
+ "customer_source_id": null,
+ "details": "blah blah",
+ "loss_reason_id": null,
+ "pipeline_id": 512676,
+ "pipeline_stage_id": 2405442,
+ "primary_contact_id": 66912020,
+ "priority": "None",
+ "status": "Won",
+ "tags": [],
+ "interaction_count": 11,
+ "monetary_unit": null,
+ "monetary_value": null,
+ "converted_unit": null,
+ "converted_value": null,
+ "win_probability": 0,
+ "date_stage_changed": 1542654284,
+ "date_last_contacted": 1543264254,
+ "leads_converted_from": [],
+ "date_lead_created": null,
+ "date_created": 1542654284,
+ "date_modified": 1543277520,
+ "custom_fields": [
+ { "custom_field_definition_id": 261066, "value": [394013] },
+ { "custom_field_definition_id": 261067, "value": 394023 }
+ ]
+ },
+ {
+ "id": 14666718,
+ "name": "Ambo Relayer",
+ "assignee_id": 680302,
+ "close_date": "11/19/2018",
+ "company_id": 29249190,
+ "company_name": "Ambo",
+ "customer_source_id": null,
+ "details": "blah blah",
+ "loss_reason_id": null,
+ "pipeline_id": 512676,
+ "pipeline_stage_id": 2405442,
+ "primary_contact_id": 66927869,
+ "priority": "None",
+ "status": "Won",
+ "tags": [],
+ "interaction_count": 126,
+ "monetary_unit": null,
+ "monetary_value": null,
+ "converted_unit": null,
+ "converted_value": null,
+ "win_probability": 0,
+ "date_stage_changed": 1542654352,
+ "date_last_contacted": 1545252349,
+ "leads_converted_from": [],
+ "date_lead_created": null,
+ "date_created": 1542654352,
+ "date_modified": 1545253761,
+ "custom_fields": [
+ { "custom_field_definition_id": 261066, "value": [394013] },
+ { "custom_field_definition_id": 261067, "value": 394023 }
+ ]
+ },
+ {
+ "id": 14164318,
+ "name": "Augur TW",
+ "assignee_id": 680302,
+ "close_date": "12/10/2018",
+ "company_id": 27778967,
+ "company_name": "Augur",
+ "customer_source_id": null,
+ "details": "blah blah",
+ "loss_reason_id": null,
+ "pipeline_id": 512676,
+ "pipeline_stage_id": 2405442,
+ "primary_contact_id": 67248692,
+ "priority": "None",
+ "status": "Won",
+ "tags": [],
+ "interaction_count": 22,
+ "monetary_unit": null,
+ "monetary_value": null,
+ "converted_unit": null,
+ "converted_value": null,
+ "win_probability": 0,
+ "date_stage_changed": 1544469362,
+ "date_last_contacted": 1544491567,
+ "leads_converted_from": [],
+ "date_lead_created": null,
+ "date_created": 1539204858,
+ "date_modified": 1544653867,
+ "custom_fields": [
+ { "custom_field_definition_id": 261066, "value": [394015] },
+ { "custom_field_definition_id": 261067, "value": 394021 }
+ ]
+ },
+ {
+ "id": 14666626,
+ "name": "Autonio",
+ "assignee_id": 680302,
+ "close_date": "12/19/2018",
+ "company_id": 27920701,
+ "company_name": "Auton",
+ "customer_source_id": null,
+ "details": "blah blah",
+ "loss_reason_id": null,
+ "pipeline_id": 512676,
+ "pipeline_stage_id": 2392931,
+ "primary_contact_id": 64742640,
+ "priority": "None",
+ "status": "Open",
+ "tags": [],
+ "interaction_count": 54,
+ "monetary_unit": null,
+ "monetary_value": null,
+ "converted_unit": null,
+ "converted_value": null,
+ "win_probability": 0,
+ "date_stage_changed": 1542653834,
+ "date_last_contacted": 1542658568,
+ "leads_converted_from": [],
+ "date_lead_created": null,
+ "date_created": 1542653834,
+ "date_modified": 1542658808,
+ "custom_fields": [
+ { "custom_field_definition_id": 261066, "value": [394013, 394019] },
+ { "custom_field_definition_id": 261067, "value": 394023 }
+ ]
+ },
+ {
+ "id": 14050921,
+ "name": "Axie Infinity 721 Marketplace",
+ "assignee_id": 680302,
+ "close_date": "11/1/2018",
+ "company_id": 27779033,
+ "company_name": "Axie Infinity",
+ "customer_source_id": null,
+ "details": "blah blah",
+ "loss_reason_id": null,
+ "pipeline_id": 512676,
+ "pipeline_stage_id": 2392931,
+ "primary_contact_id": 66499254,
+ "priority": "None",
+ "status": "Open",
+ "tags": [],
+ "interaction_count": 4,
+ "monetary_unit": null,
+ "monetary_value": null,
+ "converted_unit": null,
+ "converted_value": null,
+ "win_probability": 0,
+ "date_stage_changed": 1543861025,
+ "date_last_contacted": 1539024738,
+ "leads_converted_from": [],
+ "date_lead_created": null,
+ "date_created": 1538416687,
+ "date_modified": 1543861025,
+ "custom_fields": [
+ { "custom_field_definition_id": 261066, "value": [394014] },
+ { "custom_field_definition_id": 261067, "value": 394134 }
+ ]
+ },
+ {
+ "id": 13735617,
+ "name": "Balance TW",
+ "assignee_id": 680302,
+ "close_date": "12/10/2018",
+ "company_id": 27778968,
+ "company_name": "Balance",
+ "customer_source_id": null,
+ "details": "blah blah",
+ "loss_reason_id": null,
+ "pipeline_id": 512676,
+ "pipeline_stage_id": 2405442,
+ "primary_contact_id": 64713448,
+ "priority": "None",
+ "status": "Won",
+ "tags": [],
+ "interaction_count": 34,
+ "monetary_unit": null,
+ "monetary_value": null,
+ "converted_unit": null,
+ "converted_value": null,
+ "win_probability": 0,
+ "date_stage_changed": 1544469382,
+ "date_last_contacted": 1545082200,
+ "leads_converted_from": [],
+ "date_lead_created": null,
+ "date_created": 1535668009,
+ "date_modified": 1545082454,
+ "custom_fields": [
+ { "custom_field_definition_id": 261066, "value": [394015] },
+ { "custom_field_definition_id": 261067, "value": 394027 }
+ ]
+ },
+ {
+ "id": 14667112,
+ "name": "Bamboo Relayer",
+ "assignee_id": 680302,
+ "close_date": "11/19/2018",
+ "company_id": 29243795,
+ "company_name": "Bamboo Relay",
+ "customer_source_id": null,
+ "details": "blah blah",
+ "loss_reason_id": null,
+ "pipeline_id": 512676,
+ "pipeline_stage_id": 2405442,
+ "primary_contact_id": 66914687,
+ "priority": "None",
+ "status": "Won",
+ "tags": [],
+ "interaction_count": 46,
+ "monetary_unit": null,
+ "monetary_value": null,
+ "converted_unit": null,
+ "converted_value": null,
+ "win_probability": 0,
+ "date_stage_changed": 1542655143,
+ "date_last_contacted": 1545252349,
+ "leads_converted_from": [],
+ "date_lead_created": null,
+ "date_created": 1542655143,
+ "date_modified": 1545253761,
+ "custom_fields": [
+ { "custom_field_definition_id": 261066, "value": [394013] },
+ { "custom_field_definition_id": 261067, "value": 394023 }
+ ]
+ },
+ {
+ "id": 13627309,
+ "name": "Ben TW",
+ "assignee_id": 680302,
+ "close_date": "1/1/2019",
+ "company_id": 27702348,
+ "company_name": "Ben",
+ "customer_source_id": null,
+ "details": "blah blah",
+ "loss_reason_id": null,
+ "pipeline_id": 512676,
+ "pipeline_stage_id": 2392929,
+ "primary_contact_id": 64262622,
+ "priority": "None",
+ "status": "Open",
+ "tags": [],
+ "interaction_count": 64,
+ "monetary_unit": null,
+ "monetary_value": null,
+ "converted_unit": null,
+ "converted_value": null,
+ "win_probability": 0,
+ "date_stage_changed": 1541527279,
+ "date_last_contacted": 1541639882,
+ "leads_converted_from": [],
+ "date_lead_created": null,
+ "date_created": 1534887789,
+ "date_modified": 1541651395,
+ "custom_fields": [
+ { "custom_field_definition_id": 261066, "value": [394015] },
+ { "custom_field_definition_id": 261067, "value": 394027 }
+ ]
+ },
+ {
+ "id": 14808512,
+ "name": "Bit2Me Relayer",
+ "assignee_id": 680302,
+ "close_date": "12/3/2018",
+ "company_id": 30793050,
+ "company_name": "Bit2Me",
+ "customer_source_id": null,
+ "details": "blah blah",
+ "loss_reason_id": null,
+ "pipeline_id": 512676,
+ "pipeline_stage_id": 2405442,
+ "primary_contact_id": 70267217,
+ "priority": "None",
+ "status": "Won",
+ "tags": [],
+ "interaction_count": 0,
+ "monetary_unit": null,
+ "monetary_value": null,
+ "converted_unit": null,
+ "converted_value": null,
+ "win_probability": 0,
+ "date_stage_changed": 1543861167,
+ "date_last_contacted": null,
+ "leads_converted_from": [],
+ "date_lead_created": null,
+ "date_created": 1543861167,
+ "date_modified": 1543861189,
+ "custom_fields": [
+ { "custom_field_definition_id": 261066, "value": [394013] },
+ { "custom_field_definition_id": 261067, "value": 394023 }
+ ]
+ },
+ {
+ "id": 14050312,
+ "name": "Bitcoin.tax Reporting Integration",
+ "assignee_id": 680302,
+ "close_date": "11/1/2018",
+ "company_id": 27957614,
+ "company_name": "Bitcoin",
+ "customer_source_id": null,
+ "details": "blah blah",
+ "loss_reason_id": null,
+ "pipeline_id": 512676,
+ "pipeline_stage_id": 2392928,
+ "primary_contact_id": 66539479,
+ "priority": "None",
+ "status": "Open",
+ "tags": [],
+ "interaction_count": 5,
+ "monetary_unit": null,
+ "monetary_value": null,
+ "converted_unit": null,
+ "converted_value": null,
+ "win_probability": 0,
+ "date_stage_changed": 1538414308,
+ "date_last_contacted": 1536766098,
+ "leads_converted_from": [],
+ "date_lead_created": null,
+ "date_created": 1538414308,
+ "date_modified": 1538414314,
+ "custom_fields": [
+ { "custom_field_definition_id": 261066, "value": [394019] },
+ { "custom_field_definition_id": 261067, "value": 394026 }
+ ]
+ },
+ {
+ "id": 14331463,
+ "name": "Bitpie TW",
+ "assignee_id": 680302,
+ "close_date": "11/19/2018",
+ "company_id": 27779026,
+ "company_name": "Bitpie",
+ "customer_source_id": null,
+ "details": "blah blah",
+ "loss_reason_id": null,
+ "pipeline_id": 512676,
+ "pipeline_stage_id": 2392929,
+ "primary_contact_id": 67700943,
+ "priority": "None",
+ "status": "Open",
+ "tags": [],
+ "interaction_count": 9,
+ "monetary_unit": null,
+ "monetary_value": null,
+ "converted_unit": null,
+ "converted_value": null,
+ "win_probability": 0,
+ "date_stage_changed": 1539984566,
+ "date_last_contacted": 1541529947,
+ "leads_converted_from": [],
+ "date_lead_created": null,
+ "date_created": 1539984566,
+ "date_modified": 1541530233,
+ "custom_fields": [
+ { "custom_field_definition_id": 261066, "value": [394015] },
+ { "custom_field_definition_id": 261067, "value": 394027 }
+ ]
+ },
+ {
+ "id": 14331481,
+ "name": "Bitski Wallet SDK TW",
+ "assignee_id": 680302,
+ "close_date": "11/19/2018",
+ "company_id": 29489300,
+ "company_name": "Bitski",
+ "customer_source_id": null,
+ "details": "blah blah",
+ "loss_reason_id": null,
+ "pipeline_id": 512676,
+ "pipeline_stage_id": 2392929,
+ "primary_contact_id": 67697528,
+ "priority": "None",
+ "status": "Open",
+ "tags": [],
+ "interaction_count": 23,
+ "monetary_unit": null,
+ "monetary_value": null,
+ "converted_unit": null,
+ "converted_value": null,
+ "win_probability": 0,
+ "date_stage_changed": 1539984735,
+ "date_last_contacted": 1544811399,
+ "leads_converted_from": [],
+ "date_lead_created": null,
+ "date_created": 1539984735,
+ "date_modified": 1544818605,
+ "custom_fields": [
+ { "custom_field_definition_id": 261066, "value": [394015] },
+ { "custom_field_definition_id": 261067, "value": 394026 }
+ ]
+ },
+ {
+ "id": 14531554,
+ "name": "BitUniverse TW",
+ "assignee_id": 680302,
+ "close_date": "12/6/2018",
+ "company_id": 29901805,
+ "company_name": "BitUniverse Co., Ltd (Cryptocurrency Portfolio)",
+ "customer_source_id": null,
+ "details": "blah blah",
+ "loss_reason_id": null,
+ "pipeline_id": 512676,
+ "pipeline_stage_id": 2392929,
+ "primary_contact_id": 68692107,
+ "priority": "None",
+ "status": "Open",
+ "tags": [],
+ "interaction_count": 15,
+ "monetary_unit": null,
+ "monetary_value": null,
+ "converted_unit": null,
+ "converted_value": null,
+ "win_probability": 0,
+ "date_stage_changed": 1543861104,
+ "date_last_contacted": 1544803276,
+ "leads_converted_from": [],
+ "date_lead_created": null,
+ "date_created": 1541527110,
+ "date_modified": 1544812979,
+ "custom_fields": [
+ { "custom_field_definition_id": 261066, "value": [394015] },
+ { "custom_field_definition_id": 261067, "value": 394026 }
+ ]
+ },
+ {
+ "id": 14050895,
+ "name": "BlitzPredict PMR",
+ "assignee_id": 680302,
+ "close_date": "11/1/2018",
+ "company_id": 28758258,
+ "company_name": "BlitzPredict",
+ "customer_source_id": null,
+ "details": "blah blah",
+ "loss_reason_id": null,
+ "pipeline_id": 512676,
+ "pipeline_stage_id": 2392929,
+ "primary_contact_id": 66378659,
+ "priority": "None",
+ "status": "Open",
+ "tags": [],
+ "interaction_count": 32,
+ "monetary_unit": null,
+ "monetary_value": null,
+ "converted_unit": null,
+ "converted_value": null,
+ "win_probability": 0,
+ "date_stage_changed": 1539985501,
+ "date_last_contacted": 1544830560,
+ "leads_converted_from": [],
+ "date_lead_created": null,
+ "date_created": 1538416597,
+ "date_modified": 1544830709,
+ "custom_fields": [
+ { "custom_field_definition_id": 261066, "value": [394016] },
+ { "custom_field_definition_id": 261067, "value": 394023 }
+ ]
+ },
+ {
+ "id": 14209841,
+ "name": "Blockfolio TW",
+ "assignee_id": 680302,
+ "close_date": "11/15/2018",
+ "company_id": 29332516,
+ "company_name": "Blockfolio",
+ "customer_source_id": null,
+ "details": "blah blah",
+ "loss_reason_id": null,
+ "pipeline_id": 512676,
+ "pipeline_stage_id": 2405443,
+ "primary_contact_id": 67247027,
+ "priority": "None",
+ "status": "Open",
+ "tags": [],
+ "interaction_count": 20,
+ "monetary_unit": null,
+ "monetary_value": null,
+ "converted_unit": null,
+ "converted_value": null,
+ "win_probability": 0,
+ "date_stage_changed": 1539984098,
+ "date_last_contacted": 1539977661,
+ "leads_converted_from": [],
+ "date_lead_created": null,
+ "date_created": 1539624801,
+ "date_modified": 1539984098,
+ "custom_fields": [
+ { "custom_field_definition_id": 261066, "value": [394015] },
+ { "custom_field_definition_id": 261067, "value": 394026 }
+ ]
+ },
+ {
+ "id": 14633220,
+ "name": "BlockSwap 721 / 1155 Conversational Marketplace",
+ "assignee_id": 680302,
+ "close_date": "12/15/2018",
+ "company_id": 30210921,
+ "company_name": "BlockSwap",
+ "customer_source_id": null,
+ "details": "blah blah",
+ "loss_reason_id": null,
+ "pipeline_id": 512676,
+ "pipeline_stage_id": 2392929,
+ "primary_contact_id": 69296220,
+ "priority": "None",
+ "status": "Open",
+ "tags": [],
+ "interaction_count": 82,
+ "monetary_unit": null,
+ "monetary_value": null,
+ "converted_unit": null,
+ "converted_value": null,
+ "win_probability": 0,
+ "date_stage_changed": 1542311056,
+ "date_last_contacted": 1543536442,
+ "leads_converted_from": [],
+ "date_lead_created": null,
+ "date_created": 1542311056,
+ "date_modified": 1543557877,
+ "custom_fields": [
+ { "custom_field_definition_id": 261066, "value": [394014] },
+ { "custom_field_definition_id": 261067, "value": 394023 }
+ ]
+ }
+]
diff --git a/packages/pipeline/test/fixtures/copper/api_v1_list_opportunities.ts b/packages/pipeline/test/fixtures/copper/api_v1_list_opportunities.ts
new file mode 100644
index 000000000..3c2d4ae5e
--- /dev/null
+++ b/packages/pipeline/test/fixtures/copper/api_v1_list_opportunities.ts
@@ -0,0 +1,425 @@
+// tslint:disable:custom-no-magic-numbers
+import { CopperOpportunity } from '../../../src/entities';
+const ParsedOpportunities: CopperOpportunity[] = [
+ {
+ id: 14050269,
+ name: '8Base RaaS',
+ assigneeId: 680302,
+ closeDate: '11/19/2018',
+ companyId: 27778962,
+ companyName: '8base',
+ customerSourceId: undefined,
+ lossReasonId: undefined,
+ pipelineId: 512676,
+ pipelineStageId: 2405442,
+ primaryContactId: 66088850,
+ priority: 'None',
+ status: 'Won',
+ interactionCount: 81,
+ monetaryValue: undefined,
+ winProbability: 0,
+ dateCreated: 1538414159000,
+ dateModified: 1544769562000,
+ customFields: { '261066': 394018, '261067': 394026 },
+ },
+ {
+ id: 14631430,
+ name: 'Alice.si TW + ERC 20 Marketplace',
+ assigneeId: 680302,
+ closeDate: '12/15/2018',
+ companyId: 30238847,
+ companyName: 'Alice SI',
+ customerSourceId: undefined,
+ lossReasonId: undefined,
+ pipelineId: 512676,
+ pipelineStageId: 2392929,
+ primaryContactId: 69354024,
+ priority: 'None',
+ status: 'Open',
+ interactionCount: 4,
+ monetaryValue: undefined,
+ winProbability: 0,
+ dateCreated: 1542304481000,
+ dateModified: 1542304943000,
+ customFields: { '261066': 394015, '261067': 394023 },
+ },
+ {
+ id: 14632057,
+ name: 'Altcoin.io Relayer',
+ assigneeId: 680302,
+ closeDate: '12/15/2018',
+ companyId: 29936486,
+ companyName: 'Altcoin.io',
+ customerSourceId: undefined,
+ lossReasonId: undefined,
+ pipelineId: 512676,
+ pipelineStageId: 2392929,
+ primaryContactId: 68724646,
+ priority: 'None',
+ status: 'Open',
+ interactionCount: 22,
+ monetaryValue: undefined,
+ winProbability: 0,
+ dateCreated: 1542306827000,
+ dateModified: 1543864667000,
+ customFields: { '261066': 394017, '261067': 394023 },
+ },
+ {
+ id: 14667523,
+ name: 'Altcoin.io Relayer',
+ assigneeId: 680302,
+ closeDate: '12/19/2018',
+ companyId: 29936486,
+ companyName: 'Altcoin.io',
+ customerSourceId: undefined,
+ lossReasonId: undefined,
+ pipelineId: 512676,
+ pipelineStageId: 2392929,
+ primaryContactId: 68724646,
+ priority: 'None',
+ status: 'Open',
+ interactionCount: 21,
+ monetaryValue: undefined,
+ winProbability: 0,
+ dateCreated: 1542657437000,
+ dateModified: 1543864667000,
+ customFields: { '261066': 394017, '261067': 394023 },
+ },
+ {
+ id: 14666706,
+ name: 'Amadeus Relayer',
+ assigneeId: 680302,
+ closeDate: '11/19/2018',
+ companyId: 29243209,
+ companyName: 'Amadeus',
+ customerSourceId: undefined,
+ lossReasonId: undefined,
+ pipelineId: 512676,
+ pipelineStageId: 2405442,
+ primaryContactId: 66912020,
+ priority: 'None',
+ status: 'Won',
+ interactionCount: 11,
+ monetaryValue: undefined,
+ winProbability: 0,
+ dateCreated: 1542654284000,
+ dateModified: 1543277520000,
+ customFields: { '261066': 394013, '261067': 394023 },
+ },
+ {
+ id: 14666718,
+ name: 'Ambo Relayer',
+ assigneeId: 680302,
+ closeDate: '11/19/2018',
+ companyId: 29249190,
+ companyName: 'Ambo',
+ customerSourceId: undefined,
+ lossReasonId: undefined,
+ pipelineId: 512676,
+ pipelineStageId: 2405442,
+ primaryContactId: 66927869,
+ priority: 'None',
+ status: 'Won',
+ interactionCount: 126,
+ monetaryValue: undefined,
+ winProbability: 0,
+ dateCreated: 1542654352000,
+ dateModified: 1545253761000,
+ customFields: { '261066': 394013, '261067': 394023 },
+ },
+ {
+ id: 14164318,
+ name: 'Augur TW',
+ assigneeId: 680302,
+ closeDate: '12/10/2018',
+ companyId: 27778967,
+ companyName: 'Augur',
+ customerSourceId: undefined,
+ lossReasonId: undefined,
+ pipelineId: 512676,
+ pipelineStageId: 2405442,
+ primaryContactId: 67248692,
+ priority: 'None',
+ status: 'Won',
+ interactionCount: 22,
+ monetaryValue: undefined,
+ winProbability: 0,
+ dateCreated: 1539204858000,
+ dateModified: 1544653867000,
+ customFields: { '261066': 394015, '261067': 394021 },
+ },
+ {
+ id: 14666626,
+ name: 'Autonio',
+ assigneeId: 680302,
+ closeDate: '12/19/2018',
+ companyId: 27920701,
+ companyName: 'Auton',
+ customerSourceId: undefined,
+ lossReasonId: undefined,
+ pipelineId: 512676,
+ pipelineStageId: 2392931,
+ primaryContactId: 64742640,
+ priority: 'None',
+ status: 'Open',
+ interactionCount: 54,
+ monetaryValue: undefined,
+ winProbability: 0,
+ dateCreated: 1542653834000,
+ dateModified: 1542658808000,
+ customFields: { '261066': 394019, '261067': 394023 },
+ },
+ {
+ id: 14050921,
+ name: 'Axie Infinity 721 Marketplace',
+ assigneeId: 680302,
+ closeDate: '11/1/2018',
+ companyId: 27779033,
+ companyName: 'Axie Infinity',
+ customerSourceId: undefined,
+ lossReasonId: undefined,
+ pipelineId: 512676,
+ pipelineStageId: 2392931,
+ primaryContactId: 66499254,
+ priority: 'None',
+ status: 'Open',
+ interactionCount: 4,
+ monetaryValue: undefined,
+ winProbability: 0,
+ dateCreated: 1538416687000,
+ dateModified: 1543861025000,
+ customFields: { '261066': 394014, '261067': 394134 },
+ },
+ {
+ id: 13735617,
+ name: 'Balance TW',
+ assigneeId: 680302,
+ closeDate: '12/10/2018',
+ companyId: 27778968,
+ companyName: 'Balance',
+ customerSourceId: undefined,
+ lossReasonId: undefined,
+ pipelineId: 512676,
+ pipelineStageId: 2405442,
+ primaryContactId: 64713448,
+ priority: 'None',
+ status: 'Won',
+ interactionCount: 34,
+ monetaryValue: undefined,
+ winProbability: 0,
+ dateCreated: 1535668009000,
+ dateModified: 1545082454000,
+ customFields: { '261066': 394015, '261067': 394027 },
+ },
+ {
+ id: 14667112,
+ name: 'Bamboo Relayer',
+ assigneeId: 680302,
+ closeDate: '11/19/2018',
+ companyId: 29243795,
+ companyName: 'Bamboo Relay',
+ customerSourceId: undefined,
+ lossReasonId: undefined,
+ pipelineId: 512676,
+ pipelineStageId: 2405442,
+ primaryContactId: 66914687,
+ priority: 'None',
+ status: 'Won',
+ interactionCount: 46,
+ monetaryValue: undefined,
+ winProbability: 0,
+ dateCreated: 1542655143000,
+ dateModified: 1545253761000,
+ customFields: { '261066': 394013, '261067': 394023 },
+ },
+ {
+ id: 13627309,
+ name: 'Ben TW',
+ assigneeId: 680302,
+ closeDate: '1/1/2019',
+ companyId: 27702348,
+ companyName: 'Ben',
+ customerSourceId: undefined,
+ lossReasonId: undefined,
+ pipelineId: 512676,
+ pipelineStageId: 2392929,
+ primaryContactId: 64262622,
+ priority: 'None',
+ status: 'Open',
+ interactionCount: 64,
+ monetaryValue: undefined,
+ winProbability: 0,
+ dateCreated: 1534887789000,
+ dateModified: 1541651395000,
+ customFields: { '261066': 394015, '261067': 394027 },
+ },
+ {
+ id: 14808512,
+ name: 'Bit2Me Relayer',
+ assigneeId: 680302,
+ closeDate: '12/3/2018',
+ companyId: 30793050,
+ companyName: 'Bit2Me',
+ customerSourceId: undefined,
+ lossReasonId: undefined,
+ pipelineId: 512676,
+ pipelineStageId: 2405442,
+ primaryContactId: 70267217,
+ priority: 'None',
+ status: 'Won',
+ interactionCount: 0,
+ monetaryValue: undefined,
+ winProbability: 0,
+ dateCreated: 1543861167000,
+ dateModified: 1543861189000,
+ customFields: { '261066': 394013, '261067': 394023 },
+ },
+ {
+ id: 14050312,
+ name: 'Bitcoin.tax Reporting Integration',
+ assigneeId: 680302,
+ closeDate: '11/1/2018',
+ companyId: 27957614,
+ companyName: 'Bitcoin',
+ customerSourceId: undefined,
+ lossReasonId: undefined,
+ pipelineId: 512676,
+ pipelineStageId: 2392928,
+ primaryContactId: 66539479,
+ priority: 'None',
+ status: 'Open',
+ interactionCount: 5,
+ monetaryValue: undefined,
+ winProbability: 0,
+ dateCreated: 1538414308000,
+ dateModified: 1538414314000,
+ customFields: { '261066': 394019, '261067': 394026 },
+ },
+ {
+ id: 14331463,
+ name: 'Bitpie TW',
+ assigneeId: 680302,
+ closeDate: '11/19/2018',
+ companyId: 27779026,
+ companyName: 'Bitpie',
+ customerSourceId: undefined,
+ lossReasonId: undefined,
+ pipelineId: 512676,
+ pipelineStageId: 2392929,
+ primaryContactId: 67700943,
+ priority: 'None',
+ status: 'Open',
+ interactionCount: 9,
+ monetaryValue: undefined,
+ winProbability: 0,
+ dateCreated: 1539984566000,
+ dateModified: 1541530233000,
+ customFields: { '261066': 394015, '261067': 394027 },
+ },
+ {
+ id: 14331481,
+ name: 'Bitski Wallet SDK TW',
+ assigneeId: 680302,
+ closeDate: '11/19/2018',
+ companyId: 29489300,
+ companyName: 'Bitski',
+ customerSourceId: undefined,
+ lossReasonId: undefined,
+ pipelineId: 512676,
+ pipelineStageId: 2392929,
+ primaryContactId: 67697528,
+ priority: 'None',
+ status: 'Open',
+ interactionCount: 23,
+ monetaryValue: undefined,
+ winProbability: 0,
+ dateCreated: 1539984735000,
+ dateModified: 1544818605000,
+ customFields: { '261066': 394015, '261067': 394026 },
+ },
+ {
+ id: 14531554,
+ name: 'BitUniverse TW',
+ assigneeId: 680302,
+ closeDate: '12/6/2018',
+ companyId: 29901805,
+ companyName: 'BitUniverse Co., Ltd (Cryptocurrency Portfolio)',
+ customerSourceId: undefined,
+ lossReasonId: undefined,
+ pipelineId: 512676,
+ pipelineStageId: 2392929,
+ primaryContactId: 68692107,
+ priority: 'None',
+ status: 'Open',
+ interactionCount: 15,
+ monetaryValue: undefined,
+ winProbability: 0,
+ dateCreated: 1541527110000,
+ dateModified: 1544812979000,
+ customFields: { '261066': 394015, '261067': 394026 },
+ },
+ {
+ id: 14050895,
+ name: 'BlitzPredict PMR',
+ assigneeId: 680302,
+ closeDate: '11/1/2018',
+ companyId: 28758258,
+ companyName: 'BlitzPredict',
+ customerSourceId: undefined,
+ lossReasonId: undefined,
+ pipelineId: 512676,
+ pipelineStageId: 2392929,
+ primaryContactId: 66378659,
+ priority: 'None',
+ status: 'Open',
+ interactionCount: 32,
+ monetaryValue: undefined,
+ winProbability: 0,
+ dateCreated: 1538416597000,
+ dateModified: 1544830709000,
+ customFields: { '261066': 394016, '261067': 394023 },
+ },
+ {
+ id: 14209841,
+ name: 'Blockfolio TW',
+ assigneeId: 680302,
+ closeDate: '11/15/2018',
+ companyId: 29332516,
+ companyName: 'Blockfolio',
+ customerSourceId: undefined,
+ lossReasonId: undefined,
+ pipelineId: 512676,
+ pipelineStageId: 2405443,
+ primaryContactId: 67247027,
+ priority: 'None',
+ status: 'Open',
+ interactionCount: 20,
+ monetaryValue: undefined,
+ winProbability: 0,
+ dateCreated: 1539624801000,
+ dateModified: 1539984098000,
+ customFields: { '261066': 394015, '261067': 394026 },
+ },
+ {
+ id: 14633220,
+ name: 'BlockSwap 721 / 1155 Conversational Marketplace',
+ assigneeId: 680302,
+ closeDate: '12/15/2018',
+ companyId: 30210921,
+ companyName: 'BlockSwap',
+ customerSourceId: undefined,
+ lossReasonId: undefined,
+ pipelineId: 512676,
+ pipelineStageId: 2392929,
+ primaryContactId: 69296220,
+ priority: 'None',
+ status: 'Open',
+ interactionCount: 82,
+ monetaryValue: undefined,
+ winProbability: 0,
+ dateCreated: 1542311056000,
+ dateModified: 1543557877000,
+ customFields: { '261066': 394014, '261067': 394023 },
+ },
+];
+export { ParsedOpportunities };
diff --git a/packages/pipeline/test/fixtures/copper/parsed_entities.ts b/packages/pipeline/test/fixtures/copper/parsed_entities.ts
new file mode 100644
index 000000000..1f49d38ed
--- /dev/null
+++ b/packages/pipeline/test/fixtures/copper/parsed_entities.ts
@@ -0,0 +1,5 @@
+export { ParsedActivityTypes } from './api_v1_activity_types';
+export { ParsedCustomFields } from './api_v1_custom_field_definitions';
+export { ParsedActivities } from './api_v1_list_activities';
+export { ParsedLeads } from './api_v1_list_leads';
+export { ParsedOpportunities } from './api_v1_list_opportunities';
diff --git a/packages/pipeline/test/parsers/copper/index_test.ts b/packages/pipeline/test/parsers/copper/index_test.ts
new file mode 100644
index 000000000..bb8e70da1
--- /dev/null
+++ b/packages/pipeline/test/parsers/copper/index_test.ts
@@ -0,0 +1,87 @@
+import * as chai from 'chai';
+import 'mocha';
+
+import {
+ CopperActivity,
+ CopperActivityType,
+ CopperCustomField,
+ CopperLead,
+ CopperOpportunity,
+} from '../../../src/entities';
+import {
+ CopperActivityResponse,
+ CopperActivityTypeCategory,
+ CopperActivityTypeResponse,
+ CopperCustomFieldResponse,
+ CopperSearchResponse,
+ parseActivities,
+ parseActivityTypes,
+ parseCustomFields,
+ parseLeads,
+ parseOpportunities,
+} from '../../../src/parsers/copper';
+import { chaiSetup } from '../../utils/chai_setup';
+
+chaiSetup.configure();
+const expect = chai.expect;
+
+type CopperResponse = CopperSearchResponse | CopperCustomFieldResponse;
+type CopperEntity = CopperLead | CopperActivity | CopperOpportunity | CopperActivityType | CopperCustomField;
+
+import * as activityTypesApiResponse from '../../fixtures/copper/api_v1_activity_types.json';
+import * as customFieldsApiResponse from '../../fixtures/copper/api_v1_custom_field_definitions.json';
+import * as listActivitiesApiResponse from '../../fixtures/copper/api_v1_list_activities.json';
+import * as listLeadsApiResponse from '../../fixtures/copper/api_v1_list_leads.json';
+import * as listOpportunitiesApiResponse from '../../fixtures/copper/api_v1_list_opportunities.json';
+import {
+ ParsedActivities,
+ ParsedActivityTypes,
+ ParsedCustomFields,
+ ParsedLeads,
+ ParsedOpportunities,
+} from '../../fixtures/copper/parsed_entities';
+
+interface TestCase {
+ input: CopperResponse[];
+ expected: CopperEntity[];
+ parseFn(input: CopperResponse[]): CopperEntity[];
+}
+const testCases: TestCase[] = [
+ {
+ input: listLeadsApiResponse,
+ expected: ParsedLeads,
+ parseFn: parseLeads,
+ },
+ {
+ input: (listActivitiesApiResponse as unknown) as CopperActivityResponse[],
+ expected: ParsedActivities,
+ parseFn: parseActivities,
+ },
+ {
+ input: listOpportunitiesApiResponse,
+ expected: ParsedOpportunities,
+ parseFn: parseOpportunities,
+ },
+ {
+ input: customFieldsApiResponse,
+ expected: ParsedCustomFields,
+ parseFn: parseCustomFields,
+ },
+];
+describe('Copper parser', () => {
+ it('parses API responses', () => {
+ testCases.forEach(testCase => {
+ const actual: CopperEntity[] = testCase.parseFn(testCase.input);
+ expect(actual).deep.equal(testCase.expected);
+ });
+ });
+
+ // special case because the API response is not an array
+ it('parses activity types API response', () => {
+ const actual: CopperActivityType[] = parseActivityTypes((activityTypesApiResponse as unknown) as Map<
+ CopperActivityTypeCategory,
+ CopperActivityTypeResponse[]
+ >);
+ expect(actual).deep.equal(ParsedActivityTypes);
+ });
+});
diff --git a/packages/pipeline/test/parsers/ddex_orders/index_test.ts b/packages/pipeline/test/parsers/ddex_orders/index_test.ts
index 4a4a86bf8..f30e86b02 100644
--- a/packages/pipeline/test/parsers/ddex_orders/index_test.ts
+++ b/packages/pipeline/test/parsers/ddex_orders/index_test.ts
@@ -31,13 +31,13 @@ describe('ddex_orders', () => {
amountDecimals: 0,
};
const observedTimestamp: number = Date.now();
- const orderType: OrderType = 'bid';
+ const orderType: OrderType = OrderType.Bid;
const source: string = 'ddex';
const expected = new TokenOrder();
expected.source = 'ddex';
expected.observedTimestamp = observedTimestamp;
- expected.orderType = 'bid';
+ expected.orderType = OrderType.Bid;
expected.price = new BigNumber(0.5);
// ddex currently confuses base and quote assets.
// Switch them to maintain our internal consistency.
diff --git a/packages/pipeline/test/parsers/events/exchange_events_test.ts b/packages/pipeline/test/parsers/events/exchange_events_test.ts
index 5d4b185a5..956ad9ef8 100644
--- a/packages/pipeline/test/parsers/events/exchange_events_test.ts
+++ b/packages/pipeline/test/parsers/events/exchange_events_test.ts
@@ -6,6 +6,7 @@ import 'mocha';
import { ExchangeFillEvent } from '../../../src/entities';
import { _convertToExchangeFillEvent } from '../../../src/parsers/events/exchange_events';
+import { AssetType } from '../../../src/types';
import { chaiSetup } from '../../utils/chai_setup';
chaiSetup.configure();
@@ -62,12 +63,12 @@ describe('exchange_events', () => {
expected.takerFeePaid = new BigNumber('12345');
expected.orderHash = '0xab12ed2cbaa5615ab690b9da75a46e53ddfcf3f1a68655b5fe0d94c75a1aac4a';
expected.rawMakerAssetData = '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2';
- expected.makerAssetType = 'erc20';
+ expected.makerAssetType = AssetType.ERC20;
expected.makerAssetProxyId = '0xf47261b0';
expected.makerTokenAddress = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2';
expected.makerTokenId = null;
expected.rawTakerAssetData = '0xf47261b0000000000000000000000000e41d2489571d322189246dafa5ebde1f4699f498';
- expected.takerAssetType = 'erc20';
+ expected.takerAssetType = AssetType.ERC20;
expected.takerAssetProxyId = '0xf47261b0';
expected.takerTokenAddress = '0xe41d2489571d322189246dafa5ebde1f4699f498';
expected.takerTokenId = null;
diff --git a/packages/pipeline/test/parsers/idex_orders/index_test.ts b/packages/pipeline/test/parsers/idex_orders/index_test.ts
index d54ecb9a8..48b019732 100644
--- a/packages/pipeline/test/parsers/idex_orders/index_test.ts
+++ b/packages/pipeline/test/parsers/idex_orders/index_test.ts
@@ -31,13 +31,13 @@ describe('idex_orders', () => {
user: '0x212345667543456435324564345643453453333',
};
const observedTimestamp: number = Date.now();
- const orderType: OrderType = 'bid';
+ const orderType: OrderType = OrderType.Bid;
const source: string = 'idex';
const expected = new TokenOrder();
expected.source = 'idex';
expected.observedTimestamp = observedTimestamp;
- expected.orderType = 'bid';
+ expected.orderType = OrderType.Bid;
expected.price = new BigNumber(0.5);
expected.baseAssetSymbol = 'ABC';
expected.baseAssetAddress = '0x0000000000000000000000000000000000000000';
@@ -65,13 +65,13 @@ describe('idex_orders', () => {
user: '0x212345667543456435324564345643453453333',
};
const observedTimestamp: number = Date.now();
- const orderType: OrderType = 'ask';
+ const orderType: OrderType = OrderType.Ask;
const source: string = 'idex';
const expected = new TokenOrder();
expected.source = 'idex';
expected.observedTimestamp = observedTimestamp;
- expected.orderType = 'ask';
+ expected.orderType = OrderType.Ask;
expected.price = new BigNumber(0.5);
expected.baseAssetSymbol = 'ABC';
expected.baseAssetAddress = '0x0000000000000000000000000000000000000000';
diff --git a/packages/pipeline/test/parsers/oasis_orders/index_test.ts b/packages/pipeline/test/parsers/oasis_orders/index_test.ts
index 433bfb665..401fedff8 100644
--- a/packages/pipeline/test/parsers/oasis_orders/index_test.ts
+++ b/packages/pipeline/test/parsers/oasis_orders/index_test.ts
@@ -27,13 +27,13 @@ describe('oasis_orders', () => {
low: 0,
};
const observedTimestamp: number = Date.now();
- const orderType: OrderType = 'bid';
+ const orderType: OrderType = OrderType.Bid;
const source: string = 'oasis';
const expected = new TokenOrder();
expected.source = 'oasis';
expected.observedTimestamp = observedTimestamp;
- expected.orderType = 'bid';
+ expected.orderType = OrderType.Bid;
expected.price = new BigNumber(0.5);
expected.baseAssetSymbol = 'DEF';
expected.baseAssetAddress = null;
diff --git a/packages/pipeline/test/parsers/paradex_orders/index_test.ts b/packages/pipeline/test/parsers/paradex_orders/index_test.ts
index 6b811b90d..c5dd8751b 100644
--- a/packages/pipeline/test/parsers/paradex_orders/index_test.ts
+++ b/packages/pipeline/test/parsers/paradex_orders/index_test.ts
@@ -32,13 +32,13 @@ describe('paradex_orders', () => {
quoteTokenAddress: '0x0000000000000000000000000000000000000000',
};
const observedTimestamp: number = Date.now();
- const orderType: OrderType = 'bid';
+ const orderType: OrderType = OrderType.Bid;
const source: string = 'paradex';
const expected = new TokenOrder();
expected.source = 'paradex';
expected.observedTimestamp = observedTimestamp;
- expected.orderType = 'bid';
+ expected.orderType = OrderType.Bid;
expected.price = new BigNumber(0.1245);
expected.baseAssetSymbol = 'DEF';
expected.baseAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81';
diff --git a/packages/pipeline/test/parsers/sra_orders/index_test.ts b/packages/pipeline/test/parsers/sra_orders/index_test.ts
index ee2842ef3..838171a72 100644
--- a/packages/pipeline/test/parsers/sra_orders/index_test.ts
+++ b/packages/pipeline/test/parsers/sra_orders/index_test.ts
@@ -5,6 +5,7 @@ import 'mocha';
import { SraOrder } from '../../../src/entities';
import { _convertToEntity } from '../../../src/parsers/sra_orders';
+import { AssetType } from '../../../src/types';
import { chaiSetup } from '../../utils/chai_setup';
chaiSetup.configure();
@@ -50,12 +51,12 @@ describe('sra_orders', () => {
expected.signature =
'0x1b5a5d672b0d647b5797387ccbb89d822d5d2e873346b014f4ff816ff0783f2a7a0d2824d2d7042ec8ea375bc7f870963e1cb8248f1db03ddf125e27b5963aa11f03';
expected.rawMakerAssetData = '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2';
- expected.makerAssetType = 'erc20';
+ expected.makerAssetType = AssetType.ERC20;
expected.makerAssetProxyId = '0xf47261b0';
expected.makerTokenAddress = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2';
expected.makerTokenId = null;
expected.rawTakerAssetData = '0xf47261b000000000000000000000000042d6622dece394b54999fbd73d108123806f6a18';
- expected.takerAssetType = 'erc20';
+ expected.takerAssetType = AssetType.ERC20;
expected.takerAssetProxyId = '0xf47261b0';
expected.takerTokenAddress = '0x42d6622dece394b54999fbd73d108123806f6a18';
expected.takerTokenId = null;
diff --git a/packages/pipeline/tsconfig.json b/packages/pipeline/tsconfig.json
index 6f138f260..45e07374c 100644
--- a/packages/pipeline/tsconfig.json
+++ b/packages/pipeline/tsconfig.json
@@ -4,7 +4,15 @@
"outDir": "lib",
"rootDir": ".",
"emitDecoratorMetadata": true,
- "experimentalDecorators": true
+ "experimentalDecorators": true,
+ "resolveJsonModule": true
},
- "include": ["./src/**/*", "./test/**/*", "./migrations/**/*"]
+ "include": ["./src/**/*", "./test/**/*", "./migrations/**/*"],
+ "files": [
+ "./test/fixtures/copper/api_v1_activity_types.json",
+ "./test/fixtures/copper/api_v1_custom_field_definitions.json",
+ "./test/fixtures/copper/api_v1_list_activities.json",
+ "./test/fixtures/copper/api_v1_list_leads.json",
+ "./test/fixtures/copper/api_v1_list_opportunities.json"
+ ]
}