diff options
Diffstat (limited to 'packages/pipeline')
45 files changed, 3652 insertions, 86 deletions
diff --git a/packages/pipeline/migrations/1545440485644-CreateCopperTables.ts b/packages/pipeline/migrations/1545440485644-CreateCopperTables.ts new file mode 100644 index 000000000..64bf70af4 --- /dev/null +++ b/packages/pipeline/migrations/1545440485644-CreateCopperTables.ts @@ -0,0 +1,103 @@ +import { MigrationInterface, QueryRunner, Table } from 'typeorm'; + +const leads = new Table({ + name: 'raw.copper_leads', + columns: [ + { name: 'id', type: 'bigint', isPrimary: true }, + { name: 'name', type: 'varchar', isNullable: true }, + { name: 'first_name', type: 'varchar', isNullable: true }, + { name: 'last_name', type: 'varchar', isNullable: true }, + { name: 'middle_name', type: 'varchar', isNullable: true }, + { name: 'assignee_id', type: 'bigint', isNullable: true }, + { name: 'company_name', type: 'varchar', isNullable: true }, + { name: 'customer_source_id', type: 'bigint', isNullable: true }, + { name: 'monetary_value', type: 'integer', isNullable: true }, + { name: 'status', type: 'varchar' }, + { name: 'status_id', type: 'bigint' }, + { name: 'title', type: 'varchar', isNullable: true }, + { name: 'date_created', type: 'bigint' }, + { name: 'date_modified', type: 'bigint', isPrimary: true }, + ], +}); +const activities = new Table({ + name: 'raw.copper_activities', + columns: [ + { name: 'id', type: 'bigint', isPrimary: true }, + { name: 'parent_id', type: 'bigint' }, + { name: 'parent_type', type: 'varchar' }, + { name: 'type_id', type: 'bigint' }, + { name: 'type_category', type: 'varchar' }, + { name: 'type_name', type: 'varchar', isNullable: true }, + { name: 'user_id', type: 'bigint' }, + { name: 'old_value_id', type: 'bigint', isNullable: true }, + { name: 'old_value_name', type: 'varchar', isNullable: true }, + { name: 'new_value_id', type: 'bigint', isNullable: true }, + { name: 'new_value_name', type: 'varchar', isNullable: true }, + { name: 'date_created', type: 'bigint' }, + { name: 'date_modified', type: 'bigint', isPrimary: true }, + ], +}); + +const opportunities = new Table({ + name: 'raw.copper_opportunities', + columns: [ + { name: 'id', type: 'bigint', isPrimary: true }, + { name: 'name', type: 'varchar' }, + { name: 'assignee_id', isNullable: true, type: 'bigint' }, + { name: 'close_date', isNullable: true, type: 'varchar' }, + { name: 'company_id', isNullable: true, type: 'bigint' }, + { name: 'company_name', isNullable: true, type: 'varchar' }, + { name: 'customer_source_id', isNullable: true, type: 'bigint' }, + { name: 'loss_reason_id', isNullable: true, type: 'bigint' }, + { name: 'pipeline_id', type: 'bigint' }, + { name: 'pipeline_stage_id', type: 'bigint' }, + { name: 'primary_contact_id', isNullable: true, type: 'bigint' }, + { name: 'priority', isNullable: true, type: 'varchar' }, + { name: 'status', type: 'varchar' }, + { name: 'interaction_count', type: 'bigint' }, + { name: 'monetary_value', isNullable: true, type: 'integer' }, + { name: 'win_probability', isNullable: true, type: 'integer' }, + { name: 'date_created', type: 'bigint' }, + { name: 'date_modified', type: 'bigint', isPrimary: true }, + { name: 'custom_fields', type: 'jsonb' }, + ], +}); + +const activityTypes = new Table({ + name: 'raw.copper_activity_types', + columns: [ + { name: 'id', type: 'bigint', isPrimary: true }, + { name: 'category', type: 'varchar' }, + { name: 'name', type: 'varchar' }, + { name: 'is_disabled', type: 'boolean', isNullable: true }, + { name: 'count_as_interaction', type: 'boolean', isNullable: true }, + ], +}); + +const customFields = new Table({ + name: 'raw.copper_custom_fields', + columns: [ + { name: 'id', type: 'bigint', isPrimary: true }, + { name: 'name', type: 'varchar' }, + { name: 'data_type', type: 'varchar' }, + { name: 'field_type', type: 'varchar', isNullable: true }, + ], +}); + +export class CreateCopperTables1544055699284 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise<any> { + await queryRunner.createTable(leads); + await queryRunner.createTable(activities); + await queryRunner.createTable(opportunities); + await queryRunner.createTable(activityTypes); + await queryRunner.createTable(customFields); + } + + public async down(queryRunner: QueryRunner): Promise<any> { + await queryRunner.dropTable(leads.name); + await queryRunner.dropTable(activities.name); + await queryRunner.dropTable(opportunities.name); + await queryRunner.dropTable(activityTypes.name); + await queryRunner.dropTable(customFields.name); + } +} diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index a40f3d21c..47150300d 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -1,6 +1,6 @@ { "name": "@0x/pipeline", - "version": "1.0.2", + "version": "1.0.3", "private": true, "description": "Data pipeline for offline analysis", "scripts": { @@ -16,7 +16,7 @@ "test:coverage": "nyc npm run test:all --all && yarn coverage:report:lcov", "coverage:report:lcov": "nyc report --reporter=text-lcov > coverage/lcov.info", "clean": "shx rm -rf lib", - "lint": "tslint --project . --format stylish --exclude ./migrations/**/*", + "lint": "tslint --project . --format stylish --exclude ./migrations/**/* --exclude ./test/fixtures/**/**/*.json", "migrate:run": "yarn typeorm migration:run --config ./lib/src/ormconfig", "migrate:revert": "yarn typeorm migration:revert --config ./lib/src/ormconfig", "migrate:create": "yarn typeorm migration:create --config ./lib/src/ormconfig --dir migrations" @@ -39,16 +39,16 @@ "typescript": "3.0.1" }, "dependencies": { - "@0x/connect": "^3.0.10", - "@0x/contract-addresses": "^2.0.0", - "@0x/contract-artifacts": "^1.0.1", + "@0x/connect": "^3.0.11", + "@0x/contract-addresses": "^2.1.0", + "@0x/contract-artifacts": "^1.2.0", "@0x/contract-wrappers": "^3.0.0", - "@0x/dev-utils": "^1.0.21", - "@0x/order-utils": "^2.0.0", - "@0x/subproviders": "^2.1.8", - "@0x/types": "^1.4.1", - "@0x/utils": "^2.0.8", - "@0x/web3-wrapper": "^3.2.1", + "@0x/dev-utils": "^1.0.22", + "@0x/order-utils": "^3.1.0", + "@0x/subproviders": "^2.1.9", + "@0x/types": "^1.5.0", + "@0x/utils": "^2.1.1", + "@0x/web3-wrapper": "^3.2.2", "@types/dockerode": "^2.5.9", "@types/p-limit": "^2.0.0", "async-parallel": "^1.2.3", diff --git a/packages/pipeline/src/data_sources/copper/index.ts b/packages/pipeline/src/data_sources/copper/index.ts new file mode 100644 index 000000000..15df2fd7d --- /dev/null +++ b/packages/pipeline/src/data_sources/copper/index.ts @@ -0,0 +1,126 @@ +import { fetchAsync } from '@0x/utils'; +import Bottleneck from 'bottleneck'; + +import { + CopperActivityTypeCategory, + CopperActivityTypeResponse, + CopperCustomFieldResponse, + CopperSearchResponse, +} from '../../parsers/copper'; + +const HTTP_OK_STATUS = 200; +const COPPER_URI = 'https://api.prosperworks.com/developer_api/v1'; + +const DEFAULT_PAGINATION_PARAMS = { + page_size: 200, + sort_by: 'date_modified', + sort_direction: 'desc', +}; + +export type CopperSearchParams = CopperLeadSearchParams | CopperActivitySearchParams | CopperOpportunitySearchParams; +export interface CopperLeadSearchParams { + page_number?: number; +} + +export interface CopperActivitySearchParams { + minimum_activity_date: number; + page_number?: number; +} + +export interface CopperOpportunitySearchParams { + sort_by: string; // must override the default 'date_modified' for this endpoint + page_number?: number; +} +export enum CopperEndpoint { + Leads = '/leads/search', + Opportunities = '/opportunities/search', + Activities = '/activities/search', +} +const ONE_SECOND = 1000; + +function httpErrorCheck(response: Response): void { + if (response.status !== HTTP_OK_STATUS) { + throw new Error(`HTTP error while scraping Copper: [${JSON.stringify(response)}]`); + } +} +export class CopperSource { + private readonly _accessToken: string; + private readonly _userEmail: string; + private readonly _defaultHeaders: any; + private readonly _limiter: Bottleneck; + + constructor(maxConcurrentRequests: number, accessToken: string, userEmail: string) { + this._accessToken = accessToken; + this._userEmail = userEmail; + this._defaultHeaders = { + 'Content-Type': 'application/json', + 'X-PW-AccessToken': this._accessToken, + 'X-PW-Application': 'developer_api', + 'X-PW-UserEmail': this._userEmail, + }; + this._limiter = new Bottleneck({ + minTime: ONE_SECOND / maxConcurrentRequests, + reservoir: 30, + reservoirRefreshAmount: 30, + reservoirRefreshInterval: maxConcurrentRequests, + }); + } + + public async fetchNumberOfPagesAsync(endpoint: CopperEndpoint, searchParams?: CopperSearchParams): Promise<number> { + const resp = await this._limiter.schedule(() => + fetchAsync(COPPER_URI + endpoint, { + method: 'POST', + body: JSON.stringify({ ...DEFAULT_PAGINATION_PARAMS, ...searchParams }), + headers: this._defaultHeaders, + }), + ); + + httpErrorCheck(resp); + + // total number of records that match the request parameters + if (resp.headers.has('X-Pw-Total')) { + const totalRecords: number = parseInt(resp.headers.get('X-Pw-Total') as string, 10); // tslint:disable-line:custom-no-magic-numbers + return Math.ceil(totalRecords / DEFAULT_PAGINATION_PARAMS.page_size); + } else { + return 1; + } + } + public async fetchSearchResultsAsync<T extends CopperSearchResponse>( + endpoint: CopperEndpoint, + searchParams?: CopperSearchParams, + ): Promise<T[]> { + const request = { ...DEFAULT_PAGINATION_PARAMS, ...searchParams }; + const response = await this._limiter.schedule(() => + fetchAsync(COPPER_URI + endpoint, { + method: 'POST', + body: JSON.stringify(request), + headers: this._defaultHeaders, + }), + ); + httpErrorCheck(response); + const json: T[] = await response.json(); + return json; + } + + public async fetchActivityTypesAsync(): Promise<Map<CopperActivityTypeCategory, CopperActivityTypeResponse[]>> { + const response = await this._limiter.schedule(() => + fetchAsync(`${COPPER_URI}/activity_types`, { + method: 'GET', + headers: this._defaultHeaders, + }), + ); + httpErrorCheck(response); + return response.json(); + } + + public async fetchCustomFieldsAsync(): Promise<CopperCustomFieldResponse[]> { + const response = await this._limiter.schedule(() => + fetchAsync(`${COPPER_URI}/custom_field_definitions`, { + method: 'GET', + headers: this._defaultHeaders, + }), + ); + httpErrorCheck(response); + return response.json(); + } +} diff --git a/packages/pipeline/src/entities/copper_activity.ts b/packages/pipeline/src/entities/copper_activity.ts new file mode 100644 index 000000000..cbc034285 --- /dev/null +++ b/packages/pipeline/src/entities/copper_activity.ts @@ -0,0 +1,41 @@ +import { Column, Entity, Index, PrimaryColumn } from 'typeorm'; + +import { numberToBigIntTransformer } from '../utils'; + +@Entity({ name: 'copper_activities', schema: 'raw' }) +export class CopperActivity { + @PrimaryColumn({ type: 'bigint', transformer: numberToBigIntTransformer }) + public id!: number; + + @Index() + @Column({ name: 'parent_id', type: 'bigint', transformer: numberToBigIntTransformer }) + public parentId!: number; + @Column({ name: 'parent_type', type: 'varchar' }) + public parentType!: string; + + // join with CopperActivityType + @Index() + @Column({ name: 'type_id', type: 'bigint', transformer: numberToBigIntTransformer }) + public typeId!: number; + @Column({ name: 'type_category', type: 'varchar' }) + public typeCategory!: string; + @Column({ name: 'type_name', type: 'varchar', nullable: true }) + public typeName?: string; + + @Column({ name: 'user_id', type: 'bigint', transformer: numberToBigIntTransformer }) + public userId!: number; + @Column({ name: 'old_value_id', type: 'bigint', nullable: true, transformer: numberToBigIntTransformer }) + public oldValueId?: number; + @Column({ name: 'old_value_name', type: 'varchar', nullable: true }) + public oldValueName?: string; + @Column({ name: 'new_value_id', type: 'bigint', nullable: true, transformer: numberToBigIntTransformer }) + public newValueId?: number; + @Column({ name: 'new_value_name', type: 'varchar', nullable: true }) + public newValueName?: string; + + @Index() + @Column({ name: 'date_created', type: 'bigint', transformer: numberToBigIntTransformer }) + public dateCreated!: number; + @PrimaryColumn({ name: 'date_modified', type: 'bigint', transformer: numberToBigIntTransformer }) + public dateModified!: number; +} diff --git a/packages/pipeline/src/entities/copper_activity_type.ts b/packages/pipeline/src/entities/copper_activity_type.ts new file mode 100644 index 000000000..8fb2dcf70 --- /dev/null +++ b/packages/pipeline/src/entities/copper_activity_type.ts @@ -0,0 +1,17 @@ +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +import { numberToBigIntTransformer } from '../utils'; + +@Entity({ name: 'copper_activity_types', schema: 'raw' }) +export class CopperActivityType { + @PrimaryColumn({ type: 'bigint', transformer: numberToBigIntTransformer }) + public id!: number; + @Column({ name: 'category', type: 'varchar' }) + public category!: string; + @Column({ name: 'name', type: 'varchar' }) + public name!: string; + @Column({ name: 'is_disabled', type: 'boolean', nullable: true }) + public isDisabled?: boolean; + @Column({ name: 'count_as_interaction', type: 'boolean', nullable: true }) + public countAsInteraction?: boolean; +} diff --git a/packages/pipeline/src/entities/copper_custom_field.ts b/packages/pipeline/src/entities/copper_custom_field.ts new file mode 100644 index 000000000..f23f6ab22 --- /dev/null +++ b/packages/pipeline/src/entities/copper_custom_field.ts @@ -0,0 +1,15 @@ +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +import { numberToBigIntTransformer } from '../utils'; + +@Entity({ name: 'copper_custom_fields', schema: 'raw' }) +export class CopperCustomField { + @PrimaryColumn({ type: 'bigint', transformer: numberToBigIntTransformer }) + public id!: number; + @Column({ name: 'data_type', type: 'varchar' }) + public dataType!: string; + @Column({ name: 'field_type', type: 'varchar', nullable: true }) + public fieldType?: string; + @Column({ name: 'name', type: 'varchar' }) + public name!: string; +} diff --git a/packages/pipeline/src/entities/copper_lead.ts b/packages/pipeline/src/entities/copper_lead.ts new file mode 100644 index 000000000..c51ccd761 --- /dev/null +++ b/packages/pipeline/src/entities/copper_lead.ts @@ -0,0 +1,38 @@ +import { Column, Entity, Index, PrimaryColumn } from 'typeorm'; + +import { numberToBigIntTransformer } from '../utils'; + +@Entity({ name: 'copper_leads', schema: 'raw' }) +export class CopperLead { + @PrimaryColumn({ type: 'bigint', transformer: numberToBigIntTransformer }) + public id!: number; + + @Column({ name: 'name', type: 'varchar', nullable: true }) + public name?: string; + @Column({ name: 'first_name', type: 'varchar', nullable: true }) + public firstName?: string; + @Column({ name: 'last_name', type: 'varchar', nullable: true }) + public lastName?: string; + @Column({ name: 'middle_name', type: 'varchar', nullable: true }) + public middleName?: string; + @Column({ name: 'assignee_id', type: 'bigint', transformer: numberToBigIntTransformer, nullable: true }) + public assigneeId?: number; + @Column({ name: 'company_name', type: 'varchar', nullable: true }) + public companyName?: string; + @Column({ name: 'customer_source_id', type: 'bigint', transformer: numberToBigIntTransformer, nullable: true }) + public customerSourceId?: number; + @Column({ name: 'monetary_value', type: 'integer', nullable: true }) + public monetaryValue?: number; + @Column({ name: 'status', type: 'varchar' }) + public status!: string; + @Column({ name: 'status_id', type: 'bigint', transformer: numberToBigIntTransformer }) + public statusId!: number; + @Column({ name: 'title', type: 'varchar', nullable: true }) + public title?: string; + + @Index() + @Column({ name: 'date_created', type: 'bigint', transformer: numberToBigIntTransformer }) + public dateCreated!: number; + @PrimaryColumn({ name: 'date_modified', type: 'bigint', transformer: numberToBigIntTransformer }) + public dateModified!: number; +} diff --git a/packages/pipeline/src/entities/copper_opportunity.ts b/packages/pipeline/src/entities/copper_opportunity.ts new file mode 100644 index 000000000..e12bd69ce --- /dev/null +++ b/packages/pipeline/src/entities/copper_opportunity.ts @@ -0,0 +1,45 @@ +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +import { numberToBigIntTransformer } from '../utils'; + +@Entity({ name: 'copper_opportunities', schema: 'raw' }) +export class CopperOpportunity { + @PrimaryColumn({ name: 'id', type: 'bigint', transformer: numberToBigIntTransformer }) + public id!: number; + @Column({ name: 'name', type: 'varchar' }) + public name!: string; + @Column({ name: 'assignee_id', nullable: true, type: 'bigint', transformer: numberToBigIntTransformer }) + public assigneeId?: number; + @Column({ name: 'close_date', nullable: true, type: 'varchar' }) + public closeDate?: string; + @Column({ name: 'company_id', nullable: true, type: 'bigint', transformer: numberToBigIntTransformer }) + public companyId?: number; + @Column({ name: 'company_name', nullable: true, type: 'varchar' }) + public companyName?: string; + @Column({ name: 'customer_source_id', nullable: true, type: 'bigint', transformer: numberToBigIntTransformer }) + public customerSourceId?: number; + @Column({ name: 'loss_reason_id', nullable: true, type: 'bigint', transformer: numberToBigIntTransformer }) + public lossReasonId?: number; + @Column({ name: 'pipeline_id', type: 'bigint', transformer: numberToBigIntTransformer }) + public pipelineId!: number; + @Column({ name: 'pipeline_stage_id', type: 'bigint', transformer: numberToBigIntTransformer }) + public pipelineStageId!: number; + @Column({ name: 'primary_contact_id', nullable: true, type: 'bigint', transformer: numberToBigIntTransformer }) + public primaryContactId?: number; + @Column({ name: 'priority', nullable: true, type: 'varchar' }) + public priority?: string; + @Column({ name: 'status', type: 'varchar' }) + public status!: string; + @Column({ name: 'interaction_count', type: 'bigint', transformer: numberToBigIntTransformer }) + public interactionCount!: number; + @Column({ name: 'monetary_value', nullable: true, type: 'integer' }) + public monetaryValue?: number; + @Column({ name: 'win_probability', nullable: true, type: 'integer' }) + public winProbability?: number; + @Column({ name: 'date_created', type: 'bigint', transformer: numberToBigIntTransformer }) + public dateCreated!: number; + @PrimaryColumn({ name: 'date_modified', type: 'bigint', transformer: numberToBigIntTransformer }) + public dateModified!: number; + @Column({ name: 'custom_fields', type: 'jsonb' }) + public customFields!: { [key: number]: number }; +} diff --git a/packages/pipeline/src/entities/index.ts b/packages/pipeline/src/entities/index.ts index cc3de78bb..27c153c07 100644 --- a/packages/pipeline/src/entities/index.ts +++ b/packages/pipeline/src/entities/index.ts @@ -16,4 +16,10 @@ export { TokenOrderbookSnapshot } from './token_order'; export { Transaction } from './transaction'; export { ERC20ApprovalEvent } from './erc20_approval_event'; +export { CopperLead } from './copper_lead'; +export { CopperActivity } from './copper_activity'; +export { CopperOpportunity } from './copper_opportunity'; +export { CopperActivityType } from './copper_activity_type'; +export { CopperCustomField } from './copper_custom_field'; + export type ExchangeEvent = ExchangeFillEvent | ExchangeCancelEvent | ExchangeCancelUpToEvent; diff --git a/packages/pipeline/src/entities/token_order.ts b/packages/pipeline/src/entities/token_order.ts index 4b8f0abc3..2709747cb 100644 --- a/packages/pipeline/src/entities/token_order.ts +++ b/packages/pipeline/src/entities/token_order.ts @@ -1,7 +1,6 @@ import { BigNumber } from '@0x/utils'; import { Column, Entity, PrimaryColumn } from 'typeorm'; -import { OrderType } from '../types'; import { bigNumberTransformer, numberToBigIntTransformer } from '../utils'; @Entity({ name: 'token_orderbook_snapshots', schema: 'raw' }) @@ -11,7 +10,7 @@ export class TokenOrderbookSnapshot { @PrimaryColumn({ name: 'source' }) public source!: string; @PrimaryColumn({ name: 'order_type' }) - public orderType!: OrderType; + public orderType!: string; @PrimaryColumn({ name: 'price', type: 'numeric', transformer: bigNumberTransformer }) public price!: BigNumber; @PrimaryColumn({ name: 'base_asset_symbol' }) diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts index fe11d81d5..2700714cd 100644 --- a/packages/pipeline/src/ormconfig.ts +++ b/packages/pipeline/src/ormconfig.ts @@ -2,6 +2,11 @@ import { ConnectionOptions } from 'typeorm'; import { Block, + CopperActivity, + CopperActivityType, + CopperCustomField, + CopperLead, + CopperOpportunity, DexTrade, ERC20ApprovalEvent, ExchangeCancelEvent, @@ -18,6 +23,11 @@ import { const entities = [ Block, + CopperOpportunity, + CopperActivity, + CopperActivityType, + CopperCustomField, + CopperLead, DexTrade, ExchangeCancelEvent, ExchangeCancelUpToEvent, diff --git a/packages/pipeline/src/parsers/copper/index.ts b/packages/pipeline/src/parsers/copper/index.ts new file mode 100644 index 000000000..6c0c5abd5 --- /dev/null +++ b/packages/pipeline/src/parsers/copper/index.ts @@ -0,0 +1,259 @@ +import * as R from 'ramda'; + +import { CopperActivity, CopperActivityType, CopperCustomField, CopperLead, CopperOpportunity } from '../../entities'; + +const ONE_SECOND = 1000; +export type CopperSearchResponse = CopperLeadResponse | CopperActivityResponse | CopperOpportunityResponse; +export interface CopperLeadResponse { + id: number; + name?: string; + first_name?: string; + last_name?: string; + middle_name?: string; + assignee_id?: number; + company_name?: string; + customer_source_id?: number; + monetary_value?: number; + status: string; + status_id: number; + title?: string; + date_created: number; // in seconds + date_modified: number; // in seconds +} + +export interface CopperActivityResponse { + id: number; + parent: CopperActivityParentResponse; + type: CopperActivityTypeResponse; + user_id: number; + activity_date: number; + old_value: CopperActivityValueResponse; + new_value: CopperActivityValueResponse; + date_created: number; // in seconds + date_modified: number; // in seconds +} + +export interface CopperActivityValueResponse { + id: number; + name: string; +} +export interface CopperActivityParentResponse { + id: number; + type: string; +} + +// custom activity types +export enum CopperActivityTypeCategory { + user = 'user', + system = 'system', +} +export interface CopperActivityTypeResponse { + id: number; + category: CopperActivityTypeCategory; + name: string; + is_disabled?: boolean; + count_as_interaction?: boolean; +} + +export interface CopperOpportunityResponse { + id: number; + name: string; + assignee_id?: number; + close_date?: string; + company_id?: number; + company_name?: string; + customer_source_id?: number; + loss_reason_id?: number; + pipeline_id: number; + pipeline_stage_id: number; + primary_contact_id?: number; + priority?: string; + status: string; + tags: string[]; + interaction_count: number; + monetary_value?: number; + win_probability?: number; + date_created: number; // in seconds + date_modified: number; // in seconds + custom_fields: CopperNestedCustomFieldResponse[]; +} +interface CopperNestedCustomFieldResponse { + custom_field_definition_id: number; + value: number | number[] | null; +} +// custom fields +export enum CopperCustomFieldType { + String = 'String', + Text = 'Text', + Dropdown = 'Dropdown', + MultiSelect = 'MultiSelect', // not in API documentation but shows up in results + Date = 'Date', + Checkbox = 'Checkbox', + Float = 'Float', + URL = 'URL', + Percentage = 'Percentage', + Currency = 'Currency', + Connect = 'Connect', +} +export interface CopperCustomFieldOptionResponse { + id: number; + name: string; +} +export interface CopperCustomFieldResponse { + id: number; + name: string; + data_type: CopperCustomFieldType; + options?: CopperCustomFieldOptionResponse[]; +} +/** + * Parse response from Copper API /search/leads/ + * + * @param leads - The array of leads returned from the API + * @returns Returns an array of Copper Lead entities + */ +export function parseLeads(leads: CopperLeadResponse[]): CopperLead[] { + return leads.map(lead => { + const entity = new CopperLead(); + entity.id = lead.id; + entity.name = lead.name || undefined; + entity.firstName = lead.first_name || undefined; + entity.lastName = lead.last_name || undefined; + entity.middleName = lead.middle_name || undefined; + entity.assigneeId = lead.assignee_id || undefined; + entity.companyName = lead.company_name || undefined; + entity.customerSourceId = lead.customer_source_id || undefined; + entity.monetaryValue = lead.monetary_value || undefined; + entity.status = lead.status; + entity.statusId = lead.status_id; + entity.title = lead.title || undefined; + entity.dateCreated = lead.date_created * ONE_SECOND; + entity.dateModified = lead.date_modified * ONE_SECOND; + return entity; + }); +} + +/** + * Parse response from Copper API /search/activities/ + * + * @param activities - The array of activities returned from the API + * @returns Returns an array of Copper Activity entities + */ +export function parseActivities(activities: CopperActivityResponse[]): CopperActivity[] { + return activities.map(activity => { + const entity = new CopperActivity(); + entity.id = activity.id; + + entity.parentId = activity.parent.id; + entity.parentType = activity.parent.type; + + entity.typeId = activity.type.id; + entity.typeCategory = activity.type.category.toString(); + entity.typeName = activity.type.name; + + entity.userId = activity.user_id; + entity.dateCreated = activity.date_created * ONE_SECOND; + entity.dateModified = activity.date_modified * ONE_SECOND; + + // nested nullable fields + entity.oldValueId = R.path(['old_value', 'id'], activity); + entity.oldValueName = R.path(['old_value', 'name'], activity); + entity.newValueId = R.path(['new_value', 'id'], activity); + entity.newValueName = R.path(['new_value', 'name'], activity); + + return entity; + }); +} + +/** + * Parse response from Copper API /search/opportunities/ + * + * @param opportunities - The array of opportunities returned from the API + * @returns Returns an array of Copper Opportunity entities + */ +export function parseOpportunities(opportunities: CopperOpportunityResponse[]): CopperOpportunity[] { + return opportunities.map(opp => { + const customFields: { [key: number]: number } = opp.custom_fields + .filter(f => f.value !== null) + .map(f => ({ + ...f, + value: ([] as number[]).concat(f.value || []), // normalise all values to number[] + })) + .map(f => f.value.map(val => [f.custom_field_definition_id, val] as [number, number])) // pair each value with the custom_field_definition_id + .reduce((acc, pair) => acc.concat(pair)) // flatten + .reduce<{ [key: number]: number }>((obj, [key, value]) => { + // transform into object literal + obj[key] = value; + return obj; + }, {}); + + const entity = new CopperOpportunity(); + entity.id = opp.id; + entity.name = opp.name; + entity.assigneeId = opp.assignee_id || undefined; + entity.closeDate = opp.close_date || undefined; + entity.companyId = opp.company_id || undefined; + entity.companyName = opp.company_name || undefined; + entity.customerSourceId = opp.customer_source_id || undefined; + entity.lossReasonId = opp.loss_reason_id || undefined; + entity.pipelineId = opp.pipeline_id; + entity.pipelineStageId = opp.pipeline_stage_id; + entity.primaryContactId = opp.primary_contact_id || undefined; + entity.priority = opp.priority || undefined; + entity.status = opp.status; + entity.interactionCount = opp.interaction_count; + entity.monetaryValue = opp.monetary_value || undefined; + entity.winProbability = opp.win_probability === null ? undefined : opp.win_probability; + entity.dateCreated = opp.date_created * ONE_SECOND; + entity.dateModified = opp.date_modified * ONE_SECOND; + entity.customFields = customFields; + return entity; + }); +} + +/** + * Parse response from Copper API /activity_types/ + * + * @param activityTypeResponse - Activity Types response from the API, keyed by "user" or "system" + * @returns Returns an array of Copper Activity Type entities + */ +export function parseActivityTypes( + activityTypeResponse: Map<CopperActivityTypeCategory, CopperActivityTypeResponse[]>, +): CopperActivityType[] { + const values: CopperActivityTypeResponse[] = R.flatten(Object.values(activityTypeResponse)); + return values.map(activityType => ({ + id: activityType.id, + name: activityType.name, + category: activityType.category.toString(), + isDisabled: activityType.is_disabled, + countAsInteraction: activityType.count_as_interaction, + })); +} + +/** + * Parse response from Copper API /custom_field_definitions/ + * + * @param customFieldResponse - array of custom field definitions returned from the API, consisting of top-level fields and nested fields + * @returns Returns an array of Copper Custom Field entities + */ +export function parseCustomFields(customFieldResponse: CopperCustomFieldResponse[]): CopperCustomField[] { + function parseTopLevelField(field: CopperCustomFieldResponse): CopperCustomField[] { + const topLevelField: CopperCustomField = { + id: field.id, + name: field.name, + dataType: field.data_type.toString(), + }; + + if (field.options !== undefined) { + const nestedFields: CopperCustomField[] = field.options.map(option => ({ + id: option.id, + name: option.name, + dataType: field.name, + fieldType: 'option', + })); + return nestedFields.concat(topLevelField); + } else { + return [topLevelField]; + } + } + return R.chain(parseTopLevelField, customFieldResponse); +} diff --git a/packages/pipeline/src/parsers/ddex_orders/index.ts b/packages/pipeline/src/parsers/ddex_orders/index.ts index d7b97efbe..eeb9c9d5b 100644 --- a/packages/pipeline/src/parsers/ddex_orders/index.ts +++ b/packages/pipeline/src/parsers/ddex_orders/index.ts @@ -23,8 +23,12 @@ export function parseDdexOrders( ): TokenOrder[] { const aggregatedBids = aggregateOrders(ddexOrderbook.bids); const aggregatedAsks = aggregateOrders(ddexOrderbook.asks); - const parsedBids = aggregatedBids.map(order => parseDdexOrder(ddexMarket, observedTimestamp, 'bid', source, order)); - const parsedAsks = aggregatedAsks.map(order => parseDdexOrder(ddexMarket, observedTimestamp, 'ask', source, order)); + const parsedBids = aggregatedBids.map(order => + parseDdexOrder(ddexMarket, observedTimestamp, OrderType.Bid, source, order), + ); + const parsedAsks = aggregatedAsks.map(order => + parseDdexOrder(ddexMarket, observedTimestamp, OrderType.Ask, source, order), + ); return parsedBids.concat(parsedAsks); } diff --git a/packages/pipeline/src/parsers/events/exchange_events.ts b/packages/pipeline/src/parsers/events/exchange_events.ts index e18106c75..9c4a5f89a 100644 --- a/packages/pipeline/src/parsers/events/exchange_events.ts +++ b/packages/pipeline/src/parsers/events/exchange_events.ts @@ -5,7 +5,7 @@ import { LogWithDecodedArgs } from 'ethereum-types'; import * as R from 'ramda'; import { ExchangeCancelEvent, ExchangeCancelUpToEvent, ExchangeFillEvent } from '../../entities'; -import { bigNumbertoStringOrNull } from '../../utils'; +import { bigNumbertoStringOrNull, convertAssetProxyIdToType } from '../../utils'; /** * Parses raw event logs for a fill event and returns an array of @@ -40,9 +40,7 @@ export const parseExchangeCancelUpToEvents: ( */ export function _convertToExchangeFillEvent(eventLog: LogWithDecodedArgs<ExchangeFillEventArgs>): ExchangeFillEvent { const makerAssetData = assetDataUtils.decodeAssetDataOrThrow(eventLog.args.makerAssetData); - const makerAssetType = makerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721'; const takerAssetData = assetDataUtils.decodeAssetDataOrThrow(eventLog.args.takerAssetData); - const takerAssetType = takerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721'; const exchangeFillEvent = new ExchangeFillEvent(); exchangeFillEvent.contractAddress = eventLog.address as string; exchangeFillEvent.blockNumber = eventLog.blockNumber as number; @@ -59,16 +57,24 @@ export function _convertToExchangeFillEvent(eventLog: LogWithDecodedArgs<Exchang exchangeFillEvent.takerFeePaid = eventLog.args.takerFeePaid; exchangeFillEvent.orderHash = eventLog.args.orderHash; exchangeFillEvent.rawMakerAssetData = eventLog.args.makerAssetData; - exchangeFillEvent.makerAssetType = makerAssetType; + // tslint:disable-next-line:no-unnecessary-type-assertion + exchangeFillEvent.makerAssetType = convertAssetProxyIdToType(makerAssetData.assetProxyId as AssetProxyId); exchangeFillEvent.makerAssetProxyId = makerAssetData.assetProxyId; - exchangeFillEvent.makerTokenAddress = makerAssetData.tokenAddress; + // HACK(abandeali1): this event schema currently does not support multiple maker/taker assets, so we store the first token address from the MultiAssetProxy assetData + exchangeFillEvent.makerTokenAddress = assetDataUtils.isMultiAssetData(makerAssetData) + ? assetDataUtils.decodeMultiAssetDataRecursively(eventLog.args.makerAssetData).nestedAssetData[0].tokenAddress + : makerAssetData.tokenAddress; // tslint has a false positive here. Type assertion is required. // tslint:disable-next-line:no-unnecessary-type-assertion exchangeFillEvent.makerTokenId = bigNumbertoStringOrNull((makerAssetData as ERC721AssetData).tokenId); exchangeFillEvent.rawTakerAssetData = eventLog.args.takerAssetData; - exchangeFillEvent.takerAssetType = takerAssetType; + // tslint:disable-next-line:no-unnecessary-type-assertion + exchangeFillEvent.takerAssetType = convertAssetProxyIdToType(takerAssetData.assetProxyId as AssetProxyId); exchangeFillEvent.takerAssetProxyId = takerAssetData.assetProxyId; - exchangeFillEvent.takerTokenAddress = takerAssetData.tokenAddress; + // HACK(abandeali1): this event schema currently does not support multiple maker/taker assets, so we store the first token address from the MultiAssetProxy assetData + exchangeFillEvent.takerTokenAddress = assetDataUtils.isMultiAssetData(takerAssetData) + ? assetDataUtils.decodeMultiAssetDataRecursively(eventLog.args.takerAssetData).nestedAssetData[0].tokenAddress + : takerAssetData.tokenAddress; // tslint:disable-next-line:no-unnecessary-type-assertion exchangeFillEvent.takerTokenId = bigNumbertoStringOrNull((takerAssetData as ERC721AssetData).tokenId); return exchangeFillEvent; @@ -83,9 +89,7 @@ export function _convertToExchangeCancelEvent( eventLog: LogWithDecodedArgs<ExchangeCancelEventArgs>, ): ExchangeCancelEvent { const makerAssetData = assetDataUtils.decodeAssetDataOrThrow(eventLog.args.makerAssetData); - const makerAssetType = makerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721'; const takerAssetData = assetDataUtils.decodeAssetDataOrThrow(eventLog.args.takerAssetData); - const takerAssetType = takerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721'; const exchangeCancelEvent = new ExchangeCancelEvent(); exchangeCancelEvent.contractAddress = eventLog.address as string; exchangeCancelEvent.blockNumber = eventLog.blockNumber as number; @@ -98,15 +102,23 @@ export function _convertToExchangeCancelEvent( exchangeCancelEvent.senderAddress = eventLog.args.senderAddress; exchangeCancelEvent.orderHash = eventLog.args.orderHash; exchangeCancelEvent.rawMakerAssetData = eventLog.args.makerAssetData; - exchangeCancelEvent.makerAssetType = makerAssetType; + // tslint:disable-next-line:no-unnecessary-type-assertion + exchangeCancelEvent.makerAssetType = convertAssetProxyIdToType(makerAssetData.assetProxyId as AssetProxyId); exchangeCancelEvent.makerAssetProxyId = makerAssetData.assetProxyId; - exchangeCancelEvent.makerTokenAddress = makerAssetData.tokenAddress; + // HACK(abandeali1): this event schema currently does not support multiple maker/taker assets, so we store the first token address from the MultiAssetProxy assetData + exchangeCancelEvent.makerTokenAddress = assetDataUtils.isMultiAssetData(makerAssetData) + ? assetDataUtils.decodeMultiAssetDataRecursively(eventLog.args.makerAssetData).nestedAssetData[0].tokenAddress + : makerAssetData.tokenAddress; // tslint:disable-next-line:no-unnecessary-type-assertion exchangeCancelEvent.makerTokenId = bigNumbertoStringOrNull((makerAssetData as ERC721AssetData).tokenId); exchangeCancelEvent.rawTakerAssetData = eventLog.args.takerAssetData; - exchangeCancelEvent.takerAssetType = takerAssetType; + // tslint:disable-next-line:no-unnecessary-type-assertion + exchangeCancelEvent.takerAssetType = convertAssetProxyIdToType(takerAssetData.assetProxyId as AssetProxyId); exchangeCancelEvent.takerAssetProxyId = takerAssetData.assetProxyId; - exchangeCancelEvent.takerTokenAddress = takerAssetData.tokenAddress; + // HACK(abandeali1): this event schema currently does not support multiple maker/taker assets, so we store the first token address from the MultiAssetProxy assetData + exchangeCancelEvent.takerTokenAddress = assetDataUtils.isMultiAssetData(takerAssetData) + ? assetDataUtils.decodeMultiAssetDataRecursively(eventLog.args.takerAssetData).nestedAssetData[0].tokenAddress + : takerAssetData.tokenAddress; // tslint:disable-next-line:no-unnecessary-type-assertion exchangeCancelEvent.takerTokenId = bigNumbertoStringOrNull((takerAssetData as ERC721AssetData).tokenId); return exchangeCancelEvent; diff --git a/packages/pipeline/src/parsers/idex_orders/index.ts b/packages/pipeline/src/parsers/idex_orders/index.ts index dfe27455c..14b871195 100644 --- a/packages/pipeline/src/parsers/idex_orders/index.ts +++ b/packages/pipeline/src/parsers/idex_orders/index.ts @@ -2,7 +2,7 @@ import { BigNumber } from '@0x/utils'; import { aggregateOrders } from '../utils'; -import { IdexOrder, IdexOrderbook, IdexOrderParam } from '../../data_sources/idex'; +import { IdexOrderbook, IdexOrderParam } from '../../data_sources/idex'; import { TokenOrderbookSnapshot as TokenOrder } from '../../entities'; import { OrderType } from '../../types'; @@ -21,7 +21,9 @@ export function parseIdexOrders(idexOrderbook: IdexOrderbook, observedTimestamp: const idexBidOrder = idexOrderbook.bids[0]; const parsedBids = aggregatedBids.length > 0 - ? aggregatedBids.map(order => parseIdexOrder(idexBidOrder.params, observedTimestamp, 'bid', source, order)) + ? aggregatedBids.map(order => + parseIdexOrder(idexBidOrder.params, observedTimestamp, OrderType.Bid, source, order), + ) : []; const aggregatedAsks = aggregateOrders(idexOrderbook.asks); @@ -29,7 +31,9 @@ export function parseIdexOrders(idexOrderbook: IdexOrderbook, observedTimestamp: const idexAskOrder = idexOrderbook.asks[0]; const parsedAsks = aggregatedAsks.length > 0 - ? aggregatedAsks.map(order => parseIdexOrder(idexAskOrder.params, observedTimestamp, 'ask', source, order)) + ? aggregatedAsks.map(order => + parseIdexOrder(idexAskOrder.params, observedTimestamp, OrderType.Ask, source, order), + ) : []; return parsedBids.concat(parsedAsks); } @@ -62,7 +66,7 @@ export function parseIdexOrder( tokenOrder.baseVolume = amount; tokenOrder.quoteVolume = price.times(amount); - if (orderType === 'bid') { + if (orderType === OrderType.Bid) { tokenOrder.baseAssetSymbol = idexOrderParam.buySymbol; tokenOrder.baseAssetAddress = idexOrderParam.tokenBuy; tokenOrder.quoteAssetSymbol = idexOrderParam.sellSymbol; diff --git a/packages/pipeline/src/parsers/oasis_orders/index.ts b/packages/pipeline/src/parsers/oasis_orders/index.ts index 13997f31b..b71fb65b9 100644 --- a/packages/pipeline/src/parsers/oasis_orders/index.ts +++ b/packages/pipeline/src/parsers/oasis_orders/index.ts @@ -23,13 +23,13 @@ export function parseOasisOrders( observedTimestamp: number, source: string, ): TokenOrder[] { - const aggregatedBids = aggregateOrders(R.filter(R.propEq('act', 'bid'), oasisOrderbook)); - const aggregatedAsks = aggregateOrders(R.filter(R.propEq('act', 'ask'), oasisOrderbook)); + const aggregatedBids = aggregateOrders(R.filter(R.propEq('act', OrderType.Bid), oasisOrderbook)); + const aggregatedAsks = aggregateOrders(R.filter(R.propEq('act', OrderType.Ask), oasisOrderbook)); const parsedBids = aggregatedBids.map(order => - parseOasisOrder(oasisMarket, observedTimestamp, 'bid', source, order), + parseOasisOrder(oasisMarket, observedTimestamp, OrderType.Bid, source, order), ); const parsedAsks = aggregatedAsks.map(order => - parseOasisOrder(oasisMarket, observedTimestamp, 'ask', source, order), + parseOasisOrder(oasisMarket, observedTimestamp, OrderType.Ask, source, order), ); return parsedBids.concat(parsedAsks); } diff --git a/packages/pipeline/src/parsers/paradex_orders/index.ts b/packages/pipeline/src/parsers/paradex_orders/index.ts index 5ceeb64a4..85990dae4 100644 --- a/packages/pipeline/src/parsers/paradex_orders/index.ts +++ b/packages/pipeline/src/parsers/paradex_orders/index.ts @@ -21,10 +21,10 @@ export function parseParadexOrders( source: string, ): TokenOrder[] { const parsedBids = paradexOrderbookResponse.bids.map(order => - parseParadexOrder(paradexMarket, observedTimestamp, 'bid', source, order), + parseParadexOrder(paradexMarket, observedTimestamp, OrderType.Bid, source, order), ); const parsedAsks = paradexOrderbookResponse.asks.map(order => - parseParadexOrder(paradexMarket, observedTimestamp, 'ask', source, order), + parseParadexOrder(paradexMarket, observedTimestamp, OrderType.Ask, source, order), ); return parsedBids.concat(parsedAsks); } diff --git a/packages/pipeline/src/parsers/sra_orders/index.ts b/packages/pipeline/src/parsers/sra_orders/index.ts index ef8901e40..13fe632a4 100644 --- a/packages/pipeline/src/parsers/sra_orders/index.ts +++ b/packages/pipeline/src/parsers/sra_orders/index.ts @@ -4,7 +4,7 @@ import { AssetProxyId, ERC721AssetData } from '@0x/types'; import * as R from 'ramda'; import { SraOrder } from '../../entities'; -import { bigNumbertoStringOrNull } from '../../utils'; +import { bigNumbertoStringOrNull, convertAssetProxyIdToType } from '../../utils'; /** * Parses a raw order response from an SRA endpoint and returns an array of @@ -22,9 +22,7 @@ export function parseSraOrders(rawOrdersResponse: OrdersResponse): SraOrder[] { export function _convertToEntity(apiOrder: APIOrder): SraOrder { // TODO(albrow): refactor out common asset data decoding code. const makerAssetData = assetDataUtils.decodeAssetDataOrThrow(apiOrder.order.makerAssetData); - const makerAssetType = makerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721'; const takerAssetData = assetDataUtils.decodeAssetDataOrThrow(apiOrder.order.takerAssetData); - const takerAssetType = takerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721'; const sraOrder = new SraOrder(); sraOrder.exchangeAddress = apiOrder.order.exchangeAddress; @@ -43,16 +41,24 @@ export function _convertToEntity(apiOrder: APIOrder): SraOrder { sraOrder.signature = apiOrder.order.signature; sraOrder.rawMakerAssetData = apiOrder.order.makerAssetData; - sraOrder.makerAssetType = makerAssetType; + // tslint:disable-next-line:no-unnecessary-type-assertion + sraOrder.makerAssetType = convertAssetProxyIdToType(makerAssetData.assetProxyId as AssetProxyId); sraOrder.makerAssetProxyId = makerAssetData.assetProxyId; - sraOrder.makerTokenAddress = makerAssetData.tokenAddress; + // HACK(abandeali1): this event schema currently does not support multiple maker/taker assets, so we store the first token address from the MultiAssetProxy assetData + sraOrder.makerTokenAddress = assetDataUtils.isMultiAssetData(makerAssetData) + ? assetDataUtils.decodeMultiAssetDataRecursively(apiOrder.order.makerAssetData).nestedAssetData[0].tokenAddress + : makerAssetData.tokenAddress; // tslint has a false positive here. Type assertion is required. // tslint:disable-next-line:no-unnecessary-type-assertion sraOrder.makerTokenId = bigNumbertoStringOrNull((makerAssetData as ERC721AssetData).tokenId); sraOrder.rawTakerAssetData = apiOrder.order.takerAssetData; - sraOrder.takerAssetType = takerAssetType; + // tslint:disable-next-line:no-unnecessary-type-assertion + sraOrder.takerAssetType = convertAssetProxyIdToType(takerAssetData.assetProxyId as AssetProxyId); sraOrder.takerAssetProxyId = takerAssetData.assetProxyId; - sraOrder.takerTokenAddress = takerAssetData.tokenAddress; + // HACK(abandeali1): this event schema currently does not support multiple maker/taker assets, so we store the first token address from the MultiAssetProxy assetData + sraOrder.takerTokenAddress = assetDataUtils.isMultiAssetData(takerAssetData) + ? assetDataUtils.decodeMultiAssetDataRecursively(apiOrder.order.takerAssetData).nestedAssetData[0].tokenAddress + : takerAssetData.tokenAddress; // tslint:disable-next-line:no-unnecessary-type-assertion sraOrder.takerTokenId = bigNumbertoStringOrNull((takerAssetData as ERC721AssetData).tokenId); diff --git a/packages/pipeline/src/scripts/pull_copper.ts b/packages/pipeline/src/scripts/pull_copper.ts new file mode 100644 index 000000000..69814f209 --- /dev/null +++ b/packages/pipeline/src/scripts/pull_copper.ts @@ -0,0 +1,129 @@ +// tslint:disable:no-console +import * as R from 'ramda'; +import { Connection, ConnectionOptions, createConnection, Repository } from 'typeorm'; + +import { CopperEndpoint, CopperSearchParams, CopperSource } from '../data_sources/copper'; +import { CopperActivity, CopperActivityType, CopperCustomField, CopperLead, CopperOpportunity } from '../entities'; +import * as ormConfig from '../ormconfig'; +import { + CopperSearchResponse, + parseActivities, + parseActivityTypes, + parseCustomFields, + parseLeads, + parseOpportunities, +} from '../parsers/copper'; +import { handleError } from '../utils'; +const ONE_SECOND = 1000; +const COPPER_RATE_LIMIT = 10; +let connection: Connection; + +(async () => { + connection = await createConnection(ormConfig as ConnectionOptions); + + const accessToken = process.env.COPPER_ACCESS_TOKEN; + const userEmail = process.env.COPPER_USER_EMAIL; + if (accessToken === undefined || userEmail === undefined) { + throw new Error('Missing required env var: COPPER_ACCESS_TOKEN and/or COPPER_USER_EMAIL'); + } + const source = new CopperSource(COPPER_RATE_LIMIT, accessToken, userEmail); + + const fetchPromises = [ + fetchAndSaveLeadsAsync(source), + fetchAndSaveOpportunitiesAsync(source), + fetchAndSaveActivitiesAsync(source), + fetchAndSaveCustomFieldsAsync(source), + fetchAndSaveActivityTypesAsync(source), + ]; + fetchPromises.forEach(async fn => { + await fn; + }); +})().catch(handleError); + +async function fetchAndSaveLeadsAsync(source: CopperSource): Promise<void> { + const repository = connection.getRepository(CopperLead); + const startTime = await getMaxAsync(connection, 'date_modified', 'raw.copper_leads'); + console.log(`Fetching Copper leads starting from ${startTime}...`); + await fetchAndSaveAsync(CopperEndpoint.Leads, source, startTime, {}, parseLeads, repository); +} + +async function fetchAndSaveOpportunitiesAsync(source: CopperSource): Promise<void> { + const repository = connection.getRepository(CopperOpportunity); + const startTime = await getMaxAsync(connection, 'date_modified', 'raw.copper_opportunities'); + console.log(`Fetching Copper opportunities starting from ${startTime}...`); + await fetchAndSaveAsync( + CopperEndpoint.Opportunities, + source, + startTime, + { sort_by: 'name' }, + parseOpportunities, + repository, + ); +} + +async function fetchAndSaveActivitiesAsync(source: CopperSource): Promise<void> { + const repository = connection.getRepository(CopperActivity); + const startTime = await getMaxAsync(connection, 'date_modified', 'raw.copper_activities'); + const searchParams = { + minimum_activity_date: Math.floor(startTime / ONE_SECOND), + }; + console.log(`Fetching Copper activities starting from ${startTime}...`); + await fetchAndSaveAsync(CopperEndpoint.Activities, source, startTime, searchParams, parseActivities, repository); +} + +async function getMaxAsync(conn: Connection, sortColumn: string, tableName: string): Promise<number> { + const queryResult = await conn.query(`SELECT MAX(${sortColumn}) as _max from ${tableName};`); + if (R.isEmpty(queryResult)) { + return 0; + } else { + return queryResult[0]._max; + } +} + +// (Xianny): Copper API doesn't allow queries to filter by date. To ensure that we are filling in ascending chronological +// order and not missing any records, we are scraping all available pages. If Copper data gets larger, +// it would make sense to search for and start filling from the first page that contains a new record. +// This search would increase our network calls and is not efficient to implement with our current small volume +// of Copper records. +async function fetchAndSaveAsync<T extends CopperSearchResponse, E>( + endpoint: CopperEndpoint, + source: CopperSource, + startTime: number, + searchParams: CopperSearchParams, + parseFn: (recs: T[]) => E[], + repository: Repository<E>, +): Promise<void> { + let saved = 0; + const numPages = await source.fetchNumberOfPagesAsync(endpoint); + try { + for (let i = numPages; i > 0; i--) { + console.log(`Fetching page ${i}/${numPages} of ${endpoint}...`); + const raw = await source.fetchSearchResultsAsync<T>(endpoint, { + ...searchParams, + page_number: i, + }); + const newRecords = raw.filter(rec => rec.date_modified * ONE_SECOND > startTime); + const parsed = parseFn(newRecords); + await repository.save<any>(parsed); + saved += newRecords.length; + } + } catch (err) { + console.log(`Error fetching ${endpoint}, stopping: ${err.stack}`); + } finally { + console.log(`Saved ${saved} items from ${endpoint}, done.`); + } +} + +async function fetchAndSaveActivityTypesAsync(source: CopperSource): Promise<void> { + console.log(`Fetching Copper activity types...`); + const activityTypes = await source.fetchActivityTypesAsync(); + const repository = connection.getRepository(CopperActivityType); + await repository.save(parseActivityTypes(activityTypes)); +} + +async function fetchAndSaveCustomFieldsAsync(source: CopperSource): Promise<void> { + console.log(`Fetching Copper custom fields...`); + const customFields = await source.fetchCustomFieldsAsync(); + const repository = connection.getRepository(CopperCustomField); + await repository.save(parseCustomFields(customFields)); +} diff --git a/packages/pipeline/src/scripts/pull_missing_blocks.ts b/packages/pipeline/src/scripts/pull_missing_blocks.ts index bb5385126..ced9d99eb 100644 --- a/packages/pipeline/src/scripts/pull_missing_blocks.ts +++ b/packages/pipeline/src/scripts/pull_missing_blocks.ts @@ -14,20 +14,29 @@ import { handleError, INFURA_ROOT_URL } from '../utils'; // Number of blocks to save at once. const BATCH_SAVE_SIZE = 1000; // Maximum number of requests to send at once. -const MAX_CONCURRENCY = 10; +const MAX_CONCURRENCY = 20; // Maximum number of blocks to query for at once. This is also the maximum // number of blocks we will hold in memory prior to being saved to the database. const MAX_BLOCKS_PER_QUERY = 1000; let connection: Connection; +const tablesWithMissingBlocks = [ + 'raw.exchange_fill_events', + 'raw.exchange_cancel_events', + 'raw.exchange_cancel_up_to_events', + 'raw.erc20_approval_events', +]; + (async () => { connection = await createConnection(ormConfig as ConnectionOptions); const provider = web3Factory.getRpcProvider({ rpcUrl: INFURA_ROOT_URL, }); const web3Source = new Web3Source(provider); - await getAllMissingBlocksAsync(web3Source); + for (const tableName of tablesWithMissingBlocks) { + await getAllMissingBlocksAsync(web3Source, tableName); + } process.exit(0); })().catch(handleError); @@ -35,10 +44,11 @@ interface MissingBlocksResponse { block_number: string; } -async function getAllMissingBlocksAsync(web3Source: Web3Source): Promise<void> { +async function getAllMissingBlocksAsync(web3Source: Web3Source, tableName: string): Promise<void> { const blocksRepository = connection.getRepository(Block); while (true) { - const blockNumbers = await getMissingBlockNumbersAsync(); + console.log(`Checking for missing blocks in ${tableName}...`); + const blockNumbers = await getMissingBlockNumbersAsync(tableName); if (blockNumbers.length === 0) { // There are no more missing blocks. We're done. break; @@ -46,24 +56,14 @@ async function getAllMissingBlocksAsync(web3Source: Web3Source): Promise<void> { await getAndSaveBlocksAsync(web3Source, blocksRepository, blockNumbers); } const totalBlocks = await blocksRepository.count(); - console.log(`Done saving blocks. There are now ${totalBlocks} total blocks.`); + console.log(`Done saving blocks for ${tableName}. There are now ${totalBlocks} total blocks.`); } -async function getMissingBlockNumbersAsync(): Promise<number[]> { - // Note(albrow): The easiest way to get all the blocks we need is to - // consider all the events tables together in a single query. If this query - // gets too slow, we should consider re-architecting so that we can work on - // getting the blocks for one type of event at a time. +async function getMissingBlockNumbersAsync(tableName: string): Promise<number[]> { + // This query returns up to `MAX_BLOCKS_PER_QUERY` distinct block numbers + // which are present in `tableName` but not in `raw.blocks`. const response = (await connection.query( - `WITH all_events AS ( - SELECT block_number FROM raw.exchange_fill_events - UNION SELECT block_number FROM raw.exchange_cancel_events - UNION SELECT block_number FROM raw.exchange_cancel_up_to_events - UNION SELECT block_number FROM raw.erc20_approval_events - ) - SELECT DISTINCT(block_number) FROM all_events - WHERE block_number NOT IN (SELECT number FROM raw.blocks) - ORDER BY block_number ASC LIMIT $1`, + `SELECT DISTINCT(block_number) FROM ${tableName} LEFT JOIN raw.blocks ON ${tableName}.block_number = raw.blocks.number WHERE number IS NULL LIMIT $1;`, [MAX_BLOCKS_PER_QUERY], )) as MissingBlocksResponse[]; const blockNumberStrings = R.pluck('block_number', response); @@ -86,4 +86,5 @@ async function getAndSaveBlocksAsync( const blocks = R.map(parseBlock, rawBlocks); console.log(`Saving ${blocks.length} blocks...`); await blocksRepository.save(blocks, { chunk: Math.ceil(blocks.length / BATCH_SAVE_SIZE) }); + console.log('Done saving this batch of blocks'); } diff --git a/packages/pipeline/src/types.ts b/packages/pipeline/src/types.ts index e02b42a40..5f2121807 100644 --- a/packages/pipeline/src/types.ts +++ b/packages/pipeline/src/types.ts @@ -1,2 +1,9 @@ -export type AssetType = 'erc20' | 'erc721'; -export type OrderType = 'bid' | 'ask'; +export enum AssetType { + ERC20 = 'erc20', + ERC721 = 'erc721', + MultiAsset = 'multiAsset', +} +export enum OrderType { + Bid = 'bid', + Ask = 'ask', +} diff --git a/packages/pipeline/src/utils/transformers/asset_proxy_id_types.ts b/packages/pipeline/src/utils/transformers/asset_proxy_id_types.ts new file mode 100644 index 000000000..2cd05a616 --- /dev/null +++ b/packages/pipeline/src/utils/transformers/asset_proxy_id_types.ts @@ -0,0 +1,20 @@ +import { AssetProxyId } from '@0x/types'; + +import { AssetType } from '../../types'; + +/** + * Converts an assetProxyId to its string equivalent + * @param assetProxyId Id of AssetProxy + */ +export function convertAssetProxyIdToType(assetProxyId: AssetProxyId): AssetType { + switch (assetProxyId) { + case AssetProxyId.ERC20: + return AssetType.ERC20; + case AssetProxyId.ERC721: + return AssetType.ERC721; + case AssetProxyId.MultiAsset: + return AssetType.MultiAsset; + default: + throw new Error(`${assetProxyId} not a supported assetProxyId`); + } +} diff --git a/packages/pipeline/src/utils/transformers/index.ts b/packages/pipeline/src/utils/transformers/index.ts index 232c1c5de..31a4c9223 100644 --- a/packages/pipeline/src/utils/transformers/index.ts +++ b/packages/pipeline/src/utils/transformers/index.ts @@ -1,2 +1,3 @@ export * from './big_number'; export * from './number_to_bigint'; +export * from './asset_proxy_id_types'; diff --git a/packages/pipeline/src/utils/transformers/number_to_bigint.ts b/packages/pipeline/src/utils/transformers/number_to_bigint.ts index 85560c1f0..9736d7c18 100644 --- a/packages/pipeline/src/utils/transformers/number_to_bigint.ts +++ b/packages/pipeline/src/utils/transformers/number_to_bigint.ts @@ -9,8 +9,12 @@ const decimalRadix = 10; // https://github.com/typeorm/typeorm/issues/2400 for more information. export class NumberToBigIntTransformer implements ValueTransformer { // tslint:disable-next-line:prefer-function-over-method - public to(value: number): string { - return value.toString(); + public to(value: number): string | null { + if (value === null || value === undefined) { + return null; + } else { + return value.toString(); + } } // tslint:disable-next-line:prefer-function-over-method diff --git a/packages/pipeline/test/entities/copper_test.ts b/packages/pipeline/test/entities/copper_test.ts new file mode 100644 index 000000000..2543364e6 --- /dev/null +++ b/packages/pipeline/test/entities/copper_test.ts @@ -0,0 +1,54 @@ +import 'mocha'; +import 'reflect-metadata'; + +import { + CopperActivity, + CopperActivityType, + CopperCustomField, + CopperLead, + CopperOpportunity, +} from '../../src/entities'; +import { createDbConnectionOnceAsync } from '../db_setup'; +import { + ParsedActivities, + ParsedActivityTypes, + ParsedCustomFields, + ParsedLeads, + ParsedOpportunities, +} from '../fixtures/copper/parsed_entities'; +import { chaiSetup } from '../utils/chai_setup'; + +import { testSaveAndFindEntityAsync } from './util'; + +chaiSetup.configure(); + +describe('Copper entities', () => { + describe('save and find', async () => { + it('Copper lead', async () => { + const connection = await createDbConnectionOnceAsync(); + const repository = connection.getRepository(CopperLead); + ParsedLeads.forEach(async entity => testSaveAndFindEntityAsync(repository, entity)); + }); + it('Copper activity', async () => { + const connection = await createDbConnectionOnceAsync(); + const repository = connection.getRepository(CopperActivity); + ParsedActivities.forEach(async entity => testSaveAndFindEntityAsync(repository, entity)); + }); + // searching on jsonb fields is broken in typeorm + it.skip('Copper opportunity', async () => { + const connection = await createDbConnectionOnceAsync(); + const repository = connection.getRepository(CopperOpportunity); + ParsedOpportunities.forEach(async entity => testSaveAndFindEntityAsync(repository, entity)); + }); + it('Copper activity type', async () => { + const connection = await createDbConnectionOnceAsync(); + const repository = connection.getRepository(CopperActivityType); + ParsedActivityTypes.forEach(async entity => testSaveAndFindEntityAsync(repository, entity)); + }); + it('Copper custom field', async () => { + const connection = await createDbConnectionOnceAsync(); + const repository = connection.getRepository(CopperCustomField); + ParsedCustomFields.forEach(async entity => testSaveAndFindEntityAsync(repository, entity)); + }); + }); +}); diff --git a/packages/pipeline/test/entities/util.ts b/packages/pipeline/test/entities/util.ts index 043a3b15d..42df23a4a 100644 --- a/packages/pipeline/test/entities/util.ts +++ b/packages/pipeline/test/entities/util.ts @@ -15,9 +15,9 @@ const expect = chai.expect; * @param entity An instance of a TypeORM entity which will be saved/retrieved from the database. */ export async function testSaveAndFindEntityAsync<T>(repository: Repository<T>, entity: T): Promise<void> { - // Note(albrow): We are forced to use an 'as any' hack here because + // Note(albrow): We are forced to use an 'any' hack here because // TypeScript complains about stack depth when checking the types. - await repository.save(entity as any); + await repository.save<any>(entity); const gotEntity = await repository.findOneOrFail({ where: entity, }); diff --git a/packages/pipeline/test/fixtures/copper/api_v1_activity_types.json b/packages/pipeline/test/fixtures/copper/api_v1_activity_types.json new file mode 100644 index 000000000..dbd39c31b --- /dev/null +++ b/packages/pipeline/test/fixtures/copper/api_v1_activity_types.json @@ -0,0 +1,24 @@ +{ + "user": [ + { "id": 0, "category": "user", "name": "Note", "is_disabled": false, "count_as_interaction": false }, + { "id": 660496, "category": "user", "name": "To Do", "is_disabled": false, "count_as_interaction": false }, + { "id": 660495, "category": "user", "name": "Meeting", "is_disabled": false, "count_as_interaction": true }, + { "id": 660494, "category": "user", "name": "Phone Call", "is_disabled": false, "count_as_interaction": true } + ], + "system": [ + { + "id": 1, + "category": "system", + "name": "Property Changed", + "is_disabled": false, + "count_as_interaction": false + }, + { + "id": 3, + "category": "system", + "name": "Pipeline Stage Changed", + "is_disabled": false, + "count_as_interaction": false + } + ] +} diff --git a/packages/pipeline/test/fixtures/copper/api_v1_activity_types.ts b/packages/pipeline/test/fixtures/copper/api_v1_activity_types.ts new file mode 100644 index 000000000..fd2d62a6c --- /dev/null +++ b/packages/pipeline/test/fixtures/copper/api_v1_activity_types.ts @@ -0,0 +1,16 @@ +import { CopperActivityType } from '../../../src/entities'; +const ParsedActivityTypes: CopperActivityType[] = [ + { id: 0, name: 'Note', category: 'user', isDisabled: false, countAsInteraction: false }, + { id: 660496, name: 'To Do', category: 'user', isDisabled: false, countAsInteraction: false }, + { id: 660495, name: 'Meeting', category: 'user', isDisabled: false, countAsInteraction: true }, + { id: 660494, name: 'Phone Call', category: 'user', isDisabled: false, countAsInteraction: true }, + { id: 1, name: 'Property Changed', category: 'system', isDisabled: false, countAsInteraction: false }, + { + id: 3, + name: 'Pipeline Stage Changed', + category: 'system', + isDisabled: false, + countAsInteraction: false, + }, +]; +export { ParsedActivityTypes }; diff --git a/packages/pipeline/test/fixtures/copper/api_v1_custom_field_definitions.json b/packages/pipeline/test/fixtures/copper/api_v1_custom_field_definitions.json new file mode 100644 index 000000000..c6665cb0f --- /dev/null +++ b/packages/pipeline/test/fixtures/copper/api_v1_custom_field_definitions.json @@ -0,0 +1,38 @@ +[ + { + "id": 261066, + "name": "Integration Type", + "canonical_name": null, + "data_type": "MultiSelect", + "available_on": ["opportunity", "company", "person"], + "options": [ + { "id": 394020, "name": "Strategic Relationship", "rank": 7 }, + { "id": 394013, "name": "ERC-20 Exchange", "rank": 0 }, + { "id": 394014, "name": "ERC-721 Marketplace", "rank": 1 }, + { "id": 394015, "name": "Trade Widget", "rank": 2 }, + { "id": 394016, "name": "Prediction Market Exchange", "rank": 3 }, + { "id": 394017, "name": "Security Token Exchange", "rank": 4 }, + { "id": 394018, "name": "Complementary Company", "rank": 5 }, + { "id": 394019, "name": "Service Provider", "rank": 6 } + ] + }, + { + "id": 261067, + "name": "Company Type", + "canonical_name": null, + "data_type": "Dropdown", + "available_on": ["company", "opportunity", "person"], + "options": [ + { "id": 394129, "name": "Market Maker", "rank": 6 }, + { "id": 394130, "name": "Events", "rank": 2 }, + { "id": 394023, "name": "Exchange", "rank": 3 }, + { "id": 394024, "name": "Investor", "rank": 5 }, + { "id": 394026, "name": "Service Provider", "rank": 8 }, + { "id": 394027, "name": "Wallet", "rank": 9 }, + { "id": 394134, "name": "Game", "rank": 4 }, + { "id": 394025, "name": "OTC", "rank": 7 }, + { "id": 394021, "name": "Blockchain/Protocol", "rank": 0 }, + { "id": 394022, "name": "dApp", "rank": 1 } + ] + } +] diff --git a/packages/pipeline/test/fixtures/copper/api_v1_custom_field_definitions.ts b/packages/pipeline/test/fixtures/copper/api_v1_custom_field_definitions.ts new file mode 100644 index 000000000..a44bbd2c3 --- /dev/null +++ b/packages/pipeline/test/fixtures/copper/api_v1_custom_field_definitions.ts @@ -0,0 +1,39 @@ +import { CopperCustomField } from '../../../src/entities'; +const ParsedCustomFields: CopperCustomField[] = [ + { + id: 394020, + name: 'Strategic Relationship', + dataType: 'Integration Type', + fieldType: 'option', + }, + { id: 394013, name: 'ERC-20 Exchange', dataType: 'Integration Type', fieldType: 'option' }, + { id: 394014, name: 'ERC-721 Marketplace', dataType: 'Integration Type', fieldType: 'option' }, + { id: 394015, name: 'Trade Widget', dataType: 'Integration Type', fieldType: 'option' }, + { + id: 394016, + name: 'Prediction Market Exchange', + dataType: 'Integration Type', + fieldType: 'option', + }, + { + id: 394017, + name: 'Security Token Exchange', + dataType: 'Integration Type', + fieldType: 'option', + }, + { id: 394018, name: 'Complementary Company', dataType: 'Integration Type', fieldType: 'option' }, + { id: 394019, name: 'Service Provider', dataType: 'Integration Type', fieldType: 'option' }, + { id: 261066, name: 'Integration Type', dataType: 'MultiSelect' }, + { id: 394129, name: 'Market Maker', dataType: 'Company Type', fieldType: 'option' }, + { id: 394130, name: 'Events', dataType: 'Company Type', fieldType: 'option' }, + { id: 394023, name: 'Exchange', dataType: 'Company Type', fieldType: 'option' }, + { id: 394024, name: 'Investor', dataType: 'Company Type', fieldType: 'option' }, + { id: 394026, name: 'Service Provider', dataType: 'Company Type', fieldType: 'option' }, + { id: 394027, name: 'Wallet', dataType: 'Company Type', fieldType: 'option' }, + { id: 394134, name: 'Game', dataType: 'Company Type', fieldType: 'option' }, + { id: 394025, name: 'OTC', dataType: 'Company Type', fieldType: 'option' }, + { id: 394021, name: 'Blockchain/Protocol', dataType: 'Company Type', fieldType: 'option' }, + { id: 394022, name: 'dApp', dataType: 'Company Type', fieldType: 'option' }, + { id: 261067, name: 'Company Type', dataType: 'Dropdown' }, +]; +export { ParsedCustomFields }; diff --git a/packages/pipeline/test/fixtures/copper/api_v1_list_activities.json b/packages/pipeline/test/fixtures/copper/api_v1_list_activities.json new file mode 100644 index 000000000..a726111ac --- /dev/null +++ b/packages/pipeline/test/fixtures/copper/api_v1_list_activities.json @@ -0,0 +1,242 @@ +[ + { + "id": 5015299552, + "parent": { "id": 14667512, "type": "opportunity" }, + "type": { "id": 3, "category": "system", "name": "Stage Change" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1545329595, + "old_value": { "id": 2392929, "name": "Evaluation" }, + "new_value": { "id": 2392931, "name": "Integration Started" }, + "date_created": 1545329595, + "date_modified": 1545329595 + }, + { + "id": 5010214065, + "parent": { "id": 14978865, "type": "opportunity" }, + "type": { "id": 3, "category": "system", "name": "Stage Change" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1545245706, + "old_value": { "id": 2392928, "name": "Intro" }, + "new_value": { "id": 2392929, "name": "Evaluation" }, + "date_created": 1545245706, + "date_modified": 1545245706 + }, + { + "id": 5006149111, + "parent": { "id": 70430977, "type": "person" }, + "type": { "id": 660495, "category": "user" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1545166908, + "old_value": null, + "new_value": null, + "date_created": 1545168280, + "date_modified": 1545166908 + }, + { + "id": 5005314622, + "parent": { "id": 27778968, "type": "company" }, + "type": { "id": 660495, "category": "user" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1545080504, + "old_value": null, + "new_value": null, + "date_created": 1545160479, + "date_modified": 1545080504 + }, + { + "id": 5000006802, + "parent": { "id": 14956518, "type": "opportunity" }, + "type": { "id": 660495, "category": "user" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1545071374, + "old_value": null, + "new_value": null, + "date_created": 1545071500, + "date_modified": 1545071374 + }, + { + "id": 4985504199, + "parent": { "id": 14912790, "type": "opportunity" }, + "type": { "id": 660495, "category": "user" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1544644058, + "old_value": null, + "new_value": null, + "date_created": 1544644661, + "date_modified": 1544644058 + }, + { + "id": 4985456147, + "parent": { "id": 14912790, "type": "opportunity" }, + "type": { "id": 660495, "category": "user" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1544644048, + "old_value": null, + "new_value": null, + "date_created": 1544644053, + "date_modified": 1544644048 + }, + { + "id": 4980975996, + "parent": { "id": 14902828, "type": "opportunity" }, + "type": { "id": 660495, "category": "user" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1544563171, + "old_value": null, + "new_value": null, + "date_created": 1544563224, + "date_modified": 1544563171 + }, + { + "id": 4980910331, + "parent": { "id": 14902828, "type": "opportunity" }, + "type": { "id": 3, "category": "system", "name": "Stage Change" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1544562495, + "old_value": { "id": 2392928, "name": "Intro" }, + "new_value": { "id": 2392931, "name": "Integration Started" }, + "date_created": 1544562495, + "date_modified": 1544562495 + }, + { + "id": 4980872220, + "parent": { "id": 14888910, "type": "opportunity" }, + "type": { "id": 660495, "category": "user" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1544559279, + "old_value": null, + "new_value": null, + "date_created": 1544562118, + "date_modified": 1544559279 + }, + { + "id": 4980508097, + "parent": { "id": 14050167, "type": "opportunity" }, + "type": { "id": 1, "category": "system", "name": "Status Change" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1544558077, + "old_value": "Open", + "new_value": "Won", + "date_created": 1544558077, + "date_modified": 1544558077 + }, + { + "id": 4980508095, + "parent": { "id": 66538237, "type": "person" }, + "type": { "id": 1, "category": "system" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1544558077, + "old_value": null, + "new_value": null, + "date_created": 1544558077, + "date_modified": 1544558077 + }, + { + "id": 4980508092, + "parent": { "id": 27779020, "type": "company" }, + "type": { "id": 1, "category": "system" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1544558077, + "old_value": null, + "new_value": null, + "date_created": 1544558077, + "date_modified": 1544558077 + }, + { + "id": 4980507507, + "parent": { "id": 14050167, "type": "opportunity" }, + "type": { "id": 3, "category": "system", "name": "Stage Change" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1544558071, + "old_value": { "id": 2392931, "name": "Integration Started" }, + "new_value": { "id": 2405442, "name": "Integration Complete" }, + "date_created": 1544558071, + "date_modified": 1544558071 + }, + { + "id": 4980479684, + "parent": { "id": 14901232, "type": "opportunity" }, + "type": { "id": 3, "category": "system", "name": "Stage Change" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1544557777, + "old_value": { "id": 2392928, "name": "Intro" }, + "new_value": { "id": 2392929, "name": "Evaluation" }, + "date_created": 1544557777, + "date_modified": 1544557777 + }, + { + "id": 4980327164, + "parent": { "id": 14901232, "type": "opportunity" }, + "type": { "id": 660495, "category": "user" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1544554864, + "old_value": null, + "new_value": null, + "date_created": 1544556132, + "date_modified": 1544554864 + }, + { + "id": 4975270470, + "parent": { "id": 14888744, "type": "opportunity" }, + "type": { "id": 3, "category": "system", "name": "Stage Change" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1544469501, + "old_value": { "id": 2392928, "name": "Intro" }, + "new_value": { "id": 2392931, "name": "Integration Started" }, + "date_created": 1544469501, + "date_modified": 1544469501 + }, + { + "id": 4975255523, + "parent": { "id": 64713448, "type": "person" }, + "type": { "id": 1, "category": "system" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1544469389, + "old_value": null, + "new_value": null, + "date_created": 1544469389, + "date_modified": 1544469389 + }, + { + "id": 4975255519, + "parent": { "id": 13735617, "type": "opportunity" }, + "type": { "id": 1, "category": "system", "name": "Status Change" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1544469388, + "old_value": "Open", + "new_value": "Won", + "date_created": 1544469388, + "date_modified": 1544469388 + }, + { + "id": 4975255514, + "parent": { "id": 27778968, "type": "company" }, + "type": { "id": 1, "category": "system" }, + "user_id": 680302, + "details": "blah blah", + "activity_date": 1544469388, + "old_value": null, + "new_value": null, + "date_created": 1544469388, + "date_modified": 1544469388 + } +] diff --git a/packages/pipeline/test/fixtures/copper/api_v1_list_activities.ts b/packages/pipeline/test/fixtures/copper/api_v1_list_activities.ts new file mode 100644 index 000000000..51ee9ced3 --- /dev/null +++ b/packages/pipeline/test/fixtures/copper/api_v1_list_activities.ts @@ -0,0 +1,305 @@ +import { CopperActivity } from '../../../src/entities'; + +const ParsedActivities: CopperActivity[] = [ + { + id: 5015299552, + parentId: 14667512, + parentType: 'opportunity', + typeId: 3, + typeCategory: 'system', + typeName: 'Stage Change', + userId: 680302, + dateCreated: 1545329595000, + dateModified: 1545329595000, + oldValueId: 2392929, + oldValueName: 'Evaluation', + newValueId: 2392931, + newValueName: 'Integration Started', + }, + { + id: 5010214065, + parentId: 14978865, + parentType: 'opportunity', + typeId: 3, + typeCategory: 'system', + typeName: 'Stage Change', + userId: 680302, + dateCreated: 1545245706000, + dateModified: 1545245706000, + oldValueId: 2392928, + oldValueName: 'Intro', + newValueId: 2392929, + newValueName: 'Evaluation', + }, + { + id: 5006149111, + parentId: 70430977, + parentType: 'person', + typeId: 660495, + typeCategory: 'user', + typeName: undefined, + userId: 680302, + dateCreated: 1545168280000, + dateModified: 1545166908000, + oldValueId: undefined, + oldValueName: undefined, + newValueId: undefined, + newValueName: undefined, + }, + { + id: 5005314622, + parentId: 27778968, + parentType: 'company', + typeId: 660495, + typeCategory: 'user', + typeName: undefined, + userId: 680302, + dateCreated: 1545160479000, + dateModified: 1545080504000, + oldValueId: undefined, + oldValueName: undefined, + newValueId: undefined, + newValueName: undefined, + }, + { + id: 5000006802, + parentId: 14956518, + parentType: 'opportunity', + typeId: 660495, + typeCategory: 'user', + typeName: undefined, + userId: 680302, + dateCreated: 1545071500000, + dateModified: 1545071374000, + oldValueId: undefined, + oldValueName: undefined, + newValueId: undefined, + newValueName: undefined, + }, + { + id: 4985504199, + parentId: 14912790, + parentType: 'opportunity', + typeId: 660495, + typeCategory: 'user', + typeName: undefined, + userId: 680302, + dateCreated: 1544644661000, + dateModified: 1544644058000, + oldValueId: undefined, + oldValueName: undefined, + newValueId: undefined, + newValueName: undefined, + }, + { + id: 4985456147, + parentId: 14912790, + parentType: 'opportunity', + typeId: 660495, + typeCategory: 'user', + typeName: undefined, + userId: 680302, + dateCreated: 1544644053000, + dateModified: 1544644048000, + oldValueId: undefined, + oldValueName: undefined, + newValueId: undefined, + newValueName: undefined, + }, + { + id: 4980975996, + parentId: 14902828, + parentType: 'opportunity', + typeId: 660495, + typeCategory: 'user', + typeName: undefined, + userId: 680302, + dateCreated: 1544563224000, + dateModified: 1544563171000, + oldValueId: undefined, + oldValueName: undefined, + newValueId: undefined, + newValueName: undefined, + }, + { + id: 4980910331, + parentId: 14902828, + parentType: 'opportunity', + typeId: 3, + typeCategory: 'system', + typeName: 'Stage Change', + userId: 680302, + dateCreated: 1544562495000, + dateModified: 1544562495000, + oldValueId: 2392928, + oldValueName: 'Intro', + newValueId: 2392931, + newValueName: 'Integration Started', + }, + { + id: 4980872220, + parentId: 14888910, + parentType: 'opportunity', + typeId: 660495, + typeCategory: 'user', + typeName: undefined, + userId: 680302, + dateCreated: 1544562118000, + dateModified: 1544559279000, + oldValueId: undefined, + oldValueName: undefined, + newValueId: undefined, + newValueName: undefined, + }, + { + id: 4980508097, + parentId: 14050167, + parentType: 'opportunity', + typeId: 1, + typeCategory: 'system', + typeName: 'Status Change', + userId: 680302, + dateCreated: 1544558077000, + dateModified: 1544558077000, + oldValueId: undefined, + oldValueName: undefined, + newValueId: undefined, + newValueName: undefined, + }, + { + id: 4980508095, + parentId: 66538237, + parentType: 'person', + typeId: 1, + typeCategory: 'system', + typeName: undefined, + userId: 680302, + dateCreated: 1544558077000, + dateModified: 1544558077000, + oldValueId: undefined, + oldValueName: undefined, + newValueId: undefined, + newValueName: undefined, + }, + { + id: 4980508092, + parentId: 27779020, + parentType: 'company', + typeId: 1, + typeCategory: 'system', + typeName: undefined, + userId: 680302, + dateCreated: 1544558077000, + dateModified: 1544558077000, + oldValueId: undefined, + oldValueName: undefined, + newValueId: undefined, + newValueName: undefined, + }, + { + id: 4980507507, + parentId: 14050167, + parentType: 'opportunity', + typeId: 3, + typeCategory: 'system', + typeName: 'Stage Change', + userId: 680302, + dateCreated: 1544558071000, + dateModified: 1544558071000, + oldValueId: 2392931, + oldValueName: 'Integration Started', + newValueId: 2405442, + newValueName: 'Integration Complete', + }, + { + id: 4980479684, + parentId: 14901232, + parentType: 'opportunity', + typeId: 3, + typeCategory: 'system', + typeName: 'Stage Change', + userId: 680302, + dateCreated: 1544557777000, + dateModified: 1544557777000, + oldValueId: 2392928, + oldValueName: 'Intro', + newValueId: 2392929, + newValueName: 'Evaluation', + }, + { + id: 4980327164, + parentId: 14901232, + parentType: 'opportunity', + typeId: 660495, + typeCategory: 'user', + typeName: undefined, + userId: 680302, + dateCreated: 1544556132000, + dateModified: 1544554864000, + oldValueId: undefined, + oldValueName: undefined, + newValueId: undefined, + newValueName: undefined, + }, + { + id: 4975270470, + parentId: 14888744, + parentType: 'opportunity', + typeId: 3, + typeCategory: 'system', + typeName: 'Stage Change', + userId: 680302, + dateCreated: 1544469501000, + dateModified: 1544469501000, + oldValueId: 2392928, + oldValueName: 'Intro', + newValueId: 2392931, + newValueName: 'Integration Started', + }, + { + id: 4975255523, + parentId: 64713448, + parentType: 'person', + typeId: 1, + typeCategory: 'system', + typeName: undefined, + userId: 680302, + dateCreated: 1544469389000, + dateModified: 1544469389000, + oldValueId: undefined, + oldValueName: undefined, + newValueId: undefined, + newValueName: undefined, + }, + { + id: 4975255519, + parentId: 13735617, + parentType: 'opportunity', + typeId: 1, + typeCategory: 'system', + typeName: 'Status Change', + userId: 680302, + dateCreated: 1544469388000, + dateModified: 1544469388000, + oldValueId: undefined, + oldValueName: undefined, + newValueId: undefined, + newValueName: undefined, + }, + { + id: 4975255514, + parentId: 27778968, + parentType: 'company', + typeId: 1, + typeCategory: 'system', + typeName: undefined, + userId: 680302, + dateCreated: 1544469388000, + dateModified: 1544469388000, + oldValueId: undefined, + oldValueName: undefined, + newValueId: undefined, + newValueName: undefined, + }, +]; +export { ParsedActivities }; diff --git a/packages/pipeline/test/fixtures/copper/api_v1_list_leads.json b/packages/pipeline/test/fixtures/copper/api_v1_list_leads.json new file mode 100644 index 000000000..5223976f9 --- /dev/null +++ b/packages/pipeline/test/fixtures/copper/api_v1_list_leads.json @@ -0,0 +1,583 @@ +[ + { + "id": 9150547, + "name": "My Contact", + "prefix": null, + "first_name": "My", + "last_name": "Contact", + "middle_name": null, + "suffix": null, + "address": null, + "assignee_id": null, + "company_name": null, + "customer_source_id": null, + "details": null, + "email": { + "email": "mycontact@noemail.com", + "category": "work" + }, + "interaction_count": 0, + "monetary_value": null, + "socials": [], + "status": "New", + "status_id": 208231, + "tags": [], + "title": null, + "websites": [], + "phone_numbers": [], + "custom_fields": [ + { + "custom_field_definition_id": 100764, + "value": null + }, + { + "custom_field_definition_id": 103481, + "value": null + } + ], + "date_created": 1490045162, + "date_modified": 1490045162 + }, + { + "id": 9150552, + "name": "My Contact", + "prefix": null, + "first_name": "My", + "last_name": "Contact", + "middle_name": null, + "suffix": null, + "address": null, + "assignee_id": null, + "company_name": null, + "customer_source_id": null, + "details": null, + "email": null, + "interaction_count": 0, + "monetary_value": null, + "socials": [], + "status": "New", + "status_id": 208231, + "tags": [], + "title": null, + "websites": [], + "phone_numbers": [ + { + "number": "415-123-45678", + "category": "mobile" + } + ], + "custom_fields": [ + { + "custom_field_definition_id": 100764, + "value": null + }, + { + "custom_field_definition_id": 103481, + "value": null + } + ], + "date_created": 1490045237, + "date_modified": 1490045237 + }, + { + "id": 9150578, + "name": "My Contact", + "prefix": null, + "first_name": "My", + "last_name": "Contact", + "middle_name": null, + "suffix": null, + "address": null, + "assignee_id": null, + "company_name": null, + "customer_source_id": null, + "details": null, + "email": null, + "interaction_count": 0, + "monetary_value": null, + "socials": [], + "status": "New", + "status_id": 208231, + "tags": [], + "title": null, + "websites": [], + "phone_numbers": [ + { + "number": "415-123-45678", + "category": "mobile" + } + ], + "custom_fields": [ + { + "custom_field_definition_id": 100764, + "value": null + }, + { + "custom_field_definition_id": 103481, + "value": null + } + ], + "date_created": 1490045279, + "date_modified": 1490045279 + }, + { + "id": 8982554, + "name": "My Lead", + "prefix": null, + "first_name": "My", + "last_name": "Lead", + "middle_name": null, + "suffix": null, + "address": null, + "assignee_id": null, + "company_name": null, + "customer_source_id": null, + "details": null, + "email": { + "email": "mylead@noemail.com", + "category": "work" + }, + "interaction_count": 0, + "monetary_value": null, + "socials": [], + "status": "New", + "status_id": 208231, + "tags": [], + "title": null, + "websites": [], + "phone_numbers": [], + "custom_fields": [ + { + "custom_field_definition_id": 100764, + "value": null + }, + { + "custom_field_definition_id": 103481, + "value": null + } + ], + "date_created": 1489528899, + "date_modified": 1489528899 + }, + { + "id": 8982702, + "name": "My Lead", + "prefix": null, + "first_name": "My", + "last_name": "Lead", + "middle_name": null, + "suffix": null, + "address": null, + "assignee_id": null, + "company_name": null, + "customer_source_id": null, + "details": null, + "email": { + "email": "mylead@gmail.test", + "category": "work" + }, + "interaction_count": 0, + "monetary_value": null, + "socials": [], + "status": "New", + "status_id": 208231, + "tags": [], + "title": null, + "websites": [], + "phone_numbers": [], + "custom_fields": [ + { + "custom_field_definition_id": 100764, + "value": null + }, + { + "custom_field_definition_id": 103481, + "value": null + } + ], + "date_created": 1489531171, + "date_modified": 1489531171 + }, + { + "id": 9094361, + "name": "My Lead", + "prefix": null, + "first_name": "My", + "last_name": "Lead", + "middle_name": null, + "suffix": null, + "address": null, + "assignee_id": null, + "company_name": null, + "customer_source_id": null, + "details": null, + "email": { + "email": "mylead@noemail.com", + "category": "work" + }, + "interaction_count": 0, + "monetary_value": null, + "socials": [], + "status": "New", + "status_id": 208231, + "tags": [], + "title": null, + "websites": [], + "phone_numbers": [], + "custom_fields": [ + { + "custom_field_definition_id": 100764, + "value": null + }, + { + "custom_field_definition_id": 103481, + "value": null + } + ], + "date_created": 1489791225, + "date_modified": 1489791225 + }, + { + "id": 9094364, + "name": "My Lead", + "prefix": null, + "first_name": "My", + "last_name": "Lead", + "middle_name": null, + "suffix": null, + "address": null, + "assignee_id": null, + "company_name": null, + "customer_source_id": null, + "details": null, + "email": { + "email": "mylead@noemail.com", + "category": "work" + }, + "interaction_count": 0, + "monetary_value": null, + "socials": [], + "status": "New", + "status_id": 208231, + "tags": [], + "title": null, + "websites": [], + "phone_numbers": [], + "custom_fields": [ + { + "custom_field_definition_id": 100764, + "value": "123456789012345678901234567890" + }, + { + "custom_field_definition_id": 103481, + "value": "123456789012345678901234567890" + } + ], + "date_created": 1489791283, + "date_modified": 1489791283 + }, + { + "id": 9094371, + "name": "My Lead", + "prefix": null, + "first_name": "My", + "last_name": "Lead", + "middle_name": null, + "suffix": null, + "address": null, + "assignee_id": null, + "company_name": null, + "customer_source_id": null, + "details": null, + "email": { + "email": "mylead@noemail.com", + "category": "work" + }, + "interaction_count": 0, + "monetary_value": null, + "socials": [], + "status": "New", + "status_id": 208231, + "tags": [], + "title": null, + "websites": [], + "phone_numbers": [], + "custom_fields": [ + { + "custom_field_definition_id": 100764, + "value": + "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------" + }, + { + "custom_field_definition_id": 103481, + "value": "123456789012345678901234567890" + } + ], + "date_created": 1489791417, + "date_modified": 1489791417 + }, + { + "id": 9094372, + "name": "My Lead", + "prefix": null, + "first_name": "My", + "last_name": "Lead", + "middle_name": null, + "suffix": null, + "address": null, + "assignee_id": null, + "company_name": null, + "customer_source_id": null, + "details": null, + "email": { + "email": "mylead@noemail.com", + "category": "work" + }, + "interaction_count": 0, + "monetary_value": null, + "socials": [], + "status": "New", + "status_id": 208231, + "tags": [], + "title": null, + "websites": [], + "phone_numbers": [], + "custom_fields": [ + { + "custom_field_definition_id": 100764, + "value": + "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----" + }, + { + "custom_field_definition_id": 103481, + "value": "123456789012345678901234567890" + } + ], + "date_created": 1489791453, + "date_modified": 1489791453 + }, + { + "id": 9094373, + "name": "My Lead", + "prefix": null, + "first_name": "My", + "last_name": "Lead", + "middle_name": null, + "suffix": null, + "address": null, + "assignee_id": null, + "company_name": null, + "customer_source_id": null, + "details": null, + "email": { + "email": "mylead@noemail.com", + "category": "work" + }, + "interaction_count": 0, + "monetary_value": null, + "socials": [], + "status": "New", + "status_id": 208231, + "tags": [], + "title": null, + "websites": [], + "phone_numbers": [], + "custom_fields": [ + { + "custom_field_definition_id": 100764, + "value": + "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----" + }, + { + "custom_field_definition_id": 103481, + "value": + "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------" + } + ], + "date_created": 1489791470, + "date_modified": 1489791470 + }, + { + "id": 9094383, + "name": "My Lead", + "prefix": null, + "first_name": "My", + "last_name": "Lead", + "middle_name": null, + "suffix": null, + "address": null, + "assignee_id": null, + "company_name": null, + "customer_source_id": null, + "details": null, + "email": { + "email": "mylead@noemail.com", + "category": "work" + }, + "interaction_count": 0, + "monetary_value": null, + "socials": [], + "status": "New", + "status_id": 208231, + "tags": [], + "title": null, + "websites": [], + "phone_numbers": [], + "custom_fields": [ + { + "custom_field_definition_id": 100764, + "value": + "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5-----" + }, + { + "custom_field_definition_id": 103481, + "value": + "|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------|--------1---------2---------3---------4---------5---------6---------7---------8---------9---------" + } + ], + "date_created": 1489791672, + "date_modified": 1489791672 + }, + { + "id": 9174441, + "name": "My Lead", + "prefix": null, + "first_name": "My", + "last_name": "Lead", + "middle_name": null, + "suffix": null, + "address": null, + "assignee_id": null, + "company_name": null, + "customer_source_id": null, + "details": null, + "email": { + "email": "mylead@noemail.com", + "category": "work" + }, + "interaction_count": 0, + "monetary_value": null, + "socials": [], + "status": "New", + "status_id": 208231, + "tags": [], + "title": null, + "websites": [], + "phone_numbers": [], + "custom_fields": [ + { + "custom_field_definition_id": 100764, + "value": "Text fields are 255 chars or less!" + }, + { + "custom_field_definition_id": 103481, + "value": "text \n text" + } + ], + "date_created": 1490112942, + "date_modified": 1490112942 + }, + { + "id": 9174443, + "name": "My Lead", + "prefix": null, + "first_name": "My", + "last_name": "Lead", + "middle_name": null, + "suffix": null, + "address": null, + "assignee_id": null, + "company_name": null, + "customer_source_id": null, + "details": null, + "email": { + "email": "mylead@noemail.com", + "category": "work" + }, + "interaction_count": 0, + "monetary_value": null, + "socials": [], + "status": "New", + "status_id": 208231, + "tags": [], + "title": null, + "websites": [], + "phone_numbers": [], + "custom_fields": [ + { + "custom_field_definition_id": 100764, + "value": "Text fields are 255 chars or less!" + }, + { + "custom_field_definition_id": 103481, + "value": "text /n text" + } + ], + "date_created": 1490112953, + "date_modified": 1490112953 + }, + { + "id": 8894157, + "name": "Test Lead", + "prefix": null, + "first_name": "Test", + "last_name": "Lead", + "middle_name": null, + "suffix": null, + "address": { + "street": "301 Howard St Ste 600", + "city": "San Francisco", + "state": "CA", + "postal_code": "94105", + "country": "US" + }, + "assignee_id": 137658, + "company_name": "Lead's Company", + "customer_source_id": 331241, + "details": "This is an update", + "email": { + "email": "address@workemail.com", + "category": "work" + }, + "interaction_count": 0, + "monetary_value": 100, + "socials": [ + { + "url": "facebook.com/test_lead", + "category": "facebook" + } + ], + "status": "New", + "status_id": 208231, + "tags": ["tag 1", "tag 2"], + "title": "Title", + "websites": [ + { + "url": "www.workwebsite.com", + "category": "work" + } + ], + "phone_numbers": [ + { + "number": "415-999-4321", + "category": "mobile" + }, + { + "number": "415-555-1234", + "category": "work" + } + ], + "custom_fields": [ + { + "custom_field_definition_id": 100764, + "value": null + }, + { + "custom_field_definition_id": 103481, + "value": null + } + ], + "date_created": 1489018784, + "date_modified": 1496692911 + } +] diff --git a/packages/pipeline/test/fixtures/copper/api_v1_list_leads.ts b/packages/pipeline/test/fixtures/copper/api_v1_list_leads.ts new file mode 100644 index 000000000..b1f00cba7 --- /dev/null +++ b/packages/pipeline/test/fixtures/copper/api_v1_list_leads.ts @@ -0,0 +1,229 @@ +import { CopperLead } from '../../../src/entities'; +const ParsedLeads: CopperLead[] = [ + { + id: 9150547, + name: 'My Contact', + firstName: 'My', + lastName: 'Contact', + middleName: undefined, + assigneeId: undefined, + companyName: undefined, + customerSourceId: undefined, + monetaryValue: undefined, + status: 'New', + statusId: 208231, + title: undefined, + dateCreated: 1490045162000, + dateModified: 1490045162000, + }, + { + id: 9150552, + name: 'My Contact', + firstName: 'My', + lastName: 'Contact', + middleName: undefined, + assigneeId: undefined, + companyName: undefined, + customerSourceId: undefined, + monetaryValue: undefined, + status: 'New', + statusId: 208231, + title: undefined, + dateCreated: 1490045237000, + dateModified: 1490045237000, + }, + { + id: 9150578, + name: 'My Contact', + firstName: 'My', + lastName: 'Contact', + middleName: undefined, + assigneeId: undefined, + companyName: undefined, + customerSourceId: undefined, + monetaryValue: undefined, + status: 'New', + statusId: 208231, + title: undefined, + dateCreated: 1490045279000, + dateModified: 1490045279000, + }, + { + id: 8982554, + name: 'My Lead', + firstName: 'My', + lastName: 'Lead', + middleName: undefined, + assigneeId: undefined, + companyName: undefined, + customerSourceId: undefined, + monetaryValue: undefined, + status: 'New', + statusId: 208231, + title: undefined, + dateCreated: 1489528899000, + dateModified: 1489528899000, + }, + { + id: 8982702, + name: 'My Lead', + firstName: 'My', + lastName: 'Lead', + middleName: undefined, + assigneeId: undefined, + companyName: undefined, + customerSourceId: undefined, + monetaryValue: undefined, + status: 'New', + statusId: 208231, + title: undefined, + dateCreated: 1489531171000, + dateModified: 1489531171000, + }, + { + id: 9094361, + name: 'My Lead', + firstName: 'My', + lastName: 'Lead', + middleName: undefined, + assigneeId: undefined, + companyName: undefined, + customerSourceId: undefined, + monetaryValue: undefined, + status: 'New', + statusId: 208231, + title: undefined, + dateCreated: 1489791225000, + dateModified: 1489791225000, + }, + { + id: 9094364, + name: 'My Lead', + firstName: 'My', + lastName: 'Lead', + middleName: undefined, + assigneeId: undefined, + companyName: undefined, + customerSourceId: undefined, + monetaryValue: undefined, + status: 'New', + statusId: 208231, + title: undefined, + dateCreated: 1489791283000, + dateModified: 1489791283000, + }, + { + id: 9094371, + name: 'My Lead', + firstName: 'My', + lastName: 'Lead', + middleName: undefined, + assigneeId: undefined, + companyName: undefined, + customerSourceId: undefined, + monetaryValue: undefined, + status: 'New', + statusId: 208231, + title: undefined, + dateCreated: 1489791417000, + dateModified: 1489791417000, + }, + { + id: 9094372, + name: 'My Lead', + firstName: 'My', + lastName: 'Lead', + middleName: undefined, + assigneeId: undefined, + companyName: undefined, + customerSourceId: undefined, + monetaryValue: undefined, + status: 'New', + statusId: 208231, + title: undefined, + dateCreated: 1489791453000, + dateModified: 1489791453000, + }, + { + id: 9094373, + name: 'My Lead', + firstName: 'My', + lastName: 'Lead', + middleName: undefined, + assigneeId: undefined, + companyName: undefined, + customerSourceId: undefined, + monetaryValue: undefined, + status: 'New', + statusId: 208231, + title: undefined, + dateCreated: 1489791470000, + dateModified: 1489791470000, + }, + { + id: 9094383, + name: 'My Lead', + firstName: 'My', + lastName: 'Lead', + middleName: undefined, + assigneeId: undefined, + companyName: undefined, + customerSourceId: undefined, + monetaryValue: undefined, + status: 'New', + statusId: 208231, + title: undefined, + dateCreated: 1489791672000, + dateModified: 1489791672000, + }, + { + id: 9174441, + name: 'My Lead', + firstName: 'My', + lastName: 'Lead', + middleName: undefined, + assigneeId: undefined, + companyName: undefined, + customerSourceId: undefined, + monetaryValue: undefined, + status: 'New', + statusId: 208231, + title: undefined, + dateCreated: 1490112942000, + dateModified: 1490112942000, + }, + { + id: 9174443, + name: 'My Lead', + firstName: 'My', + lastName: 'Lead', + middleName: undefined, + assigneeId: undefined, + companyName: undefined, + customerSourceId: undefined, + monetaryValue: undefined, + status: 'New', + statusId: 208231, + title: undefined, + dateCreated: 1490112953000, + dateModified: 1490112953000, + }, + { + id: 8894157, + name: 'Test Lead', + firstName: 'Test', + lastName: 'Lead', + middleName: undefined, + assigneeId: 137658, + companyName: "Lead's Company", + customerSourceId: 331241, + monetaryValue: 100, + status: 'New', + statusId: 208231, + title: 'Title', + dateCreated: 1489018784000, + dateModified: 1496692911000, + }, +]; + +export { ParsedLeads }; diff --git a/packages/pipeline/test/fixtures/copper/api_v1_list_opportunities.json b/packages/pipeline/test/fixtures/copper/api_v1_list_opportunities.json new file mode 100644 index 000000000..34ac58c30 --- /dev/null +++ b/packages/pipeline/test/fixtures/copper/api_v1_list_opportunities.json @@ -0,0 +1,662 @@ +[ + { + "id": 14050269, + "name": "8Base RaaS", + "assignee_id": 680302, + "close_date": "11/19/2018", + "company_id": 27778962, + "company_name": "8base", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2405442, + "primary_contact_id": 66088850, + "priority": "None", + "status": "Won", + "tags": [], + "interaction_count": 81, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1542653860, + "date_last_contacted": 1544757550, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1538414159, + "date_modified": 1544769562, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394013, 394018] }, + { "custom_field_definition_id": 261067, "value": 394026 } + ] + }, + { + "id": 14631430, + "name": "Alice.si TW + ERC 20 Marketplace", + "assignee_id": 680302, + "close_date": "12/15/2018", + "company_id": 30238847, + "company_name": "Alice SI", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2392929, + "primary_contact_id": 69354024, + "priority": "None", + "status": "Open", + "tags": [], + "interaction_count": 4, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1542304481, + "date_last_contacted": 1542304800, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1542304481, + "date_modified": 1542304943, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394013, 394015] }, + { "custom_field_definition_id": 261067, "value": 394023 } + ] + }, + { + "id": 14632057, + "name": "Altcoin.io Relayer", + "assignee_id": 680302, + "close_date": "12/15/2018", + "company_id": 29936486, + "company_name": "Altcoin.io", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2392929, + "primary_contact_id": 68724646, + "priority": "None", + "status": "Open", + "tags": [], + "interaction_count": 22, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1542310909, + "date_last_contacted": 1543864597, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1542306827, + "date_modified": 1543864667, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394013, 394017] }, + { "custom_field_definition_id": 261067, "value": 394023 } + ] + }, + { + "id": 14667523, + "name": "Altcoin.io Relayer", + "assignee_id": 680302, + "close_date": "12/19/2018", + "company_id": 29936486, + "company_name": "Altcoin.io", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2392929, + "primary_contact_id": 68724646, + "priority": "None", + "status": "Open", + "tags": [], + "interaction_count": 21, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1542657437, + "date_last_contacted": 1543864597, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1542657437, + "date_modified": 1543864667, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394013, 394017] }, + { "custom_field_definition_id": 261067, "value": 394023 } + ] + }, + { + "id": 14666706, + "name": "Amadeus Relayer", + "assignee_id": 680302, + "close_date": "11/19/2018", + "company_id": 29243209, + "company_name": "Amadeus", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2405442, + "primary_contact_id": 66912020, + "priority": "None", + "status": "Won", + "tags": [], + "interaction_count": 11, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1542654284, + "date_last_contacted": 1543264254, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1542654284, + "date_modified": 1543277520, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394013] }, + { "custom_field_definition_id": 261067, "value": 394023 } + ] + }, + { + "id": 14666718, + "name": "Ambo Relayer", + "assignee_id": 680302, + "close_date": "11/19/2018", + "company_id": 29249190, + "company_name": "Ambo", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2405442, + "primary_contact_id": 66927869, + "priority": "None", + "status": "Won", + "tags": [], + "interaction_count": 126, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1542654352, + "date_last_contacted": 1545252349, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1542654352, + "date_modified": 1545253761, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394013] }, + { "custom_field_definition_id": 261067, "value": 394023 } + ] + }, + { + "id": 14164318, + "name": "Augur TW", + "assignee_id": 680302, + "close_date": "12/10/2018", + "company_id": 27778967, + "company_name": "Augur", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2405442, + "primary_contact_id": 67248692, + "priority": "None", + "status": "Won", + "tags": [], + "interaction_count": 22, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1544469362, + "date_last_contacted": 1544491567, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1539204858, + "date_modified": 1544653867, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394015] }, + { "custom_field_definition_id": 261067, "value": 394021 } + ] + }, + { + "id": 14666626, + "name": "Autonio", + "assignee_id": 680302, + "close_date": "12/19/2018", + "company_id": 27920701, + "company_name": "Auton", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2392931, + "primary_contact_id": 64742640, + "priority": "None", + "status": "Open", + "tags": [], + "interaction_count": 54, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1542653834, + "date_last_contacted": 1542658568, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1542653834, + "date_modified": 1542658808, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394013, 394019] }, + { "custom_field_definition_id": 261067, "value": 394023 } + ] + }, + { + "id": 14050921, + "name": "Axie Infinity 721 Marketplace", + "assignee_id": 680302, + "close_date": "11/1/2018", + "company_id": 27779033, + "company_name": "Axie Infinity", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2392931, + "primary_contact_id": 66499254, + "priority": "None", + "status": "Open", + "tags": [], + "interaction_count": 4, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1543861025, + "date_last_contacted": 1539024738, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1538416687, + "date_modified": 1543861025, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394014] }, + { "custom_field_definition_id": 261067, "value": 394134 } + ] + }, + { + "id": 13735617, + "name": "Balance TW", + "assignee_id": 680302, + "close_date": "12/10/2018", + "company_id": 27778968, + "company_name": "Balance", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2405442, + "primary_contact_id": 64713448, + "priority": "None", + "status": "Won", + "tags": [], + "interaction_count": 34, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1544469382, + "date_last_contacted": 1545082200, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1535668009, + "date_modified": 1545082454, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394015] }, + { "custom_field_definition_id": 261067, "value": 394027 } + ] + }, + { + "id": 14667112, + "name": "Bamboo Relayer", + "assignee_id": 680302, + "close_date": "11/19/2018", + "company_id": 29243795, + "company_name": "Bamboo Relay", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2405442, + "primary_contact_id": 66914687, + "priority": "None", + "status": "Won", + "tags": [], + "interaction_count": 46, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1542655143, + "date_last_contacted": 1545252349, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1542655143, + "date_modified": 1545253761, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394013] }, + { "custom_field_definition_id": 261067, "value": 394023 } + ] + }, + { + "id": 13627309, + "name": "Ben TW", + "assignee_id": 680302, + "close_date": "1/1/2019", + "company_id": 27702348, + "company_name": "Ben", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2392929, + "primary_contact_id": 64262622, + "priority": "None", + "status": "Open", + "tags": [], + "interaction_count": 64, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1541527279, + "date_last_contacted": 1541639882, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1534887789, + "date_modified": 1541651395, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394015] }, + { "custom_field_definition_id": 261067, "value": 394027 } + ] + }, + { + "id": 14808512, + "name": "Bit2Me Relayer", + "assignee_id": 680302, + "close_date": "12/3/2018", + "company_id": 30793050, + "company_name": "Bit2Me", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2405442, + "primary_contact_id": 70267217, + "priority": "None", + "status": "Won", + "tags": [], + "interaction_count": 0, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1543861167, + "date_last_contacted": null, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1543861167, + "date_modified": 1543861189, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394013] }, + { "custom_field_definition_id": 261067, "value": 394023 } + ] + }, + { + "id": 14050312, + "name": "Bitcoin.tax Reporting Integration", + "assignee_id": 680302, + "close_date": "11/1/2018", + "company_id": 27957614, + "company_name": "Bitcoin", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2392928, + "primary_contact_id": 66539479, + "priority": "None", + "status": "Open", + "tags": [], + "interaction_count": 5, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1538414308, + "date_last_contacted": 1536766098, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1538414308, + "date_modified": 1538414314, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394019] }, + { "custom_field_definition_id": 261067, "value": 394026 } + ] + }, + { + "id": 14331463, + "name": "Bitpie TW", + "assignee_id": 680302, + "close_date": "11/19/2018", + "company_id": 27779026, + "company_name": "Bitpie", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2392929, + "primary_contact_id": 67700943, + "priority": "None", + "status": "Open", + "tags": [], + "interaction_count": 9, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1539984566, + "date_last_contacted": 1541529947, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1539984566, + "date_modified": 1541530233, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394015] }, + { "custom_field_definition_id": 261067, "value": 394027 } + ] + }, + { + "id": 14331481, + "name": "Bitski Wallet SDK TW", + "assignee_id": 680302, + "close_date": "11/19/2018", + "company_id": 29489300, + "company_name": "Bitski", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2392929, + "primary_contact_id": 67697528, + "priority": "None", + "status": "Open", + "tags": [], + "interaction_count": 23, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1539984735, + "date_last_contacted": 1544811399, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1539984735, + "date_modified": 1544818605, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394015] }, + { "custom_field_definition_id": 261067, "value": 394026 } + ] + }, + { + "id": 14531554, + "name": "BitUniverse TW", + "assignee_id": 680302, + "close_date": "12/6/2018", + "company_id": 29901805, + "company_name": "BitUniverse Co., Ltd (Cryptocurrency Portfolio)", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2392929, + "primary_contact_id": 68692107, + "priority": "None", + "status": "Open", + "tags": [], + "interaction_count": 15, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1543861104, + "date_last_contacted": 1544803276, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1541527110, + "date_modified": 1544812979, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394015] }, + { "custom_field_definition_id": 261067, "value": 394026 } + ] + }, + { + "id": 14050895, + "name": "BlitzPredict PMR", + "assignee_id": 680302, + "close_date": "11/1/2018", + "company_id": 28758258, + "company_name": "BlitzPredict", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2392929, + "primary_contact_id": 66378659, + "priority": "None", + "status": "Open", + "tags": [], + "interaction_count": 32, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1539985501, + "date_last_contacted": 1544830560, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1538416597, + "date_modified": 1544830709, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394016] }, + { "custom_field_definition_id": 261067, "value": 394023 } + ] + }, + { + "id": 14209841, + "name": "Blockfolio TW", + "assignee_id": 680302, + "close_date": "11/15/2018", + "company_id": 29332516, + "company_name": "Blockfolio", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2405443, + "primary_contact_id": 67247027, + "priority": "None", + "status": "Open", + "tags": [], + "interaction_count": 20, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1539984098, + "date_last_contacted": 1539977661, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1539624801, + "date_modified": 1539984098, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394015] }, + { "custom_field_definition_id": 261067, "value": 394026 } + ] + }, + { + "id": 14633220, + "name": "BlockSwap 721 / 1155 Conversational Marketplace", + "assignee_id": 680302, + "close_date": "12/15/2018", + "company_id": 30210921, + "company_name": "BlockSwap", + "customer_source_id": null, + "details": "blah blah", + "loss_reason_id": null, + "pipeline_id": 512676, + "pipeline_stage_id": 2392929, + "primary_contact_id": 69296220, + "priority": "None", + "status": "Open", + "tags": [], + "interaction_count": 82, + "monetary_unit": null, + "monetary_value": null, + "converted_unit": null, + "converted_value": null, + "win_probability": 0, + "date_stage_changed": 1542311056, + "date_last_contacted": 1543536442, + "leads_converted_from": [], + "date_lead_created": null, + "date_created": 1542311056, + "date_modified": 1543557877, + "custom_fields": [ + { "custom_field_definition_id": 261066, "value": [394014] }, + { "custom_field_definition_id": 261067, "value": 394023 } + ] + } +] diff --git a/packages/pipeline/test/fixtures/copper/api_v1_list_opportunities.ts b/packages/pipeline/test/fixtures/copper/api_v1_list_opportunities.ts new file mode 100644 index 000000000..3c2d4ae5e --- /dev/null +++ b/packages/pipeline/test/fixtures/copper/api_v1_list_opportunities.ts @@ -0,0 +1,425 @@ +// tslint:disable:custom-no-magic-numbers +import { CopperOpportunity } from '../../../src/entities'; +const ParsedOpportunities: CopperOpportunity[] = [ + { + id: 14050269, + name: '8Base RaaS', + assigneeId: 680302, + closeDate: '11/19/2018', + companyId: 27778962, + companyName: '8base', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2405442, + primaryContactId: 66088850, + priority: 'None', + status: 'Won', + interactionCount: 81, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1538414159000, + dateModified: 1544769562000, + customFields: { '261066': 394018, '261067': 394026 }, + }, + { + id: 14631430, + name: 'Alice.si TW + ERC 20 Marketplace', + assigneeId: 680302, + closeDate: '12/15/2018', + companyId: 30238847, + companyName: 'Alice SI', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2392929, + primaryContactId: 69354024, + priority: 'None', + status: 'Open', + interactionCount: 4, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1542304481000, + dateModified: 1542304943000, + customFields: { '261066': 394015, '261067': 394023 }, + }, + { + id: 14632057, + name: 'Altcoin.io Relayer', + assigneeId: 680302, + closeDate: '12/15/2018', + companyId: 29936486, + companyName: 'Altcoin.io', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2392929, + primaryContactId: 68724646, + priority: 'None', + status: 'Open', + interactionCount: 22, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1542306827000, + dateModified: 1543864667000, + customFields: { '261066': 394017, '261067': 394023 }, + }, + { + id: 14667523, + name: 'Altcoin.io Relayer', + assigneeId: 680302, + closeDate: '12/19/2018', + companyId: 29936486, + companyName: 'Altcoin.io', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2392929, + primaryContactId: 68724646, + priority: 'None', + status: 'Open', + interactionCount: 21, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1542657437000, + dateModified: 1543864667000, + customFields: { '261066': 394017, '261067': 394023 }, + }, + { + id: 14666706, + name: 'Amadeus Relayer', + assigneeId: 680302, + closeDate: '11/19/2018', + companyId: 29243209, + companyName: 'Amadeus', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2405442, + primaryContactId: 66912020, + priority: 'None', + status: 'Won', + interactionCount: 11, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1542654284000, + dateModified: 1543277520000, + customFields: { '261066': 394013, '261067': 394023 }, + }, + { + id: 14666718, + name: 'Ambo Relayer', + assigneeId: 680302, + closeDate: '11/19/2018', + companyId: 29249190, + companyName: 'Ambo', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2405442, + primaryContactId: 66927869, + priority: 'None', + status: 'Won', + interactionCount: 126, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1542654352000, + dateModified: 1545253761000, + customFields: { '261066': 394013, '261067': 394023 }, + }, + { + id: 14164318, + name: 'Augur TW', + assigneeId: 680302, + closeDate: '12/10/2018', + companyId: 27778967, + companyName: 'Augur', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2405442, + primaryContactId: 67248692, + priority: 'None', + status: 'Won', + interactionCount: 22, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1539204858000, + dateModified: 1544653867000, + customFields: { '261066': 394015, '261067': 394021 }, + }, + { + id: 14666626, + name: 'Autonio', + assigneeId: 680302, + closeDate: '12/19/2018', + companyId: 27920701, + companyName: 'Auton', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2392931, + primaryContactId: 64742640, + priority: 'None', + status: 'Open', + interactionCount: 54, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1542653834000, + dateModified: 1542658808000, + customFields: { '261066': 394019, '261067': 394023 }, + }, + { + id: 14050921, + name: 'Axie Infinity 721 Marketplace', + assigneeId: 680302, + closeDate: '11/1/2018', + companyId: 27779033, + companyName: 'Axie Infinity', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2392931, + primaryContactId: 66499254, + priority: 'None', + status: 'Open', + interactionCount: 4, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1538416687000, + dateModified: 1543861025000, + customFields: { '261066': 394014, '261067': 394134 }, + }, + { + id: 13735617, + name: 'Balance TW', + assigneeId: 680302, + closeDate: '12/10/2018', + companyId: 27778968, + companyName: 'Balance', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2405442, + primaryContactId: 64713448, + priority: 'None', + status: 'Won', + interactionCount: 34, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1535668009000, + dateModified: 1545082454000, + customFields: { '261066': 394015, '261067': 394027 }, + }, + { + id: 14667112, + name: 'Bamboo Relayer', + assigneeId: 680302, + closeDate: '11/19/2018', + companyId: 29243795, + companyName: 'Bamboo Relay', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2405442, + primaryContactId: 66914687, + priority: 'None', + status: 'Won', + interactionCount: 46, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1542655143000, + dateModified: 1545253761000, + customFields: { '261066': 394013, '261067': 394023 }, + }, + { + id: 13627309, + name: 'Ben TW', + assigneeId: 680302, + closeDate: '1/1/2019', + companyId: 27702348, + companyName: 'Ben', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2392929, + primaryContactId: 64262622, + priority: 'None', + status: 'Open', + interactionCount: 64, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1534887789000, + dateModified: 1541651395000, + customFields: { '261066': 394015, '261067': 394027 }, + }, + { + id: 14808512, + name: 'Bit2Me Relayer', + assigneeId: 680302, + closeDate: '12/3/2018', + companyId: 30793050, + companyName: 'Bit2Me', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2405442, + primaryContactId: 70267217, + priority: 'None', + status: 'Won', + interactionCount: 0, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1543861167000, + dateModified: 1543861189000, + customFields: { '261066': 394013, '261067': 394023 }, + }, + { + id: 14050312, + name: 'Bitcoin.tax Reporting Integration', + assigneeId: 680302, + closeDate: '11/1/2018', + companyId: 27957614, + companyName: 'Bitcoin', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2392928, + primaryContactId: 66539479, + priority: 'None', + status: 'Open', + interactionCount: 5, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1538414308000, + dateModified: 1538414314000, + customFields: { '261066': 394019, '261067': 394026 }, + }, + { + id: 14331463, + name: 'Bitpie TW', + assigneeId: 680302, + closeDate: '11/19/2018', + companyId: 27779026, + companyName: 'Bitpie', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2392929, + primaryContactId: 67700943, + priority: 'None', + status: 'Open', + interactionCount: 9, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1539984566000, + dateModified: 1541530233000, + customFields: { '261066': 394015, '261067': 394027 }, + }, + { + id: 14331481, + name: 'Bitski Wallet SDK TW', + assigneeId: 680302, + closeDate: '11/19/2018', + companyId: 29489300, + companyName: 'Bitski', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2392929, + primaryContactId: 67697528, + priority: 'None', + status: 'Open', + interactionCount: 23, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1539984735000, + dateModified: 1544818605000, + customFields: { '261066': 394015, '261067': 394026 }, + }, + { + id: 14531554, + name: 'BitUniverse TW', + assigneeId: 680302, + closeDate: '12/6/2018', + companyId: 29901805, + companyName: 'BitUniverse Co., Ltd (Cryptocurrency Portfolio)', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2392929, + primaryContactId: 68692107, + priority: 'None', + status: 'Open', + interactionCount: 15, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1541527110000, + dateModified: 1544812979000, + customFields: { '261066': 394015, '261067': 394026 }, + }, + { + id: 14050895, + name: 'BlitzPredict PMR', + assigneeId: 680302, + closeDate: '11/1/2018', + companyId: 28758258, + companyName: 'BlitzPredict', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2392929, + primaryContactId: 66378659, + priority: 'None', + status: 'Open', + interactionCount: 32, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1538416597000, + dateModified: 1544830709000, + customFields: { '261066': 394016, '261067': 394023 }, + }, + { + id: 14209841, + name: 'Blockfolio TW', + assigneeId: 680302, + closeDate: '11/15/2018', + companyId: 29332516, + companyName: 'Blockfolio', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2405443, + primaryContactId: 67247027, + priority: 'None', + status: 'Open', + interactionCount: 20, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1539624801000, + dateModified: 1539984098000, + customFields: { '261066': 394015, '261067': 394026 }, + }, + { + id: 14633220, + name: 'BlockSwap 721 / 1155 Conversational Marketplace', + assigneeId: 680302, + closeDate: '12/15/2018', + companyId: 30210921, + companyName: 'BlockSwap', + customerSourceId: undefined, + lossReasonId: undefined, + pipelineId: 512676, + pipelineStageId: 2392929, + primaryContactId: 69296220, + priority: 'None', + status: 'Open', + interactionCount: 82, + monetaryValue: undefined, + winProbability: 0, + dateCreated: 1542311056000, + dateModified: 1543557877000, + customFields: { '261066': 394014, '261067': 394023 }, + }, +]; +export { ParsedOpportunities }; diff --git a/packages/pipeline/test/fixtures/copper/parsed_entities.ts b/packages/pipeline/test/fixtures/copper/parsed_entities.ts new file mode 100644 index 000000000..1f49d38ed --- /dev/null +++ b/packages/pipeline/test/fixtures/copper/parsed_entities.ts @@ -0,0 +1,5 @@ +export { ParsedActivityTypes } from './api_v1_activity_types'; +export { ParsedCustomFields } from './api_v1_custom_field_definitions'; +export { ParsedActivities } from './api_v1_list_activities'; +export { ParsedLeads } from './api_v1_list_leads'; +export { ParsedOpportunities } from './api_v1_list_opportunities'; diff --git a/packages/pipeline/test/parsers/copper/index_test.ts b/packages/pipeline/test/parsers/copper/index_test.ts new file mode 100644 index 000000000..bb8e70da1 --- /dev/null +++ b/packages/pipeline/test/parsers/copper/index_test.ts @@ -0,0 +1,87 @@ +import * as chai from 'chai'; +import 'mocha'; + +import { + CopperActivity, + CopperActivityType, + CopperCustomField, + CopperLead, + CopperOpportunity, +} from '../../../src/entities'; +import { + CopperActivityResponse, + CopperActivityTypeCategory, + CopperActivityTypeResponse, + CopperCustomFieldResponse, + CopperSearchResponse, + parseActivities, + parseActivityTypes, + parseCustomFields, + parseLeads, + parseOpportunities, +} from '../../../src/parsers/copper'; +import { chaiSetup } from '../../utils/chai_setup'; + +chaiSetup.configure(); +const expect = chai.expect; + +type CopperResponse = CopperSearchResponse | CopperCustomFieldResponse; +type CopperEntity = CopperLead | CopperActivity | CopperOpportunity | CopperActivityType | CopperCustomField; + +import * as activityTypesApiResponse from '../../fixtures/copper/api_v1_activity_types.json'; +import * as customFieldsApiResponse from '../../fixtures/copper/api_v1_custom_field_definitions.json'; +import * as listActivitiesApiResponse from '../../fixtures/copper/api_v1_list_activities.json'; +import * as listLeadsApiResponse from '../../fixtures/copper/api_v1_list_leads.json'; +import * as listOpportunitiesApiResponse from '../../fixtures/copper/api_v1_list_opportunities.json'; +import { + ParsedActivities, + ParsedActivityTypes, + ParsedCustomFields, + ParsedLeads, + ParsedOpportunities, +} from '../../fixtures/copper/parsed_entities'; + +interface TestCase { + input: CopperResponse[]; + expected: CopperEntity[]; + parseFn(input: CopperResponse[]): CopperEntity[]; +} +const testCases: TestCase[] = [ + { + input: listLeadsApiResponse, + expected: ParsedLeads, + parseFn: parseLeads, + }, + { + input: (listActivitiesApiResponse as unknown) as CopperActivityResponse[], + expected: ParsedActivities, + parseFn: parseActivities, + }, + { + input: listOpportunitiesApiResponse, + expected: ParsedOpportunities, + parseFn: parseOpportunities, + }, + { + input: customFieldsApiResponse, + expected: ParsedCustomFields, + parseFn: parseCustomFields, + }, +]; +describe('Copper parser', () => { + it('parses API responses', () => { + testCases.forEach(testCase => { + const actual: CopperEntity[] = testCase.parseFn(testCase.input); + expect(actual).deep.equal(testCase.expected); + }); + }); + + // special case because the API response is not an array + it('parses activity types API response', () => { + const actual: CopperActivityType[] = parseActivityTypes((activityTypesApiResponse as unknown) as Map< + CopperActivityTypeCategory, + CopperActivityTypeResponse[] + >); + expect(actual).deep.equal(ParsedActivityTypes); + }); +}); diff --git a/packages/pipeline/test/parsers/ddex_orders/index_test.ts b/packages/pipeline/test/parsers/ddex_orders/index_test.ts index 4a4a86bf8..f30e86b02 100644 --- a/packages/pipeline/test/parsers/ddex_orders/index_test.ts +++ b/packages/pipeline/test/parsers/ddex_orders/index_test.ts @@ -31,13 +31,13 @@ describe('ddex_orders', () => { amountDecimals: 0, }; const observedTimestamp: number = Date.now(); - const orderType: OrderType = 'bid'; + const orderType: OrderType = OrderType.Bid; const source: string = 'ddex'; const expected = new TokenOrder(); expected.source = 'ddex'; expected.observedTimestamp = observedTimestamp; - expected.orderType = 'bid'; + expected.orderType = OrderType.Bid; expected.price = new BigNumber(0.5); // ddex currently confuses base and quote assets. // Switch them to maintain our internal consistency. diff --git a/packages/pipeline/test/parsers/events/exchange_events_test.ts b/packages/pipeline/test/parsers/events/exchange_events_test.ts index 5d4b185a5..956ad9ef8 100644 --- a/packages/pipeline/test/parsers/events/exchange_events_test.ts +++ b/packages/pipeline/test/parsers/events/exchange_events_test.ts @@ -6,6 +6,7 @@ import 'mocha'; import { ExchangeFillEvent } from '../../../src/entities'; import { _convertToExchangeFillEvent } from '../../../src/parsers/events/exchange_events'; +import { AssetType } from '../../../src/types'; import { chaiSetup } from '../../utils/chai_setup'; chaiSetup.configure(); @@ -62,12 +63,12 @@ describe('exchange_events', () => { expected.takerFeePaid = new BigNumber('12345'); expected.orderHash = '0xab12ed2cbaa5615ab690b9da75a46e53ddfcf3f1a68655b5fe0d94c75a1aac4a'; expected.rawMakerAssetData = '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2'; - expected.makerAssetType = 'erc20'; + expected.makerAssetType = AssetType.ERC20; expected.makerAssetProxyId = '0xf47261b0'; expected.makerTokenAddress = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2'; expected.makerTokenId = null; expected.rawTakerAssetData = '0xf47261b0000000000000000000000000e41d2489571d322189246dafa5ebde1f4699f498'; - expected.takerAssetType = 'erc20'; + expected.takerAssetType = AssetType.ERC20; expected.takerAssetProxyId = '0xf47261b0'; expected.takerTokenAddress = '0xe41d2489571d322189246dafa5ebde1f4699f498'; expected.takerTokenId = null; diff --git a/packages/pipeline/test/parsers/idex_orders/index_test.ts b/packages/pipeline/test/parsers/idex_orders/index_test.ts index d54ecb9a8..48b019732 100644 --- a/packages/pipeline/test/parsers/idex_orders/index_test.ts +++ b/packages/pipeline/test/parsers/idex_orders/index_test.ts @@ -31,13 +31,13 @@ describe('idex_orders', () => { user: '0x212345667543456435324564345643453453333', }; const observedTimestamp: number = Date.now(); - const orderType: OrderType = 'bid'; + const orderType: OrderType = OrderType.Bid; const source: string = 'idex'; const expected = new TokenOrder(); expected.source = 'idex'; expected.observedTimestamp = observedTimestamp; - expected.orderType = 'bid'; + expected.orderType = OrderType.Bid; expected.price = new BigNumber(0.5); expected.baseAssetSymbol = 'ABC'; expected.baseAssetAddress = '0x0000000000000000000000000000000000000000'; @@ -65,13 +65,13 @@ describe('idex_orders', () => { user: '0x212345667543456435324564345643453453333', }; const observedTimestamp: number = Date.now(); - const orderType: OrderType = 'ask'; + const orderType: OrderType = OrderType.Ask; const source: string = 'idex'; const expected = new TokenOrder(); expected.source = 'idex'; expected.observedTimestamp = observedTimestamp; - expected.orderType = 'ask'; + expected.orderType = OrderType.Ask; expected.price = new BigNumber(0.5); expected.baseAssetSymbol = 'ABC'; expected.baseAssetAddress = '0x0000000000000000000000000000000000000000'; diff --git a/packages/pipeline/test/parsers/oasis_orders/index_test.ts b/packages/pipeline/test/parsers/oasis_orders/index_test.ts index 433bfb665..401fedff8 100644 --- a/packages/pipeline/test/parsers/oasis_orders/index_test.ts +++ b/packages/pipeline/test/parsers/oasis_orders/index_test.ts @@ -27,13 +27,13 @@ describe('oasis_orders', () => { low: 0, }; const observedTimestamp: number = Date.now(); - const orderType: OrderType = 'bid'; + const orderType: OrderType = OrderType.Bid; const source: string = 'oasis'; const expected = new TokenOrder(); expected.source = 'oasis'; expected.observedTimestamp = observedTimestamp; - expected.orderType = 'bid'; + expected.orderType = OrderType.Bid; expected.price = new BigNumber(0.5); expected.baseAssetSymbol = 'DEF'; expected.baseAssetAddress = null; diff --git a/packages/pipeline/test/parsers/paradex_orders/index_test.ts b/packages/pipeline/test/parsers/paradex_orders/index_test.ts index 6b811b90d..c5dd8751b 100644 --- a/packages/pipeline/test/parsers/paradex_orders/index_test.ts +++ b/packages/pipeline/test/parsers/paradex_orders/index_test.ts @@ -32,13 +32,13 @@ describe('paradex_orders', () => { quoteTokenAddress: '0x0000000000000000000000000000000000000000', }; const observedTimestamp: number = Date.now(); - const orderType: OrderType = 'bid'; + const orderType: OrderType = OrderType.Bid; const source: string = 'paradex'; const expected = new TokenOrder(); expected.source = 'paradex'; expected.observedTimestamp = observedTimestamp; - expected.orderType = 'bid'; + expected.orderType = OrderType.Bid; expected.price = new BigNumber(0.1245); expected.baseAssetSymbol = 'DEF'; expected.baseAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81'; diff --git a/packages/pipeline/test/parsers/sra_orders/index_test.ts b/packages/pipeline/test/parsers/sra_orders/index_test.ts index ee2842ef3..838171a72 100644 --- a/packages/pipeline/test/parsers/sra_orders/index_test.ts +++ b/packages/pipeline/test/parsers/sra_orders/index_test.ts @@ -5,6 +5,7 @@ import 'mocha'; import { SraOrder } from '../../../src/entities'; import { _convertToEntity } from '../../../src/parsers/sra_orders'; +import { AssetType } from '../../../src/types'; import { chaiSetup } from '../../utils/chai_setup'; chaiSetup.configure(); @@ -50,12 +51,12 @@ describe('sra_orders', () => { expected.signature = '0x1b5a5d672b0d647b5797387ccbb89d822d5d2e873346b014f4ff816ff0783f2a7a0d2824d2d7042ec8ea375bc7f870963e1cb8248f1db03ddf125e27b5963aa11f03'; expected.rawMakerAssetData = '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2'; - expected.makerAssetType = 'erc20'; + expected.makerAssetType = AssetType.ERC20; expected.makerAssetProxyId = '0xf47261b0'; expected.makerTokenAddress = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2'; expected.makerTokenId = null; expected.rawTakerAssetData = '0xf47261b000000000000000000000000042d6622dece394b54999fbd73d108123806f6a18'; - expected.takerAssetType = 'erc20'; + expected.takerAssetType = AssetType.ERC20; expected.takerAssetProxyId = '0xf47261b0'; expected.takerTokenAddress = '0x42d6622dece394b54999fbd73d108123806f6a18'; expected.takerTokenId = null; diff --git a/packages/pipeline/tsconfig.json b/packages/pipeline/tsconfig.json index 6f138f260..45e07374c 100644 --- a/packages/pipeline/tsconfig.json +++ b/packages/pipeline/tsconfig.json @@ -4,7 +4,15 @@ "outDir": "lib", "rootDir": ".", "emitDecoratorMetadata": true, - "experimentalDecorators": true + "experimentalDecorators": true, + "resolveJsonModule": true }, - "include": ["./src/**/*", "./test/**/*", "./migrations/**/*"] + "include": ["./src/**/*", "./test/**/*", "./migrations/**/*"], + "files": [ + "./test/fixtures/copper/api_v1_activity_types.json", + "./test/fixtures/copper/api_v1_custom_field_definitions.json", + "./test/fixtures/copper/api_v1_list_activities.json", + "./test/fixtures/copper/api_v1_list_leads.json", + "./test/fixtures/copper/api_v1_list_opportunities.json" + ] } |