diff options
author | Leonid Logvinov <logvinov.leon@gmail.com> | 2019-01-10 18:51:13 +0800 |
---|---|---|
committer | Leonid Logvinov <logvinov.leon@gmail.com> | 2019-01-10 18:56:15 +0800 |
commit | cee29542451d3bf8c99bd08963a2108768072195 (patch) | |
tree | f3d657be53459ce3851fcf9632bd94f32ee80184 /packages/pipeline/src/data_sources/copper/index.ts | |
parent | a8d9263062e586b90ee4c303d3d3aca72e428edc (diff) | |
parent | 686f27a96f0cd749f6315d7edd2bb56cf1819245 (diff) | |
download | dexon-0x-contracts-cee29542451d3bf8c99bd08963a2108768072195.tar dexon-0x-contracts-cee29542451d3bf8c99bd08963a2108768072195.tar.gz dexon-0x-contracts-cee29542451d3bf8c99bd08963a2108768072195.tar.bz2 dexon-0x-contracts-cee29542451d3bf8c99bd08963a2108768072195.tar.lz dexon-0x-contracts-cee29542451d3bf8c99bd08963a2108768072195.tar.xz dexon-0x-contracts-cee29542451d3bf8c99bd08963a2108768072195.tar.zst dexon-0x-contracts-cee29542451d3bf8c99bd08963a2108768072195.zip |
Merge development
Diffstat (limited to 'packages/pipeline/src/data_sources/copper/index.ts')
-rw-r--r-- | packages/pipeline/src/data_sources/copper/index.ts | 126 |
1 files changed, 126 insertions, 0 deletions
diff --git a/packages/pipeline/src/data_sources/copper/index.ts b/packages/pipeline/src/data_sources/copper/index.ts new file mode 100644 index 000000000..15df2fd7d --- /dev/null +++ b/packages/pipeline/src/data_sources/copper/index.ts @@ -0,0 +1,126 @@ +import { fetchAsync } from '@0x/utils'; +import Bottleneck from 'bottleneck'; + +import { + CopperActivityTypeCategory, + CopperActivityTypeResponse, + CopperCustomFieldResponse, + CopperSearchResponse, +} from '../../parsers/copper'; + +const HTTP_OK_STATUS = 200; +const COPPER_URI = 'https://api.prosperworks.com/developer_api/v1'; + +const DEFAULT_PAGINATION_PARAMS = { + page_size: 200, + sort_by: 'date_modified', + sort_direction: 'desc', +}; + +export type CopperSearchParams = CopperLeadSearchParams | CopperActivitySearchParams | CopperOpportunitySearchParams; +export interface CopperLeadSearchParams { + page_number?: number; +} + +export interface CopperActivitySearchParams { + minimum_activity_date: number; + page_number?: number; +} + +export interface CopperOpportunitySearchParams { + sort_by: string; // must override the default 'date_modified' for this endpoint + page_number?: number; +} +export enum CopperEndpoint { + Leads = '/leads/search', + Opportunities = '/opportunities/search', + Activities = '/activities/search', +} +const ONE_SECOND = 1000; + +function httpErrorCheck(response: Response): void { + if (response.status !== HTTP_OK_STATUS) { + throw new Error(`HTTP error while scraping Copper: [${JSON.stringify(response)}]`); + } +} +export class CopperSource { + private readonly _accessToken: string; + private readonly _userEmail: string; + private readonly _defaultHeaders: any; + private readonly _limiter: Bottleneck; + + constructor(maxConcurrentRequests: number, accessToken: string, userEmail: string) { + this._accessToken = accessToken; + this._userEmail = userEmail; + this._defaultHeaders = { + 'Content-Type': 'application/json', + 'X-PW-AccessToken': this._accessToken, + 'X-PW-Application': 'developer_api', + 'X-PW-UserEmail': this._userEmail, + }; + this._limiter = new Bottleneck({ + minTime: ONE_SECOND / maxConcurrentRequests, + reservoir: 30, + reservoirRefreshAmount: 30, + reservoirRefreshInterval: maxConcurrentRequests, + }); + } + + public async fetchNumberOfPagesAsync(endpoint: CopperEndpoint, searchParams?: CopperSearchParams): Promise<number> { + const resp = await this._limiter.schedule(() => + fetchAsync(COPPER_URI + endpoint, { + method: 'POST', + body: JSON.stringify({ ...DEFAULT_PAGINATION_PARAMS, ...searchParams }), + headers: this._defaultHeaders, + }), + ); + + httpErrorCheck(resp); + + // total number of records that match the request parameters + if (resp.headers.has('X-Pw-Total')) { + const totalRecords: number = parseInt(resp.headers.get('X-Pw-Total') as string, 10); // tslint:disable-line:custom-no-magic-numbers + return Math.ceil(totalRecords / DEFAULT_PAGINATION_PARAMS.page_size); + } else { + return 1; + } + } + public async fetchSearchResultsAsync<T extends CopperSearchResponse>( + endpoint: CopperEndpoint, + searchParams?: CopperSearchParams, + ): Promise<T[]> { + const request = { ...DEFAULT_PAGINATION_PARAMS, ...searchParams }; + const response = await this._limiter.schedule(() => + fetchAsync(COPPER_URI + endpoint, { + method: 'POST', + body: JSON.stringify(request), + headers: this._defaultHeaders, + }), + ); + httpErrorCheck(response); + const json: T[] = await response.json(); + return json; + } + + public async fetchActivityTypesAsync(): Promise<Map<CopperActivityTypeCategory, CopperActivityTypeResponse[]>> { + const response = await this._limiter.schedule(() => + fetchAsync(`${COPPER_URI}/activity_types`, { + method: 'GET', + headers: this._defaultHeaders, + }), + ); + httpErrorCheck(response); + return response.json(); + } + + public async fetchCustomFieldsAsync(): Promise<CopperCustomFieldResponse[]> { + const response = await this._limiter.schedule(() => + fetchAsync(`${COPPER_URI}/custom_field_definitions`, { + method: 'GET', + headers: this._defaultHeaders, + }), + ); + httpErrorCheck(response); + return response.json(); + } +} |