2 files changed, 127 insertions, 2 deletions
diff --git a/packages/pipeline/src/data_sources/copper/index.ts b/packages/pipeline/src/data_sources/copper/index.ts
new file mode 100644
index 000000000..15df2fd7d
--- /dev/null
+++ b/packages/pipeline/src/data_sources/copper/index.ts
@@ -0,0 +1,126 @@
+import { fetchAsync } from '@0x/utils';
+import Bottleneck from 'bottleneck';
+
+import {
+    CopperActivityTypeCategory,
+    CopperActivityTypeResponse,
+    CopperCustomFieldResponse,
+    CopperSearchResponse,
+} from '../../parsers/copper';
+
+const HTTP_OK_STATUS = 200;
+const COPPER_URI = 'https://api.prosperworks.com/developer_api/v1';
+
+const DEFAULT_PAGINATION_PARAMS = {
+    page_size: 200,
+    sort_by: 'date_modified',
+    sort_direction: 'desc',
+};
+
+export type CopperSearchParams = CopperLeadSearchParams | CopperActivitySearchParams | CopperOpportunitySearchParams;
+export interface CopperLeadSearchParams {
+    page_number?: number;
+}
+
+export interface CopperActivitySearchParams {
+    minimum_activity_date: number;
+    page_number?: number;
+}
+
+export interface CopperOpportunitySearchParams {
+    sort_by: string; // must override the default 'date_modified' for this endpoint
+    page_number?: number;
+}
+export enum CopperEndpoint {
+    Leads = '/leads/search',
+    Opportunities = '/opportunities/search',
+    Activities = '/activities/search',
+}
+const ONE_SECOND = 1000;
+
+function httpErrorCheck(response: Response): void {
+    if (response.status !== HTTP_OK_STATUS) {
+        throw new Error(`HTTP error while scraping Copper: [${JSON.stringify(response)}]`);
+    }
+}
+export class CopperSource {
+    private readonly _accessToken: string;
+    private readonly _userEmail: string;
+    private readonly _defaultHeaders: any;
+    private readonly _limiter: Bottleneck;
+
+    constructor(maxConcurrentRequests: number, accessToken: string, userEmail: string) {
+        this._accessToken = accessToken;
+        this._userEmail = userEmail;
+        this._defaultHeaders = {
+            'Content-Type': 'application/json',
+            'X-PW-AccessToken': this._accessToken,
+            'X-PW-Application': 'developer_api',
+            'X-PW-UserEmail': this._userEmail,
+        };
+        this._limiter = new Bottleneck({
+            minTime: ONE_SECOND / maxConcurrentRequests,
+            reservoir: 30,
+            reservoirRefreshAmount: 30,
+            reservoirRefreshInterval: maxConcurrentRequests,
+        });
+    }
+
+    public async fetchNumberOfPagesAsync(endpoint: CopperEndpoint, searchParams?: CopperSearchParams): Promise<number> {
+        const resp = await this._limiter.schedule(() =>
+            fetchAsync(COPPER_URI + endpoint, {
+                method: 'POST',
+                body: JSON.stringify({ ...DEFAULT_PAGINATION_PARAMS, ...searchParams }),
+                headers: this._defaultHeaders,
+            }),
+        );
+
+        httpErrorCheck(resp);
+
+        // total number of records that match the request parameters
+        if (resp.headers.has('X-Pw-Total')) {
+            const totalRecords: number = parseInt(resp.headers.get('X-Pw-Total') as string, 10); // tslint:disable-line:custom-no-magic-numbers
+            return Math.ceil(totalRecords / DEFAULT_PAGINATION_PARAMS.page_size);
+        } else {
+            return 1;
+        }
+    }
+    public async fetchSearchResultsAsync<T extends CopperSearchResponse>(
+        endpoint: CopperEndpoint,
+        searchParams?: CopperSearchParams,
+    ): Promise<T[]> {
+        const request = { ...DEFAULT_PAGINATION_PARAMS, ...searchParams };
+        const response = await this._limiter.schedule(() =>
+            fetchAsync(COPPER_URI + endpoint, {
+                method: 'POST',
+                body: JSON.stringify(request),
+                headers: this._defaultHeaders,
+            }),
+        );
+        httpErrorCheck(response);
+        const json: T[] = await response.json();
+        return json;
+    }
+
+    public async fetchActivityTypesAsync(): Promise<Map<CopperActivityTypeCategory, CopperActivityTypeResponse[]>> {
+        const response = await this._limiter.schedule(() =>
+            fetchAsync(`${COPPER_URI}/activity_types`, {
+                method: 'GET',
+                headers: this._defaultHeaders,
+            }),
+        );
+        httpErrorCheck(response);
+        return response.json();
+    }
+
+    public async fetchCustomFieldsAsync(): Promise<CopperCustomFieldResponse[]> {
+        const response = await this._limiter.schedule(() =>
+            fetchAsync(`${COPPER_URI}/custom_field_definitions`, {
+                method: 'GET',
+                headers: this._defaultHeaders,
+            }),
+        );
+        httpErrorCheck(response);
+        return response.json();
+    }
+}
diff --git a/packages/pipeline/src/data_sources/ddex/index.ts b/packages/pipeline/src/data_sources/ddex/index.ts
index 2bbd8c29b..7ef92b90f 100644
--- a/packages/pipeline/src/data_sources/ddex/index.ts
+++ b/packages/pipeline/src/data_sources/ddex/index.ts
@@ -1,6 +1,6 @@
 import { fetchAsync, logUtils } from '@0x/utils';
 
-const DDEX_BASE_URL = 'https://api.ddex.io/v2';
+const DDEX_BASE_URL = 'https://api.ddex.io/v3';
 const ACTIVE_MARKETS_URL = `${DDEX_BASE_URL}/markets`;
 const NO_AGGREGATION_LEVEL = 3; // See https://docs.ddex.io/#get-orderbook
 const ORDERBOOK_ENDPOINT = `/orderbook?level=${NO_AGGREGATION_LEVEL}`;
@@ -23,7 +23,6 @@ export interface DdexMarket {
     baseTokenDecimals: number;
     baseTokenAddress: string;
     minOrderSize: string;
-    maxOrderSize: string;
     pricePrecision: number;
     priceDecimals: number;
     amountDecimals: number;