Merge development

11 files changed, 724 insertions, 0 deletions

diff --git a/packages/pipeline/test/parsers/bloxy/index_test.ts b/packages/pipeline/test/parsers/bloxy/index_test.ts
new file mode 100644
index 000000000..6aabb091d
--- /dev/null
+++ b/packages/pipeline/test/parsers/bloxy/index_test.ts
@@ -0,0 +1,98 @@
+// tslint:disable:custom-no-magic-numbers
+import { BigNumber } from '@0x/utils';
+import * as chai from 'chai';
+import 'mocha';
+import * as R from 'ramda';
+
+import { BLOXY_DEX_TRADES_URL, BloxyTrade } from '../../../src/data_sources/bloxy';
+import { DexTrade } from '../../../src/entities';
+import { _parseBloxyTrade } from '../../../src/parsers/bloxy';
+import { chaiSetup } from '../../utils/chai_setup';
+
+chaiSetup.configure();
+const expect = chai.expect;
+
+const baseInput: BloxyTrade = {
+    tx_hash: '0xb93a7faf92efbbb5405c9a73cd4efd99702fe27c03ff22baee1f1b1e37b3a0bf',
+    tx_time: '2018-11-21T09:06:28.000+00:00',
+    tx_date: '2018-11-21',
+    tx_sender: '0x00923b9a074762b93650716333b3e1473a15048e',
+    smart_contract_id: 7091917,
+    smart_contract_address: '0x818e6fecd516ecc3849daf6845e3ec868087b755',
+    contract_type: 'DEX/Kyber Network Proxy',
+    maker: '0x0000000000000000000000000000000000000001',
+    taker: '0x0000000000000000000000000000000000000002',
+    amountBuy: 1.011943163078103,
+    makerFee: 38.912083,
+    buyCurrencyId: 1,
+    buySymbol: 'ETH',
+    amountSell: 941.4997928436911,
+    takerFee: 100.39,
+    sellCurrencyId: 16610,
+    sellSymbol: 'ELF',
+    maker_annotation: 'random annotation',
+    taker_annotation: 'random other annotation',
+    protocol: 'Kyber Network Proxy',
+    buyAddress: '0xbf2179859fc6d5bee9bf9158632dc51678a4100d',
+    sellAddress: '0xbf2179859fc6d5bee9bf9158632dc51678a4100e',
+};
+
+const baseExpected: DexTrade = {
+    sourceUrl: BLOXY_DEX_TRADES_URL,
+    txHash: '0xb93a7faf92efbbb5405c9a73cd4efd99702fe27c03ff22baee1f1b1e37b3a0bf',
+    txTimestamp: 1542791188000,
+    txDate: '2018-11-21',
+    txSender: '0x00923b9a074762b93650716333b3e1473a15048e',
+    smartContractId: 7091917,
+    smartContractAddress: '0x818e6fecd516ecc3849daf6845e3ec868087b755',
+    contractType: 'DEX/Kyber Network Proxy',
+    maker: '0x0000000000000000000000000000000000000001',
+    taker: '0x0000000000000000000000000000000000000002',
+    amountBuy: new BigNumber('1.011943163078103'),
+    makerFeeAmount: new BigNumber('38.912083'),
+    buyCurrencyId: 1,
+    buySymbol: 'ETH',
+    amountSell: new BigNumber('941.4997928436911'),
+    takerFeeAmount: new BigNumber('100.39'),
+    sellCurrencyId: 16610,
+    sellSymbol: 'ELF',
+    makerAnnotation: 'random annotation',
+    takerAnnotation: 'random other annotation',
+    protocol: 'Kyber Network Proxy',
+    buyAddress: '0xbf2179859fc6d5bee9bf9158632dc51678a4100d',
+    sellAddress: '0xbf2179859fc6d5bee9bf9158632dc51678a4100e',
+};
+
+interface TestCase {
+    input: BloxyTrade;
+    expected: DexTrade;
+}
+
+const testCases: TestCase[] = [
+    {
+        input: baseInput,
+        expected: baseExpected,
+    },
+    {
+        input: R.merge(baseInput, { buyAddress: null, sellAddress: null }),
+        expected: R.merge(baseExpected, { buyAddress: null, sellAddress: null }),
+    },
+    {
+        input: R.merge(baseInput, {
+            buySymbol:
+                'RING\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000',
+        }),
+        expected: R.merge(baseExpected, { buySymbol: 'RING' }),
+    },
+];
+
+describe('bloxy', () => {
+    describe('_parseBloxyTrade', () => {
+        for (const [i, testCase] of testCases.entries()) {
+            it(`converts BloxyTrade to DexTrade entity (${i + 1}/${testCases.length})`, () => {
+                const actual = _parseBloxyTrade(testCase.input);
+                expect(actual).deep.equal(testCase.expected);
+            });
+        }
+    });
+});
diff --git a/packages/pipeline/test/parsers/copper/index_test.ts b/packages/pipeline/test/parsers/copper/index_test.ts
new file mode 100644
index 000000000..bb8e70da1
--- /dev/null
+++ b/packages/pipeline/test/parsers/copper/index_test.ts
@@ -0,0 +1,87 @@
+import * as chai from 'chai';
+import 'mocha';
+
+import {
+    CopperActivity,
+    CopperActivityType,
+    CopperCustomField,
+    CopperLead,
+    CopperOpportunity,
+} from '../../../src/entities';
+import {
+    CopperActivityResponse,
+    CopperActivityTypeCategory,
+    CopperActivityTypeResponse,
+    CopperCustomFieldResponse,
+    CopperSearchResponse,
+    parseActivities,
+    parseActivityTypes,
+    parseCustomFields,
+    parseLeads,
+    parseOpportunities,
+} from '../../../src/parsers/copper';
+import { chaiSetup } from '../../utils/chai_setup';
+
+chaiSetup.configure();
+const expect = chai.expect;
+
+type CopperResponse = CopperSearchResponse | CopperCustomFieldResponse;
+type CopperEntity = CopperLead | CopperActivity | CopperOpportunity | CopperActivityType | CopperCustomField;
+
+import * as activityTypesApiResponse from '../../fixtures/copper/api_v1_activity_types.json';
+import * as customFieldsApiResponse from '../../fixtures/copper/api_v1_custom_field_definitions.json';
+import * as listActivitiesApiResponse from '../../fixtures/copper/api_v1_list_activities.json';
+import * as listLeadsApiResponse from '../../fixtures/copper/api_v1_list_leads.json';
+import * as listOpportunitiesApiResponse from '../../fixtures/copper/api_v1_list_opportunities.json';
+import {
+    ParsedActivities,
+    ParsedActivityTypes,
+    ParsedCustomFields,
+    ParsedLeads,
+    ParsedOpportunities,
+} from '../../fixtures/copper/parsed_entities';
+
+interface TestCase {
+    input: CopperResponse[];
+    expected: CopperEntity[];
+    parseFn(input: CopperResponse[]): CopperEntity[];
+}
+const testCases: TestCase[] = [
+    {
+        input: listLeadsApiResponse,
+        expected: ParsedLeads,
+        parseFn: parseLeads,
+    },
+    {
+        input: (listActivitiesApiResponse as unknown) as CopperActivityResponse[],
+        expected: ParsedActivities,
+        parseFn: parseActivities,
+    },
+    {
+        input: listOpportunitiesApiResponse,
+        expected: ParsedOpportunities,
+        parseFn: parseOpportunities,
+    },
+    {
+        input: customFieldsApiResponse,
+        expected: ParsedCustomFields,
+        parseFn: parseCustomFields,
+    },
+];
+describe('Copper parser', () => {
+    it('parses API responses', () => {
+        testCases.forEach(testCase => {
+            const actual: CopperEntity[] = testCase.parseFn(testCase.input);
+            expect(actual).deep.equal(testCase.expected);
+        });
+    });
+
+    // special case because the API response is not an array
+    it('parses activity types API response', () => {
+        const actual: CopperActivityType[] = parseActivityTypes((activityTypesApiResponse as unknown) as Map<
+            CopperActivityTypeCategory,
+            CopperActivityTypeResponse[]
+        >);
+        expect(actual).deep.equal(ParsedActivityTypes);
+    });
+});
diff --git a/packages/pipeline/test/parsers/ddex_orders/index_test.ts b/packages/pipeline/test/parsers/ddex_orders/index_test.ts
new file mode 100644
index 000000000..f30e86b02
--- /dev/null
+++ b/packages/pipeline/test/parsers/ddex_orders/index_test.ts
@@ -0,0 +1,55 @@
+import { BigNumber } from '@0x/utils';
+import * as chai from 'chai';
+import 'mocha';
+
+import { DdexMarket } from '../../../src/data_sources/ddex';
+import { TokenOrderbookSnapshot as TokenOrder } from '../../../src/entities';
+import { parseDdexOrder } from '../../../src/parsers/ddex_orders';
+import { OrderType } from '../../../src/types';
+import { chaiSetup } from '../../utils/chai_setup';
+
+chaiSetup.configure();
+const expect = chai.expect;
+
+// tslint:disable:custom-no-magic-numbers
+describe('ddex_orders', () => {
+    describe('parseDdexOrder', () => {
+        it('converts ddexOrder to TokenOrder entity', () => {
+            const ddexOrder: [string, BigNumber] = ['0.5', new BigNumber(10)];
+            const ddexMarket: DdexMarket = {
+                id: 'ABC-DEF',
+                quoteToken: 'ABC',
+                quoteTokenDecimals: 5,
+                quoteTokenAddress: '0x0000000000000000000000000000000000000000',
+                baseToken: 'DEF',
+                baseTokenDecimals: 2,
+                baseTokenAddress: '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81',
+                minOrderSize: '0.1',
+                maxOrderSize: '1000',
+                pricePrecision: 1,
+                priceDecimals: 1,
+                amountDecimals: 0,
+            };
+            const observedTimestamp: number = Date.now();
+            const orderType: OrderType = OrderType.Bid;
+            const source: string = 'ddex';
+
+            const expected = new TokenOrder();
+            expected.source = 'ddex';
+            expected.observedTimestamp = observedTimestamp;
+            expected.orderType = OrderType.Bid;
+            expected.price = new BigNumber(0.5);
+            // ddex currently confuses base and quote assets.
+            // Switch them to maintain our internal consistency.
+            expected.baseAssetSymbol = 'ABC';
+            expected.baseAssetAddress = '0x0000000000000000000000000000000000000000';
+            expected.baseVolume = new BigNumber(10);
+            expected.quoteAssetSymbol = 'DEF';
+            expected.quoteAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81';
+            expected.quoteVolume = new BigNumber(5);
+
+            const actual = parseDdexOrder(ddexMarket, observedTimestamp, orderType, source, ddexOrder);
+            expect(actual).deep.equal(expected);
+        });
+    });
+});
diff --git a/packages/pipeline/test/parsers/events/erc20_events_test.ts b/packages/pipeline/test/parsers/events/erc20_events_test.ts
new file mode 100644
index 000000000..962c50f98
--- /dev/null
+++ b/packages/pipeline/test/parsers/events/erc20_events_test.ts
@@ -0,0 +1,54 @@
+import { ERC20TokenApprovalEventArgs } from '@0x/contract-wrappers';
+import { BigNumber } from '@0x/utils';
+import * as chai from 'chai';
+import { LogWithDecodedArgs } from 'ethereum-types';
+import 'mocha';
+
+import { ERC20ApprovalEvent } from '../../../src/entities';
+import { _convertToERC20ApprovalEvent } from '../../../src/parsers/events/erc20_events';
+import { _convertToExchangeFillEvent } from '../../../src/parsers/events/exchange_events';
+import { chaiSetup } from '../../utils/chai_setup';
+
+chaiSetup.configure();
+const expect = chai.expect;
+
+// tslint:disable:custom-no-magic-numbers
+describe('erc20_events', () => {
+    describe('_convertToERC20ApprovalEvent', () => {
+        it('converts LogWithDecodedArgs to ERC20ApprovalEvent entity', () => {
+            const input: LogWithDecodedArgs<ERC20TokenApprovalEventArgs> = {
+                address: '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2',
+                blockHash: '0xd2d7aafaa7102aec0bca8ef026d5a85133e87892334c46ee1e92e42912991c9b',
+                blockNumber: 6281577,
+                data: '0x000000000000000000000000000000000000000000000002b9cba5ee21ad3df9',
+                logIndex: 43,
+                topics: [
+                    '0x8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925',
+                    '0x0000000000000000000000000b65c5f6f3a05d6be5588a72b603360773b3fe04',
+                    '0x000000000000000000000000448a5065aebb8e423f0896e6c5d525c040f59af3',
+                ],
+                transactionHash: '0xcb46b19c786376a0a0140d51e3e606a4c4f926d8ca5434e96d2f69d04d8d9c7f',
+                transactionIndex: 103,
+                event: 'Approval',
+                args: {
+                    _owner: '0x0b65c5f6f3a05d6be5588a72b603360773b3fe04',
+                    _spender: '0x448a5065aebb8e423f0896e6c5d525c040f59af3',
+                    _value: new BigNumber('50281464906893835769'),
+                },
+            };
+
+            const expected = new ERC20ApprovalEvent();
+            expected.tokenAddress = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2';
+            expected.blockNumber = 6281577;
+            expected.rawData = '0x000000000000000000000000000000000000000000000002b9cba5ee21ad3df9';
+            expected.logIndex = 43;
+            expected.transactionHash = '0xcb46b19c786376a0a0140d51e3e606a4c4f926d8ca5434e96d2f69d04d8d9c7f';
+            expected.ownerAddress = '0x0b65c5f6f3a05d6be5588a72b603360773b3fe04';
+            expected.spenderAddress = '0x448a5065aebb8e423f0896e6c5d525c040f59af3';
+            expected.amount = new BigNumber('50281464906893835769');
+
+            const actual = _convertToERC20ApprovalEvent(input);
+            expect(actual).deep.equal(expected);
+        });
+    });
+});
diff --git a/packages/pipeline/test/parsers/events/exchange_events_test.ts b/packages/pipeline/test/parsers/events/exchange_events_test.ts
new file mode 100644
index 000000000..956ad9ef8
--- /dev/null
+++ b/packages/pipeline/test/parsers/events/exchange_events_test.ts
@@ -0,0 +1,79 @@
+import { ExchangeFillEventArgs } from '@0x/contract-wrappers';
+import { BigNumber } from '@0x/utils';
+import * as chai from 'chai';
+import { LogWithDecodedArgs } from 'ethereum-types';
+import 'mocha';
+
+import { ExchangeFillEvent } from '../../../src/entities';
+import { _convertToExchangeFillEvent } from '../../../src/parsers/events/exchange_events';
+import { AssetType } from '../../../src/types';
+import { chaiSetup } from '../../utils/chai_setup';
+
+chaiSetup.configure();
+const expect = chai.expect;
+
+// tslint:disable:custom-no-magic-numbers
+describe('exchange_events', () => {
+    describe('_convertToExchangeFillEvent', () => {
+        it('converts LogWithDecodedArgs to ExchangeFillEvent entity', () => {
+            const input: LogWithDecodedArgs<ExchangeFillEventArgs> = {
+                logIndex: 102,
+                transactionIndex: 38,
+                transactionHash: '0x6dd106d002873746072fc5e496dd0fb2541b68c77bcf9184ae19a42fd33657fe',
+                blockHash: '',
+                blockNumber: 6276262,
+                address: '0x4f833a24e1f95d70f028921e27040ca56e09ab0b',
+                data:
+                    '0x000000000000000000000000f6da68519f78b0d0bc93c701e86affcb75c92428000000000000000000000000f6da68519f78b0d0bc93c701e86affcb75c92428000000000000000000000000000000000000000000000000002386f26fc10000000000000000000000000000000000000000000000000000016345785d8a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000001600000000000000000000000000000000000000000000000000000000000000024f47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000024f47261b0000000000000000000000000e41d2489571d322189246dafa5ebde1f4699f49800000000000000000000000000000000000000000000000000000000',
+                topics: [
+                    '0x0bcc4c97732e47d9946f229edb95f5b6323f601300e4690de719993f3c371129',
+                    '0x000000000000000000000000f6da68519f78b0d0bc93c701e86affcb75c92428',
+                    '0x000000000000000000000000c370d2a5920344aa6b7d8d11250e3e861434cbdd',
+                    '0xab12ed2cbaa5615ab690b9da75a46e53ddfcf3f1a68655b5fe0d94c75a1aac4a',
+                ],
+                event: 'Fill',
+                args: {
+                    makerAddress: '0xf6da68519f78b0d0bc93c701e86affcb75c92428',
+                    feeRecipientAddress: '0xc370d2a5920344aa6b7d8d11250e3e861434cbdd',
+                    takerAddress: '0xf6da68519f78b0d0bc93c701e86affcb75c92428',
+                    senderAddress: '0xf6da68519f78b0d0bc93c701e86affcb75c92428',
+                    makerAssetFilledAmount: new BigNumber('10000000000000000'),
+                    takerAssetFilledAmount: new BigNumber('100000000000000000'),
+                    makerFeePaid: new BigNumber('0'),
+                    takerFeePaid: new BigNumber('12345'),
+                    orderHash: '0xab12ed2cbaa5615ab690b9da75a46e53ddfcf3f1a68655b5fe0d94c75a1aac4a',
+                    makerAssetData: '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2',
+                    takerAssetData: '0xf47261b0000000000000000000000000e41d2489571d322189246dafa5ebde1f4699f498',
+                },
+            };
+            const expected = new ExchangeFillEvent();
+            expected.contractAddress = '0x4f833a24e1f95d70f028921e27040ca56e09ab0b';
+            expected.blockNumber = 6276262;
+            expected.logIndex = 102;
+            expected.rawData =
+                '0x000000000000000000000000f6da68519f78b0d0bc93c701e86affcb75c92428000000000000000000000000f6da68519f78b0d0bc93c701e86affcb75c92428000000000000000000000000000000000000000000000000002386f26fc10000000000000000000000000000000000000000000000000000016345785d8a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000001600000000000000000000000000000000000000000000000000000000000000024f47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000024f47261b0000000000000000000000000e41d2489571d322189246dafa5ebde1f4699f49800000000000000000000000000000000000000000000000000000000';
+            expected.transactionHash = '0x6dd106d002873746072fc5e496dd0fb2541b68c77bcf9184ae19a42fd33657fe';
+            expected.makerAddress = '0xf6da68519f78b0d0bc93c701e86affcb75c92428';
+            expected.takerAddress = '0xf6da68519f78b0d0bc93c701e86affcb75c92428';
+            expected.feeRecipientAddress = '0xc370d2a5920344aa6b7d8d11250e3e861434cbdd';
+            expected.senderAddress = '0xf6da68519f78b0d0bc93c701e86affcb75c92428';
+            expected.makerAssetFilledAmount = new BigNumber('10000000000000000');
+            expected.takerAssetFilledAmount = new BigNumber('100000000000000000');
+            expected.makerFeePaid = new BigNumber('0');
+            expected.takerFeePaid = new BigNumber('12345');
+            expected.orderHash = '0xab12ed2cbaa5615ab690b9da75a46e53ddfcf3f1a68655b5fe0d94c75a1aac4a';
+            expected.rawMakerAssetData = '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2';
+            expected.makerAssetType = AssetType.ERC20;
+            expected.makerAssetProxyId = '0xf47261b0';
+            expected.makerTokenAddress = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2';
+            expected.makerTokenId = null;
+            expected.rawTakerAssetData = '0xf47261b0000000000000000000000000e41d2489571d322189246dafa5ebde1f4699f498';
+            expected.takerAssetType = AssetType.ERC20;
+            expected.takerAssetProxyId = '0xf47261b0';
+            expected.takerTokenAddress = '0xe41d2489571d322189246dafa5ebde1f4699f498';
+            expected.takerTokenId = null;
+            const actual = _convertToExchangeFillEvent(input);
+            expect(actual).deep.equal(expected);
+        });
+    });
+});
diff --git a/packages/pipeline/test/parsers/idex_orders/index_test.ts b/packages/pipeline/test/parsers/idex_orders/index_test.ts
new file mode 100644
index 000000000..48b019732
--- /dev/null
+++ b/packages/pipeline/test/parsers/idex_orders/index_test.ts
@@ -0,0 +1,87 @@
+import { BigNumber } from '@0x/utils';
+import * as chai from 'chai';
+import 'mocha';
+
+import { IdexOrderParam } from '../../../src/data_sources/idex';
+import { TokenOrderbookSnapshot as TokenOrder } from '../../../src/entities';
+import { parseIdexOrder } from '../../../src/parsers/idex_orders';
+import { OrderType } from '../../../src/types';
+import { chaiSetup } from '../../utils/chai_setup';
+
+chaiSetup.configure();
+const expect = chai.expect;
+
+// tslint:disable:custom-no-magic-numbers
+describe('idex_orders', () => {
+    describe('parseIdexOrder', () => {
+        // for market listed as 'DEF_ABC'.
+        it('correctly converts bid type idexOrder to TokenOrder entity', () => {
+            const idexOrder: [string, BigNumber] = ['0.5', new BigNumber(10)];
+            const idexOrderParam: IdexOrderParam = {
+                tokenBuy: '0x0000000000000000000000000000000000000000',
+                buySymbol: 'ABC',
+                buyPrecision: 2,
+                amountBuy: '10',
+                tokenSell: '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81',
+                sellSymbol: 'DEF',
+                sellPrecision: 2,
+                amountSell: '5',
+                expires: Date.now() + 100000,
+                nonce: 1,
+                user: '0x212345667543456435324564345643453453333',
+            };
+            const observedTimestamp: number = Date.now();
+            const orderType: OrderType = OrderType.Bid;
+            const source: string = 'idex';
+
+            const expected = new TokenOrder();
+            expected.source = 'idex';
+            expected.observedTimestamp = observedTimestamp;
+            expected.orderType = OrderType.Bid;
+            expected.price = new BigNumber(0.5);
+            expected.baseAssetSymbol = 'ABC';
+            expected.baseAssetAddress = '0x0000000000000000000000000000000000000000';
+            expected.baseVolume = new BigNumber(10);
+            expected.quoteAssetSymbol = 'DEF';
+            expected.quoteAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81';
+            expected.quoteVolume = new BigNumber(5);
+
+            const actual = parseIdexOrder(idexOrderParam, observedTimestamp, orderType, source, idexOrder);
+            expect(actual).deep.equal(expected);
+        });
+        it('correctly converts ask type idexOrder to TokenOrder entity', () => {
+            const idexOrder: [string, BigNumber] = ['0.5', new BigNumber(10)];
+            const idexOrderParam: IdexOrderParam = {
+                tokenBuy: '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81',
+                buySymbol: 'DEF',
+                buyPrecision: 2,
+                amountBuy: '5',
+                tokenSell: '0x0000000000000000000000000000000000000000',
+                sellSymbol: 'ABC',
+                sellPrecision: 2,
+                amountSell: '10',
+                expires: Date.now() + 100000,
+                nonce: 1,
+                user: '0x212345667543456435324564345643453453333',
+            };
+            const observedTimestamp: number = Date.now();
+            const orderType: OrderType = OrderType.Ask;
+            const source: string = 'idex';
+
+            const expected = new TokenOrder();
+            expected.source = 'idex';
+            expected.observedTimestamp = observedTimestamp;
+            expected.orderType = OrderType.Ask;
+            expected.price = new BigNumber(0.5);
+            expected.baseAssetSymbol = 'ABC';
+            expected.baseAssetAddress = '0x0000000000000000000000000000000000000000';
+            expected.baseVolume = new BigNumber(10);
+            expected.quoteAssetSymbol = 'DEF';
+            expected.quoteAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81';
+            expected.quoteVolume = new BigNumber(5);
+
+            const actual = parseIdexOrder(idexOrderParam, observedTimestamp, orderType, source, idexOrder);
+            expect(actual).deep.equal(expected);
+        });
+    });
+});
diff --git a/packages/pipeline/test/parsers/oasis_orders/index_test.ts b/packages/pipeline/test/parsers/oasis_orders/index_test.ts
new file mode 100644
index 000000000..401fedff8
--- /dev/null
+++ b/packages/pipeline/test/parsers/oasis_orders/index_test.ts
@@ -0,0 +1,49 @@
+import { BigNumber } from '@0x/utils';
+import * as chai from 'chai';
+import 'mocha';
+
+import { OasisMarket } from '../../../src/data_sources/oasis';
+import { TokenOrderbookSnapshot as TokenOrder } from '../../../src/entities';
+import { parseOasisOrder } from '../../../src/parsers/oasis_orders';
+import { OrderType } from '../../../src/types';
+import { chaiSetup } from '../../utils/chai_setup';
+
+chaiSetup.configure();
+const expect = chai.expect;
+
+// tslint:disable:custom-no-magic-numbers
+describe('oasis_orders', () => {
+    describe('parseOasisOrder', () => {
+        it('converts oasisOrder to TokenOrder entity', () => {
+            const oasisOrder: [string, BigNumber] = ['0.5', new BigNumber(10)];
+            const oasisMarket: OasisMarket = {
+                id: 'ABCDEF',
+                base: 'DEF',
+                quote: 'ABC',
+                buyVol: 100,
+                sellVol: 200,
+                price: 1,
+                high: 1,
+                low: 0,
+            };
+            const observedTimestamp: number = Date.now();
+            const orderType: OrderType = OrderType.Bid;
+            const source: string = 'oasis';
+
+            const expected = new TokenOrder();
+            expected.source = 'oasis';
+            expected.observedTimestamp = observedTimestamp;
+            expected.orderType = OrderType.Bid;
+            expected.price = new BigNumber(0.5);
+            expected.baseAssetSymbol = 'DEF';
+            expected.baseAssetAddress = null;
+            expected.baseVolume = new BigNumber(10);
+            expected.quoteAssetSymbol = 'ABC';
+            expected.quoteAssetAddress = null;
+            expected.quoteVolume = new BigNumber(5);
+
+            const actual = parseOasisOrder(oasisMarket, observedTimestamp, orderType, source, oasisOrder);
+            expect(actual).deep.equal(expected);
+        });
+    });
+});
diff --git a/packages/pipeline/test/parsers/ohlcv_external/crypto_compare_test.ts b/packages/pipeline/test/parsers/ohlcv_external/crypto_compare_test.ts
new file mode 100644
index 000000000..118cafc5e
--- /dev/null
+++ b/packages/pipeline/test/parsers/ohlcv_external/crypto_compare_test.ts
@@ -0,0 +1,62 @@
+import * as chai from 'chai';
+import 'mocha';
+import * as R from 'ramda';
+
+import { CryptoCompareOHLCVRecord } from '../../../src/data_sources/ohlcv_external/crypto_compare';
+import { OHLCVExternal } from '../../../src/entities';
+import { OHLCVMetadata, parseRecords } from '../../../src/parsers/ohlcv_external/crypto_compare';
+import { chaiSetup } from '../../utils/chai_setup';
+
+chaiSetup.configure();
+const expect = chai.expect;
+
+// tslint:disable:custom-no-magic-numbers
+describe('ohlcv_external parser (Crypto Compare)', () => {
+    describe('parseRecords', () => {
+        const record: CryptoCompareOHLCVRecord = {
+            time: 200,
+            close: 100,
+            high: 101,
+            low: 99,
+            open: 98,
+            volumefrom: 1234,
+            volumeto: 4321,
+        };
+
+        const metadata: OHLCVMetadata = {
+            fromSymbol: 'ETH',
+            toSymbol: 'ZRX',
+            exchange: 'CCCAGG',
+            source: 'CryptoCompare',
+            observedTimestamp: new Date().getTime(),
+            interval: 100000,
+        };
+
+        const entity = new OHLCVExternal();
+        entity.exchange = metadata.exchange;
+        entity.fromSymbol = metadata.fromSymbol;
+        entity.toSymbol = metadata.toSymbol;
+        entity.startTime = 100000;
+        entity.endTime = 200000;
+        entity.open = record.open;
+        entity.close = record.close;
+        entity.low = record.low;
+        entity.high = record.high;
+        entity.volumeFrom = record.volumefrom;
+        entity.volumeTo = record.volumeto;
+        entity.source = metadata.source;
+        entity.observedTimestamp = metadata.observedTimestamp;
+
+        it('converts Crypto Compare OHLCV records to OHLCVExternal entity', () => {
+            const input = [record, R.merge(record, { time: 300 }), R.merge(record, { time: 400 })];
+            const expected = [
+                entity,
+                R.merge(entity, { startTime: 200000, endTime: 300000 }),
+                R.merge(entity, { startTime: 300000, endTime: 400000 }),
+            ];
+
+            const actual = parseRecords(input, metadata);
+            expect(actual).deep.equal(expected);
+        });
+    });
+});
diff --git a/packages/pipeline/test/parsers/paradex_orders/index_test.ts b/packages/pipeline/test/parsers/paradex_orders/index_test.ts
new file mode 100644
index 000000000..c5dd8751b
--- /dev/null
+++ b/packages/pipeline/test/parsers/paradex_orders/index_test.ts
@@ -0,0 +1,54 @@
+import { BigNumber } from '@0x/utils';
+import * as chai from 'chai';
+import 'mocha';
+
+import { ParadexMarket, ParadexOrder } from '../../../src/data_sources/paradex';
+import { TokenOrderbookSnapshot as TokenOrder } from '../../../src/entities';
+import { parseParadexOrder } from '../../../src/parsers/paradex_orders';
+import { OrderType } from '../../../src/types';
+import { chaiSetup } from '../../utils/chai_setup';
+
+chaiSetup.configure();
+const expect = chai.expect;
+
+// tslint:disable:custom-no-magic-numbers
+describe('paradex_orders', () => {
+    describe('parseParadexOrder', () => {
+        it('converts ParadexOrder to TokenOrder entity', () => {
+            const paradexOrder: ParadexOrder = {
+                amount: '412',
+                price: '0.1245',
+            };
+            const paradexMarket: ParadexMarket = {
+                id: '2',
+                symbol: 'ABC/DEF',
+                baseToken: 'DEF',
+                quoteToken: 'ABC',
+                minOrderSize: '0.1',
+                maxOrderSize: '1000',
+                priceMaxDecimals: 5,
+                amountMaxDecimals: 5,
+                baseTokenAddress: '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81',
+                quoteTokenAddress: '0x0000000000000000000000000000000000000000',
+            };
+            const observedTimestamp: number = Date.now();
+            const orderType: OrderType = OrderType.Bid;
+            const source: string = 'paradex';
+
+            const expected = new TokenOrder();
+            expected.source = 'paradex';
+            expected.observedTimestamp = observedTimestamp;
+            expected.orderType = OrderType.Bid;
+            expected.price = new BigNumber(0.1245);
+            expected.baseAssetSymbol = 'DEF';
+            expected.baseAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81';
+            expected.baseVolume = new BigNumber(412);
+            expected.quoteAssetSymbol = 'ABC';
+            expected.quoteAssetAddress = '0x0000000000000000000000000000000000000000';
+            expected.quoteVolume = new BigNumber(412 * 0.1245);
+
+            const actual = parseParadexOrder(paradexMarket, observedTimestamp, orderType, source, paradexOrder);
+            expect(actual).deep.equal(expected);
+        });
+    });
+});
diff --git a/packages/pipeline/test/parsers/sra_orders/index_test.ts b/packages/pipeline/test/parsers/sra_orders/index_test.ts
new file mode 100644
index 000000000..838171a72
--- /dev/null
+++ b/packages/pipeline/test/parsers/sra_orders/index_test.ts
@@ -0,0 +1,69 @@
+import { APIOrder } from '@0x/types';
+import { BigNumber } from '@0x/utils';
+import * as chai from 'chai';
+import 'mocha';
+
+import { SraOrder } from '../../../src/entities';
+import { _convertToEntity } from '../../../src/parsers/sra_orders';
+import { AssetType } from '../../../src/types';
+import { chaiSetup } from '../../utils/chai_setup';
+
+chaiSetup.configure();
+const expect = chai.expect;
+
+// tslint:disable:custom-no-magic-numbers
+describe('sra_orders', () => {
+    describe('_convertToEntity', () => {
+        it('converts ApiOrder to SraOrder entity', () => {
+            const input: APIOrder = {
+                order: {
+                    makerAddress: '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81',
+                    takerAddress: '0x0000000000000000000000000000000000000000',
+                    feeRecipientAddress: '0xa258b39954cef5cb142fd567a46cddb31a670124',
+                    senderAddress: '0x0000000000000000000000000000000000000000',
+                    makerAssetAmount: new BigNumber('1619310371000000000'),
+                    takerAssetAmount: new BigNumber('8178335207070707070707'),
+                    makerFee: new BigNumber('0'),
+                    takerFee: new BigNumber('0'),
+                    exchangeAddress: '0x4f833a24e1f95d70f028921e27040ca56e09ab0b',
+                    expirationTimeSeconds: new BigNumber('1538529488'),
+                    signature:
+                        '0x1b5a5d672b0d647b5797387ccbb89d822d5d2e873346b014f4ff816ff0783f2a7a0d2824d2d7042ec8ea375bc7f870963e1cb8248f1db03ddf125e27b5963aa11f03',
+                    salt: new BigNumber('1537924688891'),
+                    makerAssetData: '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2',
+                    takerAssetData: '0xf47261b000000000000000000000000042d6622dece394b54999fbd73d108123806f6a18',
+                },
+                metaData: { isThisArbitraryData: true, powerLevel: 9001 },
+            };
+            const expected = new SraOrder();
+            expected.exchangeAddress = '0x4f833a24e1f95d70f028921e27040ca56e09ab0b';
+            expected.orderHashHex = '0x1bdbeb0d088a33da28b9ee6d94e8771452f90f4a69107da2fa75195d61b9a1c9';
+            expected.makerAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81';
+            expected.takerAddress = '0x0000000000000000000000000000000000000000';
+            expected.feeRecipientAddress = '0xa258b39954cef5cb142fd567a46cddb31a670124';
+            expected.senderAddress = '0x0000000000000000000000000000000000000000';
+            expected.makerAssetAmount = new BigNumber('1619310371000000000');
+            expected.takerAssetAmount = new BigNumber('8178335207070707070707');
+            expected.makerFee = new BigNumber('0');
+            expected.takerFee = new BigNumber('0');
+            expected.expirationTimeSeconds = new BigNumber('1538529488');
+            expected.salt = new BigNumber('1537924688891');
+            expected.signature =
+                '0x1b5a5d672b0d647b5797387ccbb89d822d5d2e873346b014f4ff816ff0783f2a7a0d2824d2d7042ec8ea375bc7f870963e1cb8248f1db03ddf125e27b5963aa11f03';
+            expected.rawMakerAssetData = '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2';
+            expected.makerAssetType = AssetType.ERC20;
+            expected.makerAssetProxyId = '0xf47261b0';
+            expected.makerTokenAddress = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2';
+            expected.makerTokenId = null;
+            expected.rawTakerAssetData = '0xf47261b000000000000000000000000042d6622dece394b54999fbd73d108123806f6a18';
+            expected.takerAssetType = AssetType.ERC20;
+            expected.takerAssetProxyId = '0xf47261b0';
+            expected.takerTokenAddress = '0x42d6622dece394b54999fbd73d108123806f6a18';
+            expected.takerTokenId = null;
+            expected.metadataJson = '{"isThisArbitraryData":true,"powerLevel":9001}';
+
+            const actual = _convertToEntity(input);
+            expect(actual).deep.equal(expected);
+        });
+    });
+});
diff --git a/packages/pipeline/test/parsers/utils/index_test.ts b/packages/pipeline/test/parsers/utils/index_test.ts
new file mode 100644
index 000000000..5a0d0f182
--- /dev/null
+++ b/packages/pipeline/test/parsers/utils/index_test.ts
@@ -0,0 +1,30 @@
+import { BigNumber } from '@0x/utils';
+import * as chai from 'chai';
+import 'mocha';
+
+import { aggregateOrders, GenericRawOrder } from '../../../src/parsers/utils';
+import { chaiSetup } from '../../utils/chai_setup';
+
+chaiSetup.configure();
+const expect = chai.expect;
+
+// tslint:disable:custom-no-magic-numbers
+describe('aggregateOrders', () => {
+    it('aggregates order by price point', () => {
+        const input = [
+            { price: '1', amount: '20', orderHash: 'testtest', total: '20' },
+            { price: '1', amount: '30', orderHash: 'testone', total: '30' },
+            { price: '2', amount: '100', orderHash: 'testtwo', total: '200' },
+        ];
+        const expected = [['1', new BigNumber(50)], ['2', new BigNumber(100)]];
+        const actual = aggregateOrders(input);
+        expect(actual).deep.equal(expected);
+    });
+
+    it('handles empty orders gracefully', () => {
+        const input: GenericRawOrder[] = [];
+        const expected: Array<[string, BigNumber]> = [];
+        const actual = aggregateOrders(input);
+        expect(actual).deep.equal(expected);
+    });
+});