Refactor event scraping and add support for scraping ERC20 approval events (#1401)

* Refactor event scraping and add support for scraping ERC20 approval events

* Add tests for data_sources/contract-wrappers/utils

5 files changed, 193 insertions, 2 deletions

diff --git a/packages/pipeline/test/data_sources/contract-wrappers/utils_test.ts b/packages/pipeline/test/data_sources/contract-wrappers/utils_test.ts
new file mode 100644
index 000000000..06f1a5e86
--- /dev/null
+++ b/packages/pipeline/test/data_sources/contract-wrappers/utils_test.ts
@@ -0,0 +1,109 @@
+// tslint:disable:custom-no-magic-numbers
+import * as chai from 'chai';
+import { LogWithDecodedArgs } from 'ethereum-types';
+import 'mocha';
+
+import { _getEventsWithRetriesAsync } from '../../../src/data_sources/contract-wrappers/utils';
+import { chaiSetup } from '../../utils/chai_setup';
+
+chaiSetup.configure();
+const expect = chai.expect;
+
+const retryableMessage = 'network timeout: (simulated network timeout error)';
+const retryableError = new Error(retryableMessage);
+
+describe('data_sources/contract-wrappers/utils', () => {
+    describe('_getEventsWithRetriesAsync', () => {
+        it('sends a single request if it was successful', async () => {
+            // Pre-declare values for the fromBlock and toBlock arguments.
+            const expectedFromBlock = 100;
+            const expectedToBlock = 200;
+            const expectedLogs: Array<LogWithDecodedArgs<any>> = [
+                {
+                    logIndex: 123,
+                    transactionIndex: 456,
+                    transactionHash: '0x6dd106d002873746072fc5e496dd0fb2541b68c77bcf9184ae19a42fd33657fe',
+                    blockHash: '0x6dd106d002873746072fc5e496dd0fb2541b68c77bcf9184ae19a42fd33657ff',
+                    blockNumber: 789,
+                    address: '0x6dd106d002873746072fc5e496dd0fb2541b68c77bcf9184ae19a42fd3365800',
+                    data: 'fake raw data',
+                    topics: [],
+                    event: 'TEST_EVENT',
+                    args: [1, 2, 3],
+                },
+            ];
+
+            // mockGetEventsAsync checks its arguments, increments `callCount`
+            // and returns `expectedLogs`.
+            let callCount = 0;
+            const mockGetEventsAsync = async (
+                fromBlock: number,
+                toBlock: number,
+            ): Promise<Array<LogWithDecodedArgs<any>>> => {
+                expect(fromBlock).equals(expectedFromBlock);
+                expect(toBlock).equals(expectedToBlock);
+                callCount += 1;
+                return expectedLogs;
+            };
+
+            // Make sure that we get what we expected and that the mock function
+            // was called exactly once.
+            const gotLogs = await _getEventsWithRetriesAsync(mockGetEventsAsync, 3, expectedFromBlock, expectedToBlock);
+            expect(gotLogs).deep.equals(expectedLogs);
+            expect(callCount).equals(
+                1,
+                'getEventsAsync function was called more than once even though it was successful',
+            );
+        });
+        it('retries and eventually succeeds', async () => {
+            const numRetries = 5;
+            let callCount = 0;
+            // mockGetEventsAsync throws unless callCount == numRetries + 1.
+            const mockGetEventsAsync = async (
+                _fromBlock: number,
+                _toBlock: number,
+            ): Promise<Array<LogWithDecodedArgs<any>>> => {
+                callCount += 1;
+                if (callCount === numRetries + 1) {
+                    return [];
+                }
+                throw retryableError;
+            };
+            await _getEventsWithRetriesAsync(mockGetEventsAsync, numRetries, 100, 300);
+            expect(callCount).equals(numRetries + 1, 'getEventsAsync function was called the wrong number of times');
+        });
+        it('throws for non-retryable errors', async () => {
+            const numRetries = 5;
+            const expectedMessage = 'Non-retryable error';
+            // mockGetEventsAsync always throws a non-retryable error.
+            const mockGetEventsAsync = async (
+                _fromBlock: number,
+                _toBlock: number,
+            ): Promise<Array<LogWithDecodedArgs<any>>> => {
+                throw new Error(expectedMessage);
+            };
+            // Note(albrow): This does actually return a promise (or at least a
+            // "promise-like object" and is a false positive in TSLint.
+            // tslint:disable-next-line:await-promise
+            await expect(_getEventsWithRetriesAsync(mockGetEventsAsync, numRetries, 100, 300)).to.be.rejectedWith(
+                expectedMessage,
+            );
+        });
+        it('throws after too many retries', async () => {
+            const numRetries = 5;
+            // mockGetEventsAsync always throws a retryable error.
+            const mockGetEventsAsync = async (
+                _fromBlock: number,
+                _toBlock: number,
+            ): Promise<Array<LogWithDecodedArgs<any>>> => {
+                throw retryableError;
+            };
+            // Note(albrow): This does actually return a promise (or at least a
+            // "promise-like object" and is a false positive in TSLint.
+            // tslint:disable-next-line:await-promise
+            await expect(_getEventsWithRetriesAsync(mockGetEventsAsync, numRetries, 100, 300)).to.be.rejectedWith(
+                retryableMessage,
+            );
+        });
+    });
+});
diff --git a/packages/pipeline/test/entities/erc20_approval_events_test.ts b/packages/pipeline/test/entities/erc20_approval_events_test.ts
new file mode 100644
index 000000000..1ecf41ee5
--- /dev/null
+++ b/packages/pipeline/test/entities/erc20_approval_events_test.ts
@@ -0,0 +1,29 @@
+import { BigNumber } from '@0x/utils';
+import 'mocha';
+import 'reflect-metadata';
+
+import { ERC20ApprovalEvent } from '../../src/entities';
+import { createDbConnectionOnceAsync } from '../db_setup';
+import { chaiSetup } from '../utils/chai_setup';
+
+import { testSaveAndFindEntityAsync } from './util';
+
+chaiSetup.configure();
+
+// tslint:disable:custom-no-magic-numbers
+describe('ERC20ApprovalEvent entity', () => {
+    it('save/find', async () => {
+        const connection = await createDbConnectionOnceAsync();
+        const event = new ERC20ApprovalEvent();
+        event.tokenAddress = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2';
+        event.blockNumber = 6281577;
+        event.rawData = '0x000000000000000000000000000000000000000000000002b9cba5ee21ad3df9';
+        event.logIndex = 43;
+        event.transactionHash = '0xcb46b19c786376a0a0140d51e3e606a4c4f926d8ca5434e96d2f69d04d8d9c7f';
+        event.ownerAddress = '0x0b65c5f6f3a05d6be5588a72b603360773b3fe04';
+        event.spenderAddress = '0x448a5065aebb8e423f0896e6c5d525c040f59af3';
+        event.amount = new BigNumber('50281464906893835769');
+        const blocksRepository = connection.getRepository(ERC20ApprovalEvent);
+        await testSaveAndFindEntityAsync(blocksRepository, event);
+    });
+});
diff --git a/packages/pipeline/test/parsers/bloxy/index_test.ts b/packages/pipeline/test/parsers/bloxy/index_test.ts
index 2b8d68f98..6aabb091d 100644
--- a/packages/pipeline/test/parsers/bloxy/index_test.ts
+++ b/packages/pipeline/test/parsers/bloxy/index_test.ts
@@ -7,7 +7,6 @@ import * as R from 'ramda';
 import { BLOXY_DEX_TRADES_URL, BloxyTrade } from '../../../src/data_sources/bloxy';
 import { DexTrade } from '../../../src/entities';
 import { _parseBloxyTrade } from '../../../src/parsers/bloxy';
-import { _convertToExchangeFillEvent } from '../../../src/parsers/events';
 import { chaiSetup } from '../../utils/chai_setup';
 
 chaiSetup.configure();
diff --git a/packages/pipeline/test/parsers/events/erc20_events_test.ts b/packages/pipeline/test/parsers/events/erc20_events_test.ts
new file mode 100644
index 000000000..962c50f98
--- /dev/null
+++ b/packages/pipeline/test/parsers/events/erc20_events_test.ts
@@ -0,0 +1,54 @@
+import { ERC20TokenApprovalEventArgs } from '@0x/contract-wrappers';
+import { BigNumber } from '@0x/utils';
+import * as chai from 'chai';
+import { LogWithDecodedArgs } from 'ethereum-types';
+import 'mocha';
+
+import { ERC20ApprovalEvent } from '../../../src/entities';
+import { _convertToERC20ApprovalEvent } from '../../../src/parsers/events/erc20_events';
+import { _convertToExchangeFillEvent } from '../../../src/parsers/events/exchange_events';
+import { chaiSetup } from '../../utils/chai_setup';
+
+chaiSetup.configure();
+const expect = chai.expect;
+
+// tslint:disable:custom-no-magic-numbers
+describe('erc20_events', () => {
+    describe('_convertToERC20ApprovalEvent', () => {
+        it('converts LogWithDecodedArgs to ERC20ApprovalEvent entity', () => {
+            const input: LogWithDecodedArgs<ERC20TokenApprovalEventArgs> = {
+                address: '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2',
+                blockHash: '0xd2d7aafaa7102aec0bca8ef026d5a85133e87892334c46ee1e92e42912991c9b',
+                blockNumber: 6281577,
+                data: '0x000000000000000000000000000000000000000000000002b9cba5ee21ad3df9',
+                logIndex: 43,
+                topics: [
+                    '0x8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925',
+                    '0x0000000000000000000000000b65c5f6f3a05d6be5588a72b603360773b3fe04',
+                    '0x000000000000000000000000448a5065aebb8e423f0896e6c5d525c040f59af3',
+                ],
+                transactionHash: '0xcb46b19c786376a0a0140d51e3e606a4c4f926d8ca5434e96d2f69d04d8d9c7f',
+                transactionIndex: 103,
+                event: 'Approval',
+                args: {
+                    _owner: '0x0b65c5f6f3a05d6be5588a72b603360773b3fe04',
+                    _spender: '0x448a5065aebb8e423f0896e6c5d525c040f59af3',
+                    _value: new BigNumber('50281464906893835769'),
+                },
+            };
+
+            const expected = new ERC20ApprovalEvent();
+            expected.tokenAddress = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2';
+            expected.blockNumber = 6281577;
+            expected.rawData = '0x000000000000000000000000000000000000000000000002b9cba5ee21ad3df9';
+            expected.logIndex = 43;
+            expected.transactionHash = '0xcb46b19c786376a0a0140d51e3e606a4c4f926d8ca5434e96d2f69d04d8d9c7f';
+            expected.ownerAddress = '0x0b65c5f6f3a05d6be5588a72b603360773b3fe04';
+            expected.spenderAddress = '0x448a5065aebb8e423f0896e6c5d525c040f59af3';
+            expected.amount = new BigNumber('50281464906893835769');
+
+            const actual = _convertToERC20ApprovalEvent(input);
+            expect(actual).deep.equal(expected);
+        });
+    });
+});
diff --git a/packages/pipeline/test/parsers/events/index_test.ts b/packages/pipeline/test/parsers/events/exchange_events_test.ts
index 7e439ce39..5d4b185a5 100644
--- a/packages/pipeline/test/parsers/events/index_test.ts
+++ b/packages/pipeline/test/parsers/events/exchange_events_test.ts
@@ -5,7 +5,7 @@ import { LogWithDecodedArgs } from 'ethereum-types';
 import 'mocha';
 
 import { ExchangeFillEvent } from '../../../src/entities';
-import { _convertToExchangeFillEvent } from '../../../src/parsers/events';
+import { _convertToExchangeFillEvent } from '../../../src/parsers/events/exchange_events';
 import { chaiSetup } from '../../utils/chai_setup';
 
 chaiSetup.configure();