diff options
| author | xianny <xianny@gmail.com> | 2018-12-07 06:28:22 +0800 |
|---|---|---|
| committer | xianny <xianny@gmail.com> | 2018-12-08 01:43:55 +0800 |
| commit | 7b1471ffe80661a209ef3b4b9d4b67116708c210 (patch) | |
| tree | 8e0a8aca0cdc974e7e6abd555ee25c90fc36e9eb /packages/pipeline | |
| parent | 8b3b4d983f0f942fc4e42365d4cfcaf793d1d283 (diff) | |
| download | dexon-sol-tools-7b1471ffe80661a209ef3b4b9d4b67116708c210.tar dexon-sol-tools-7b1471ffe80661a209ef3b4b9d4b67116708c210.tar.gz dexon-sol-tools-7b1471ffe80661a209ef3b4b9d4b67116708c210.tar.bz2 dexon-sol-tools-7b1471ffe80661a209ef3b4b9d4b67116708c210.tar.lz dexon-sol-tools-7b1471ffe80661a209ef3b4b9d4b67116708c210.tar.xz dexon-sol-tools-7b1471ffe80661a209ef3b4b9d4b67116708c210.tar.zst dexon-sol-tools-7b1471ffe80661a209ef3b4b9d4b67116708c210.zip | |
query CC with larger batch size
Diffstat (limited to 'packages/pipeline')
3 files changed, 7 insertions, 5 deletions
diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json index 4fde906b8..5c95344b8 100644 --- a/packages/pipeline/package.json +++ b/packages/pipeline/package.json @@ -57,6 +57,7 @@ "ethereum-types": "^1.0.6", "pg": "^7.5.0", "prettier": "^1.15.3", + "pw-app-sdk": "^0.3.4", "ramda": "^0.25.0", "reflect-metadata": "^0.1.12", "sqlite3": "^4.0.2", diff --git a/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts b/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts index 998ef6bf3..16f205629 100644 --- a/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts +++ b/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts @@ -33,18 +33,18 @@ export interface CryptoCompareOHLCVParams { toTs?: number; } -const ONE_WEEK = 7 * 24 * 60 * 60 * 1000; // tslint:disable-line:custom-no-magic-numbers const ONE_HOUR = 60 * 60 * 1000; // tslint:disable-line:custom-no-magic-numbers const ONE_MINUTE = 60 * 1000; // tslint:disable-line:custom-no-magic-numbers const ONE_SECOND = 1000; const ONE_HOUR_AGO = new Date().getTime() - ONE_HOUR; const HTTP_OK_STATUS = 200; const CRYPTO_COMPARE_VALID_EMPTY_RESPONSE_TYPE = 96; +const MAX_LIMIT = 2000; export class CryptoCompareOHLCVSource { - public readonly interval = ONE_WEEK; // the hourly API returns data for one week at a time - public readonly default_exchange = 'CCCAGG'; public readonly intervalBetweenRecords = ONE_HOUR; + public readonly default_exchange = 'CCCAGG'; + public readonly interval = ONE_HOUR * MAX_LIMIT; // the hourly API returns data for one interval at a time private readonly _url: string = 'https://min-api.cryptocompare.com/data/histohour?'; // rate-limit for all API calls through this class instance @@ -65,6 +65,7 @@ export class CryptoCompareOHLCVSource { e: this.default_exchange, fsym: pair.fromSymbol, tsym: pair.toSymbol, + limit: MAX_LIMIT, toTs: Math.floor((pair.latestSavedTime + this.interval) / ONE_SECOND), // CryptoCompare uses timestamp in seconds. not ms }; const url = this._url + stringify(params); diff --git a/packages/pipeline/test/data_sources/ohlcv_external/crypto_compare_test.ts b/packages/pipeline/test/data_sources/ohlcv_external/crypto_compare_test.ts index cb374bbb1..2efe3f5ec 100644 --- a/packages/pipeline/test/data_sources/ohlcv_external/crypto_compare_test.ts +++ b/packages/pipeline/test/data_sources/ohlcv_external/crypto_compare_test.ts @@ -13,7 +13,7 @@ const expect = chai.expect; describe('ohlcv_external data source (Crypto Compare)', () => { describe('generateBackfillIntervals', () => { it('generates pairs with intervals to query', () => { - const source = new CryptoCompareOHLCVSource(); + const source = new CryptoCompareOHLCVSource(20); const pair: TradingPair = { fromSymbol: 'ETH', toSymbol: 'ZRX', @@ -31,7 +31,7 @@ describe('ohlcv_external data source (Crypto Compare)', () => { }); it('returns single pair if no backfill is needed', () => { - const source = new CryptoCompareOHLCVSource(); + const source = new CryptoCompareOHLCVSource(20); const pair: TradingPair = { fromSymbol: 'ETH', toSymbol: 'ZRX', |