From 7b1471ffe80661a209ef3b4b9d4b67116708c210 Mon Sep 17 00:00:00 2001 From: xianny Date: Thu, 6 Dec 2018 14:28:22 -0800 Subject: query CC with larger batch size --- .../pipeline/src/data_sources/ohlcv_external/crypto_compare.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'packages/pipeline/src/data_sources') diff --git a/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts b/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts index 998ef6bf3..16f205629 100644 --- a/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts +++ b/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts @@ -33,18 +33,18 @@ export interface CryptoCompareOHLCVParams { toTs?: number; } -const ONE_WEEK = 7 * 24 * 60 * 60 * 1000; // tslint:disable-line:custom-no-magic-numbers const ONE_HOUR = 60 * 60 * 1000; // tslint:disable-line:custom-no-magic-numbers const ONE_MINUTE = 60 * 1000; // tslint:disable-line:custom-no-magic-numbers const ONE_SECOND = 1000; const ONE_HOUR_AGO = new Date().getTime() - ONE_HOUR; const HTTP_OK_STATUS = 200; const CRYPTO_COMPARE_VALID_EMPTY_RESPONSE_TYPE = 96; +const MAX_LIMIT = 2000; export class CryptoCompareOHLCVSource { - public readonly interval = ONE_WEEK; // the hourly API returns data for one week at a time - public readonly default_exchange = 'CCCAGG'; public readonly intervalBetweenRecords = ONE_HOUR; + public readonly default_exchange = 'CCCAGG'; + public readonly interval = ONE_HOUR * MAX_LIMIT; // the hourly API returns data for one interval at a time private readonly _url: string = 'https://min-api.cryptocompare.com/data/histohour?'; // rate-limit for all API calls through this class instance @@ -65,6 +65,7 @@ export class CryptoCompareOHLCVSource { e: this.default_exchange, fsym: pair.fromSymbol, tsym: pair.toSymbol, + limit: MAX_LIMIT, toTs: Math.floor((pair.latestSavedTime + this.interval) / ONE_SECOND), // CryptoCompare uses timestamp in seconds. not ms }; const url = this._url + stringify(params); -- cgit v1.2.3