aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts5
-rw-r--r--packages/pipeline/src/scripts/pull_ohlcv_cryptocompare.ts4
-rw-r--r--yarn.lock5
3 files changed, 10 insertions, 4 deletions
diff --git a/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts b/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts
index e83e3b67d..998ef6bf3 100644
--- a/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts
+++ b/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts
@@ -49,13 +49,14 @@ export class CryptoCompareOHLCVSource {
// rate-limit for all API calls through this class instance
private readonly _limiter: Bottleneck;
- constructor(maxReqsPerSecond: number = 40) {
+ constructor(maxReqsPerSecond: number) {
this._limiter = new Bottleneck({
- minTime: Math.ceil(ONE_SECOND / maxReqsPerSecond),
+ minTime: ONE_SECOND / maxReqsPerSecond,
reservoir: 2000,
reservoirRefreshAmount: 2000,
reservoirRefreshInterval: ONE_MINUTE,
});
+ console.log('mintime', Math.ceil(ONE_SECOND / maxReqsPerSecond)); // tslint:disable-line:no-console
}
// gets OHLCV records starting from pair.latest
diff --git a/packages/pipeline/src/scripts/pull_ohlcv_cryptocompare.ts b/packages/pipeline/src/scripts/pull_ohlcv_cryptocompare.ts
index 7377a64d8..a29a13bfc 100644
--- a/packages/pipeline/src/scripts/pull_ohlcv_cryptocompare.ts
+++ b/packages/pipeline/src/scripts/pull_ohlcv_cryptocompare.ts
@@ -11,7 +11,7 @@ import { fetchOHLCVTradingPairsAsync, TradingPair } from '../utils/get_ohlcv_tra
const SOURCE_NAME = 'CryptoCompare';
const TWO_HOURS_AGO = new Date().getTime() - 2 * 60 * 60 * 1000; // tslint:disable-line:custom-no-magic-numbers
-const MAX_CONCURRENT_REQUESTS = parseInt(process.env.CRYPTOCOMPARE_MAX_CONCURRENT_REQUESTS || '14', 10); // tslint:disable-line:custom-no-magic-numbers
+const MAX_REQS_PER_SECOND = parseInt(process.env.CRYPTOCOMPARE_MAX_REQS_PER_SECOND || '15', 10); // tslint:disable-line:custom-no-magic-numbers
const EARLIEST_BACKFILL_DATE = process.env.OHLCV_EARLIEST_BACKFILL_DATE || '2014-06-01';
const EARLIEST_BACKFILL_TIME = new Date(EARLIEST_BACKFILL_DATE).getTime();
@@ -20,7 +20,7 @@ let connection: Connection;
(async () => {
connection = await createConnection(ormConfig as ConnectionOptions);
const repository = connection.getRepository(OHLCVExternal);
- const source = new CryptoCompareOHLCVSource(MAX_CONCURRENT_REQUESTS);
+ const source = new CryptoCompareOHLCVSource(MAX_REQS_PER_SECOND);
const jobTime = new Date().getTime();
const tradingPairs = await fetchOHLCVTradingPairsAsync(connection, SOURCE_NAME, EARLIEST_BACKFILL_TIME);
diff --git a/yarn.lock b/yarn.lock
index 89f234476..8e16b6aba 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -3522,6 +3522,11 @@ boom@5.x.x:
dependencies:
hoek "4.x.x"
+bottleneck@^2.13.2:
+ version "2.13.2"
+ resolved "https://registry.yarnpkg.com/bottleneck/-/bottleneck-2.13.2.tgz#f3f28f0ddf82cdd3e44072aee3104a42adcca352"
+ integrity sha512-DVS4Uv7xr4Ql0w9valPBaueLRnEtBepeoevDhWO0LBhyihICJ7RySyzPfyvPswanrXAAbWaF8Zx4QpxmIxHa/g==
+
bowser@^1.7.3, bowser@^1.9.3:
version "1.9.3"
resolved "https://registry.yarnpkg.com/bowser/-/bowser-1.9.3.tgz#6643ae4d783f31683f6d23156976b74183862162"