aboutsummaryrefslogtreecommitdiffstats
path: root/packages/pipeline
diff options
context:
space:
mode:
Diffstat (limited to 'packages/pipeline')
-rw-r--r--packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts5
-rw-r--r--packages/pipeline/src/scripts/pull_ohlcv_cryptocompare.ts4
2 files changed, 5 insertions, 4 deletions
diff --git a/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts b/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts
index e83e3b67d..998ef6bf3 100644
--- a/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts
+++ b/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts
@@ -49,13 +49,14 @@ export class CryptoCompareOHLCVSource {
// rate-limit for all API calls through this class instance
private readonly _limiter: Bottleneck;
- constructor(maxReqsPerSecond: number = 40) {
+ constructor(maxReqsPerSecond: number) {
this._limiter = new Bottleneck({
- minTime: Math.ceil(ONE_SECOND / maxReqsPerSecond),
+ minTime: ONE_SECOND / maxReqsPerSecond,
reservoir: 2000,
reservoirRefreshAmount: 2000,
reservoirRefreshInterval: ONE_MINUTE,
});
+ console.log('mintime', Math.ceil(ONE_SECOND / maxReqsPerSecond)); // tslint:disable-line:no-console
}
// gets OHLCV records starting from pair.latest
diff --git a/packages/pipeline/src/scripts/pull_ohlcv_cryptocompare.ts b/packages/pipeline/src/scripts/pull_ohlcv_cryptocompare.ts
index 7377a64d8..a29a13bfc 100644
--- a/packages/pipeline/src/scripts/pull_ohlcv_cryptocompare.ts
+++ b/packages/pipeline/src/scripts/pull_ohlcv_cryptocompare.ts
@@ -11,7 +11,7 @@ import { fetchOHLCVTradingPairsAsync, TradingPair } from '../utils/get_ohlcv_tra
const SOURCE_NAME = 'CryptoCompare';
const TWO_HOURS_AGO = new Date().getTime() - 2 * 60 * 60 * 1000; // tslint:disable-line:custom-no-magic-numbers
-const MAX_CONCURRENT_REQUESTS = parseInt(process.env.CRYPTOCOMPARE_MAX_CONCURRENT_REQUESTS || '14', 10); // tslint:disable-line:custom-no-magic-numbers
+const MAX_REQS_PER_SECOND = parseInt(process.env.CRYPTOCOMPARE_MAX_REQS_PER_SECOND || '15', 10); // tslint:disable-line:custom-no-magic-numbers
const EARLIEST_BACKFILL_DATE = process.env.OHLCV_EARLIEST_BACKFILL_DATE || '2014-06-01';
const EARLIEST_BACKFILL_TIME = new Date(EARLIEST_BACKFILL_DATE).getTime();
@@ -20,7 +20,7 @@ let connection: Connection;
(async () => {
connection = await createConnection(ormConfig as ConnectionOptions);
const repository = connection.getRepository(OHLCVExternal);
- const source = new CryptoCompareOHLCVSource(MAX_CONCURRENT_REQUESTS);
+ const source = new CryptoCompareOHLCVSource(MAX_REQS_PER_SECOND);
const jobTime = new Date().getTime();
const tradingPairs = await fetchOHLCVTradingPairsAsync(connection, SOURCE_NAME, EARLIEST_BACKFILL_TIME);