aboutsummaryrefslogtreecommitdiffstats
path: root/packages/pipeline
diff options
context:
space:
mode:
authorxianny <xianny@gmail.com>2018-12-07 06:28:22 +0800
committerxianny <xianny@gmail.com>2018-12-08 01:43:55 +0800
commit7b1471ffe80661a209ef3b4b9d4b67116708c210 (patch)
tree8e0a8aca0cdc974e7e6abd555ee25c90fc36e9eb /packages/pipeline
parent8b3b4d983f0f942fc4e42365d4cfcaf793d1d283 (diff)
downloaddexon-sol-tools-7b1471ffe80661a209ef3b4b9d4b67116708c210.tar
dexon-sol-tools-7b1471ffe80661a209ef3b4b9d4b67116708c210.tar.gz
dexon-sol-tools-7b1471ffe80661a209ef3b4b9d4b67116708c210.tar.bz2
dexon-sol-tools-7b1471ffe80661a209ef3b4b9d4b67116708c210.tar.lz
dexon-sol-tools-7b1471ffe80661a209ef3b4b9d4b67116708c210.tar.xz
dexon-sol-tools-7b1471ffe80661a209ef3b4b9d4b67116708c210.tar.zst
dexon-sol-tools-7b1471ffe80661a209ef3b4b9d4b67116708c210.zip
query CC with larger batch size
Diffstat (limited to 'packages/pipeline')
-rw-r--r--packages/pipeline/package.json1
-rw-r--r--packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts7
-rw-r--r--packages/pipeline/test/data_sources/ohlcv_external/crypto_compare_test.ts4
3 files changed, 7 insertions, 5 deletions
diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json
index 4fde906b8..5c95344b8 100644
--- a/packages/pipeline/package.json
+++ b/packages/pipeline/package.json
@@ -57,6 +57,7 @@
"ethereum-types": "^1.0.6",
"pg": "^7.5.0",
"prettier": "^1.15.3",
+ "pw-app-sdk": "^0.3.4",
"ramda": "^0.25.0",
"reflect-metadata": "^0.1.12",
"sqlite3": "^4.0.2",
diff --git a/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts b/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts
index 998ef6bf3..16f205629 100644
--- a/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts
+++ b/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts
@@ -33,18 +33,18 @@ export interface CryptoCompareOHLCVParams {
toTs?: number;
}
-const ONE_WEEK = 7 * 24 * 60 * 60 * 1000; // tslint:disable-line:custom-no-magic-numbers
const ONE_HOUR = 60 * 60 * 1000; // tslint:disable-line:custom-no-magic-numbers
const ONE_MINUTE = 60 * 1000; // tslint:disable-line:custom-no-magic-numbers
const ONE_SECOND = 1000;
const ONE_HOUR_AGO = new Date().getTime() - ONE_HOUR;
const HTTP_OK_STATUS = 200;
const CRYPTO_COMPARE_VALID_EMPTY_RESPONSE_TYPE = 96;
+const MAX_LIMIT = 2000;
export class CryptoCompareOHLCVSource {
- public readonly interval = ONE_WEEK; // the hourly API returns data for one week at a time
- public readonly default_exchange = 'CCCAGG';
public readonly intervalBetweenRecords = ONE_HOUR;
+ public readonly default_exchange = 'CCCAGG';
+ public readonly interval = ONE_HOUR * MAX_LIMIT; // the hourly API returns data for one interval at a time
private readonly _url: string = 'https://min-api.cryptocompare.com/data/histohour?';
// rate-limit for all API calls through this class instance
@@ -65,6 +65,7 @@ export class CryptoCompareOHLCVSource {
e: this.default_exchange,
fsym: pair.fromSymbol,
tsym: pair.toSymbol,
+ limit: MAX_LIMIT,
toTs: Math.floor((pair.latestSavedTime + this.interval) / ONE_SECOND), // CryptoCompare uses timestamp in seconds. not ms
};
const url = this._url + stringify(params);
diff --git a/packages/pipeline/test/data_sources/ohlcv_external/crypto_compare_test.ts b/packages/pipeline/test/data_sources/ohlcv_external/crypto_compare_test.ts
index cb374bbb1..2efe3f5ec 100644
--- a/packages/pipeline/test/data_sources/ohlcv_external/crypto_compare_test.ts
+++ b/packages/pipeline/test/data_sources/ohlcv_external/crypto_compare_test.ts
@@ -13,7 +13,7 @@ const expect = chai.expect;
describe('ohlcv_external data source (Crypto Compare)', () => {
describe('generateBackfillIntervals', () => {
it('generates pairs with intervals to query', () => {
- const source = new CryptoCompareOHLCVSource();
+ const source = new CryptoCompareOHLCVSource(20);
const pair: TradingPair = {
fromSymbol: 'ETH',
toSymbol: 'ZRX',
@@ -31,7 +31,7 @@ describe('ohlcv_external data source (Crypto Compare)', () => {
});
it('returns single pair if no backfill is needed', () => {
- const source = new CryptoCompareOHLCVSource();
+ const source = new CryptoCompareOHLCVSource(20);
const pair: TradingPair = {
fromSymbol: 'ETH',
toSymbol: 'ZRX',