1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
|
// tslint:disable:no-duplicate-imports
import { fetchAsync } from '@0x/utils';
import Bottleneck from 'bottleneck';
import { stringify } from 'querystring';
import * as R from 'ramda';
import { TradingPair } from '../../utils/get_ohlcv_trading_pairs';
export interface CryptoCompareOHLCVResponse {
Data: CryptoCompareOHLCVRecord[];
Response: string;
Message: string;
Type: number;
}
export interface CryptoCompareOHLCVRecord {
time: number; // in seconds, not milliseconds
close: number;
high: number;
low: number;
open: number;
volumefrom: number;
volumeto: number;
}
export interface CryptoCompareOHLCVParams {
fsym: string;
tsym: string;
e?: string;
aggregate?: string;
aggregatePredictableTimePeriods?: boolean;
limit?: number;
toTs?: number;
}
const ONE_WEEK = 7 * 24 * 60 * 60 * 1000; // tslint:disable-line:custom-no-magic-numbers
const ONE_HOUR = 60 * 60 * 1000; // tslint:disable-line:custom-no-magic-numbers
const ONE_MINUTE = 60 * 1000; // tslint:disable-line:custom-no-magic-numbers
const ONE_SECOND = 1000;
const ONE_HOUR_AGO = new Date().getTime() - ONE_HOUR;
const HTTP_OK_STATUS = 200;
const CRYPTO_COMPARE_VALID_EMPTY_RESPONSE_TYPE = 96;
export class CryptoCompareOHLCVSource {
public readonly interval = ONE_WEEK; // the hourly API returns data for one week at a time
public readonly default_exchange = 'CCCAGG';
public readonly intervalBetweenRecords = ONE_HOUR;
private readonly _url: string = 'https://min-api.cryptocompare.com/data/histohour?';
// rate-limit for all API calls through this class instance
private readonly _limiter: Bottleneck;
constructor(maxReqsPerSecond: number) {
this._limiter = new Bottleneck({
minTime: ONE_SECOND / maxReqsPerSecond,
reservoir: 2000,
reservoirRefreshAmount: 2000,
reservoirRefreshInterval: ONE_MINUTE,
});
console.log('mintime', Math.ceil(ONE_SECOND / maxReqsPerSecond)); // tslint:disable-line:no-console
}
// gets OHLCV records starting from pair.latest
public async getHourlyOHLCVAsync(pair: TradingPair): Promise<CryptoCompareOHLCVRecord[]> {
const params = {
e: this.default_exchange,
fsym: pair.fromSymbol,
tsym: pair.toSymbol,
toTs: Math.floor((pair.latestSavedTime + this.interval) / ONE_SECOND), // CryptoCompare uses timestamp in seconds. not ms
};
const url = this._url + stringify(params);
const response = await this._limiter.schedule(() => fetchAsync(url));
if (response.status !== HTTP_OK_STATUS) {
throw new Error(`HTTP error while scraping Crypto Compare: [${response}]`);
}
const json: CryptoCompareOHLCVResponse = await response.json();
if (
(json.Response === 'Error' || json.Data.length === 0) &&
json.Type !== CRYPTO_COMPARE_VALID_EMPTY_RESPONSE_TYPE
) {
throw new Error(JSON.stringify(json));
}
return json.Data.filter(rec => {
return (
// Crypto Compare takes ~30 mins to finalise records
rec.time * ONE_SECOND < ONE_HOUR_AGO && rec.time * ONE_SECOND > pair.latestSavedTime && hasData(rec)
);
});
}
public generateBackfillIntervals(pair: TradingPair): TradingPair[] {
const now = new Date().getTime();
const f = (p: TradingPair): false | [TradingPair, TradingPair] => {
if (p.latestSavedTime > now) {
return false;
} else {
return [p, R.merge(p, { latestSavedTime: p.latestSavedTime + this.interval })];
}
};
return R.unfold(f, pair);
}
}
function hasData(record: CryptoCompareOHLCVRecord): boolean {
return (
record.close !== 0 ||
record.open !== 0 ||
record.high !== 0 ||
record.low !== 0 ||
record.volumefrom !== 0 ||
record.volumeto !== 0
);
}
|
