aboutsummaryrefslogtreecommitdiffstats
path: root/packages/pipeline/src/data_sources
diff options
context:
space:
mode:
Diffstat (limited to 'packages/pipeline/src/data_sources')
-rw-r--r--packages/pipeline/src/data_sources/bloxy/index.ts143
-rw-r--r--packages/pipeline/src/data_sources/contract-wrappers/erc20_events.ts45
-rw-r--r--packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts59
-rw-r--r--packages/pipeline/src/data_sources/contract-wrappers/utils.ts67
-rw-r--r--packages/pipeline/src/data_sources/copper/index.ts126
-rw-r--r--packages/pipeline/src/data_sources/ddex/index.ts77
-rw-r--r--packages/pipeline/src/data_sources/idex/index.ts82
-rw-r--r--packages/pipeline/src/data_sources/oasis/index.ts103
-rw-r--r--packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts110
-rw-r--r--packages/pipeline/src/data_sources/paradex/index.ts92
-rw-r--r--packages/pipeline/src/data_sources/relayer-registry/index.ts33
-rw-r--r--packages/pipeline/src/data_sources/trusted_tokens/index.ts29
-rw-r--r--packages/pipeline/src/data_sources/web3/index.ts22
13 files changed, 0 insertions, 988 deletions
diff --git a/packages/pipeline/src/data_sources/bloxy/index.ts b/packages/pipeline/src/data_sources/bloxy/index.ts
deleted file mode 100644
index 22ab195b3..000000000
--- a/packages/pipeline/src/data_sources/bloxy/index.ts
+++ /dev/null
@@ -1,143 +0,0 @@
-import axios from 'axios';
-import * as R from 'ramda';
-
-import { logUtils } from '@0x/utils';
-
-// URL to use for getting dex trades from Bloxy.
-export const BLOXY_DEX_TRADES_URL = 'https://bloxy.info/api/dex/trades';
-// Number of trades to get at once. Must be less than or equal to MAX_OFFSET.
-const TRADES_PER_QUERY = 10000;
-// Maximum offset supported by the Bloxy API.
-const MAX_OFFSET = 100000;
-// Buffer to subtract from offset. This means we will request some trades twice
-// but we have less chance on missing out on any data.
-const OFFSET_BUFFER = 1000;
-// Maximum number of days supported by the Bloxy API.
-const MAX_DAYS = 30;
-// Buffer used for comparing the last seen timestamp to the last returned
-// timestamp. Increasing this reduces chances of data loss but also creates more
-// redundancy and can impact performance.
-// tslint:disable-next-line:custom-no-magic-numbers
-const LAST_SEEN_TIMESTAMP_BUFFER_MS = 1000 * 60 * 30; // 30 minutes
-
-// tslint:disable-next-line:custom-no-magic-numbers
-const millisecondsPerDay = 1000 * 60 * 60 * 24; // ms/d = ms/s * s/m * m/h * h/d
-
-export interface BloxyTrade {
- tx_hash: string;
- tx_time: string;
- tx_date: string;
- tx_sender: string;
- tradeIndex: string;
- smart_contract_id: number;
- smart_contract_address: string;
- contract_type: string;
- maker: string;
- taker: string;
- amountBuy: number;
- makerFee: number;
- buyCurrencyId: number;
- buySymbol: string;
- amountSell: number;
- takerFee: number;
- sellCurrencyId: number;
- sellSymbol: string;
- maker_annotation: string;
- taker_annotation: string;
- protocol: string;
- buyAddress: string | null;
- sellAddress: string | null;
-}
-
-interface BloxyError {
- error: string;
-}
-
-type BloxyResponse<T> = T | BloxyError;
-type BloxyTradeResponse = BloxyResponse<BloxyTrade[]>;
-
-function isError<T>(response: BloxyResponse<T>): response is BloxyError {
- return (response as BloxyError).error !== undefined;
-}
-
-export class BloxySource {
- private readonly _apiKey: string;
-
- constructor(apiKey: string) {
- this._apiKey = apiKey;
- }
-
- /**
- * Gets all latest trades between the lastSeenTimestamp (minus some buffer)
- * and the current time. Note that because the Bloxy API has some hard
- * limits it might not always be possible to get *all* the trades in the
- * desired time range.
- * @param lastSeenTimestamp The latest timestamp for trades that have
- * already been seen.
- */
- public async getDexTradesAsync(lastSeenTimestamp: number): Promise<BloxyTrade[]> {
- const allTrades = await this._scrapeAllDexTradesAsync(lastSeenTimestamp);
- logUtils.log(`Removing duplicates from ${allTrades.length} entries`);
- const uniqueTrades = R.uniqBy((trade: BloxyTrade) => `${trade.tradeIndex}-${trade.tx_hash}`, allTrades);
- logUtils.log(`Removed ${allTrades.length - uniqueTrades.length} duplicate entries`);
- return uniqueTrades;
- }
-
- // Potentially returns duplicate trades.
- private async _scrapeAllDexTradesAsync(lastSeenTimestamp: number): Promise<BloxyTrade[]> {
- let allTrades: BloxyTrade[] = [];
-
- // Clamp numberOfDays so that it is always between 1 and MAX_DAYS (inclusive)
- const numberOfDays = R.clamp(1, MAX_DAYS, getDaysSinceTimestamp(lastSeenTimestamp));
-
- // Keep getting trades until we hit one of the following conditions:
- //
- // 1. Offset hits MAX_OFFSET (we can't go back any further).
- // 2. There are no more trades in the response.
- // 3. We see a tx_time equal to or earlier than lastSeenTimestamp (plus
- // some buffer).
- //
- for (let offset = 0; offset <= MAX_OFFSET; offset += TRADES_PER_QUERY - OFFSET_BUFFER) {
- const trades = await this._getTradesWithOffsetAsync(numberOfDays, offset);
- if (trades.length === 0) {
- // There are no more trades left for the days we are querying.
- // This means we are done.
- return allTrades;
- }
- const sortedTrades = R.reverse(R.sortBy(trade => trade.tx_time, trades));
- allTrades = allTrades.concat(sortedTrades);
-
- // Check if lastReturnedTimestamp < lastSeenTimestamp
- const lastReturnedTimestamp = new Date(sortedTrades[0].tx_time).getTime();
- if (lastReturnedTimestamp < lastSeenTimestamp - LAST_SEEN_TIMESTAMP_BUFFER_MS) {
- // We are at the point where we have already seen trades for the
- // timestamp range that is being returned. We're done.
- return allTrades;
- }
- }
- return allTrades;
- }
-
- private async _getTradesWithOffsetAsync(numberOfDays: number, offset: number): Promise<BloxyTrade[]> {
- const resp = await axios.get<BloxyTradeResponse>(BLOXY_DEX_TRADES_URL, {
- params: {
- key: this._apiKey,
- days: numberOfDays,
- limit: TRADES_PER_QUERY,
- offset,
- },
- });
- if (isError(resp.data)) {
- throw new Error(`Error in Bloxy API response: ${resp.data.error}`);
- }
- return resp.data;
- }
-}
-
-// Computes the number of days between the given timestamp and the current
-// timestamp (rounded up).
-function getDaysSinceTimestamp(timestamp: number): number {
- const msSinceTimestamp = Date.now() - timestamp;
- const daysSinceTimestamp = msSinceTimestamp / millisecondsPerDay;
- return Math.ceil(daysSinceTimestamp);
-}
diff --git a/packages/pipeline/src/data_sources/contract-wrappers/erc20_events.ts b/packages/pipeline/src/data_sources/contract-wrappers/erc20_events.ts
deleted file mode 100644
index e0098122f..000000000
--- a/packages/pipeline/src/data_sources/contract-wrappers/erc20_events.ts
+++ /dev/null
@@ -1,45 +0,0 @@
-import {
- ContractWrappers,
- ERC20TokenApprovalEventArgs,
- ERC20TokenEvents,
- ERC20TokenWrapper,
-} from '@0x/contract-wrappers';
-import { Web3ProviderEngine } from '@0x/subproviders';
-import { LogWithDecodedArgs } from 'ethereum-types';
-
-import { GetEventsFunc, getEventsWithPaginationAsync } from './utils';
-
-export class ERC20EventsSource {
- private readonly _erc20Wrapper: ERC20TokenWrapper;
- private readonly _tokenAddress: string;
- constructor(provider: Web3ProviderEngine, networkId: number, tokenAddress: string) {
- const contractWrappers = new ContractWrappers(provider, { networkId });
- this._erc20Wrapper = contractWrappers.erc20Token;
- this._tokenAddress = tokenAddress;
- }
-
- public async getApprovalEventsAsync(
- startBlock: number,
- endBlock: number,
- ): Promise<Array<LogWithDecodedArgs<ERC20TokenApprovalEventArgs>>> {
- return getEventsWithPaginationAsync(
- this._getApprovalEventsForRangeAsync.bind(this) as GetEventsFunc<ERC20TokenApprovalEventArgs>,
- startBlock,
- endBlock,
- );
- }
-
- // Gets all approval events of for a specific sub-range. This getter
- // function will be called during each step of pagination.
- private async _getApprovalEventsForRangeAsync(
- fromBlock: number,
- toBlock: number,
- ): Promise<Array<LogWithDecodedArgs<ERC20TokenApprovalEventArgs>>> {
- return this._erc20Wrapper.getLogsAsync<ERC20TokenApprovalEventArgs>(
- this._tokenAddress,
- ERC20TokenEvents.Approval,
- { fromBlock, toBlock },
- {},
- );
- }
-}
diff --git a/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts b/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts
deleted file mode 100644
index 58691e2ab..000000000
--- a/packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts
+++ /dev/null
@@ -1,59 +0,0 @@
-import {
- ContractWrappers,
- ExchangeCancelEventArgs,
- ExchangeCancelUpToEventArgs,
- ExchangeEventArgs,
- ExchangeEvents,
- ExchangeFillEventArgs,
- ExchangeWrapper,
-} from '@0x/contract-wrappers';
-import { Web3ProviderEngine } from '@0x/subproviders';
-import { LogWithDecodedArgs } from 'ethereum-types';
-
-import { GetEventsFunc, getEventsWithPaginationAsync } from './utils';
-
-export class ExchangeEventsSource {
- private readonly _exchangeWrapper: ExchangeWrapper;
- constructor(provider: Web3ProviderEngine, networkId: number) {
- const contractWrappers = new ContractWrappers(provider, { networkId });
- this._exchangeWrapper = contractWrappers.exchange;
- }
-
- public async getFillEventsAsync(
- startBlock: number,
- endBlock: number,
- ): Promise<Array<LogWithDecodedArgs<ExchangeFillEventArgs>>> {
- const getFillEventsForRangeAsync = this._makeGetterFuncForEventType<ExchangeFillEventArgs>(ExchangeEvents.Fill);
- return getEventsWithPaginationAsync(getFillEventsForRangeAsync, startBlock, endBlock);
- }
-
- public async getCancelEventsAsync(
- startBlock: number,
- endBlock: number,
- ): Promise<Array<LogWithDecodedArgs<ExchangeCancelEventArgs>>> {
- const getCancelEventsForRangeAsync = this._makeGetterFuncForEventType<ExchangeCancelEventArgs>(
- ExchangeEvents.Cancel,
- );
- return getEventsWithPaginationAsync(getCancelEventsForRangeAsync, startBlock, endBlock);
- }
-
- public async getCancelUpToEventsAsync(
- startBlock: number,
- endBlock: number,
- ): Promise<Array<LogWithDecodedArgs<ExchangeCancelUpToEventArgs>>> {
- const getCancelUpToEventsForRangeAsync = this._makeGetterFuncForEventType<ExchangeCancelUpToEventArgs>(
- ExchangeEvents.CancelUpTo,
- );
- return getEventsWithPaginationAsync(getCancelUpToEventsForRangeAsync, startBlock, endBlock);
- }
-
- // Returns a getter function which gets all events of a specific type for a
- // specific sub-range. This getter function will be called during each step
- // of pagination.
- private _makeGetterFuncForEventType<ArgsType extends ExchangeEventArgs>(
- eventType: ExchangeEvents,
- ): GetEventsFunc<ArgsType> {
- return async (fromBlock: number, toBlock: number) =>
- this._exchangeWrapper.getLogsAsync<ArgsType>(eventType, { fromBlock, toBlock }, {});
- }
-}
diff --git a/packages/pipeline/src/data_sources/contract-wrappers/utils.ts b/packages/pipeline/src/data_sources/contract-wrappers/utils.ts
deleted file mode 100644
index 67660a37e..000000000
--- a/packages/pipeline/src/data_sources/contract-wrappers/utils.ts
+++ /dev/null
@@ -1,67 +0,0 @@
-import { DecodedLogArgs, LogWithDecodedArgs } from 'ethereum-types';
-
-const NUM_BLOCKS_PER_QUERY = 10000; // Number of blocks to query for events at a time.
-const NUM_RETRIES = 3; // Number of retries if a request fails or times out.
-
-export type GetEventsFunc<ArgsType extends DecodedLogArgs> = (
- fromBlock: number,
- toBlock: number,
-) => Promise<Array<LogWithDecodedArgs<ArgsType>>>;
-
-/**
- * Gets all events between the given startBlock and endBlock by querying for
- * NUM_BLOCKS_PER_QUERY at a time. Accepts a getter function in order to
- * maximize code re-use and allow for getting different types of events for
- * different contracts. If the getter function throws with a retryable error,
- * it will automatically be retried up to NUM_RETRIES times.
- * @param getEventsAsync A getter function which will be called for each step during pagination.
- * @param startBlock The start of the entire block range to get events for.
- * @param endBlock The end of the entire block range to get events for.
- */
-export async function getEventsWithPaginationAsync<ArgsType extends DecodedLogArgs>(
- getEventsAsync: GetEventsFunc<ArgsType>,
- startBlock: number,
- endBlock: number,
-): Promise<Array<LogWithDecodedArgs<ArgsType>>> {
- let events: Array<LogWithDecodedArgs<ArgsType>> = [];
- for (let fromBlock = startBlock; fromBlock <= endBlock; fromBlock += NUM_BLOCKS_PER_QUERY) {
- const toBlock = Math.min(fromBlock + NUM_BLOCKS_PER_QUERY - 1, endBlock);
- const eventsInRange = await _getEventsWithRetriesAsync(getEventsAsync, NUM_RETRIES, fromBlock, toBlock);
- events = events.concat(eventsInRange);
- }
- return events;
-}
-
-/**
- * Calls the getEventsAsync function and retries up to numRetries times if it
- * throws with an error that is considered retryable.
- * @param getEventsAsync a function that will be called on each iteration.
- * @param numRetries the maximum number times to retry getEventsAsync if it fails with a retryable error.
- * @param fromBlock the start of the sub-range of blocks we are getting events for.
- * @param toBlock the end of the sub-range of blocks we are getting events for.
- */
-export async function _getEventsWithRetriesAsync<ArgsType extends DecodedLogArgs>(
- getEventsAsync: GetEventsFunc<ArgsType>,
- numRetries: number,
- fromBlock: number,
- toBlock: number,
-): Promise<Array<LogWithDecodedArgs<ArgsType>>> {
- let eventsInRange: Array<LogWithDecodedArgs<ArgsType>> = [];
- for (let i = 0; i <= numRetries; i++) {
- try {
- eventsInRange = await getEventsAsync(fromBlock, toBlock);
- } catch (err) {
- if (isErrorRetryable(err) && i < numRetries) {
- continue;
- } else {
- throw err;
- }
- }
- break;
- }
- return eventsInRange;
-}
-
-function isErrorRetryable(err: Error): boolean {
- return err.message.includes('network timeout');
-}
diff --git a/packages/pipeline/src/data_sources/copper/index.ts b/packages/pipeline/src/data_sources/copper/index.ts
deleted file mode 100644
index 15df2fd7d..000000000
--- a/packages/pipeline/src/data_sources/copper/index.ts
+++ /dev/null
@@ -1,126 +0,0 @@
-import { fetchAsync } from '@0x/utils';
-import Bottleneck from 'bottleneck';
-
-import {
- CopperActivityTypeCategory,
- CopperActivityTypeResponse,
- CopperCustomFieldResponse,
- CopperSearchResponse,
-} from '../../parsers/copper';
-
-const HTTP_OK_STATUS = 200;
-const COPPER_URI = 'https://api.prosperworks.com/developer_api/v1';
-
-const DEFAULT_PAGINATION_PARAMS = {
- page_size: 200,
- sort_by: 'date_modified',
- sort_direction: 'desc',
-};
-
-export type CopperSearchParams = CopperLeadSearchParams | CopperActivitySearchParams | CopperOpportunitySearchParams;
-export interface CopperLeadSearchParams {
- page_number?: number;
-}
-
-export interface CopperActivitySearchParams {
- minimum_activity_date: number;
- page_number?: number;
-}
-
-export interface CopperOpportunitySearchParams {
- sort_by: string; // must override the default 'date_modified' for this endpoint
- page_number?: number;
-}
-export enum CopperEndpoint {
- Leads = '/leads/search',
- Opportunities = '/opportunities/search',
- Activities = '/activities/search',
-}
-const ONE_SECOND = 1000;
-
-function httpErrorCheck(response: Response): void {
- if (response.status !== HTTP_OK_STATUS) {
- throw new Error(`HTTP error while scraping Copper: [${JSON.stringify(response)}]`);
- }
-}
-export class CopperSource {
- private readonly _accessToken: string;
- private readonly _userEmail: string;
- private readonly _defaultHeaders: any;
- private readonly _limiter: Bottleneck;
-
- constructor(maxConcurrentRequests: number, accessToken: string, userEmail: string) {
- this._accessToken = accessToken;
- this._userEmail = userEmail;
- this._defaultHeaders = {
- 'Content-Type': 'application/json',
- 'X-PW-AccessToken': this._accessToken,
- 'X-PW-Application': 'developer_api',
- 'X-PW-UserEmail': this._userEmail,
- };
- this._limiter = new Bottleneck({
- minTime: ONE_SECOND / maxConcurrentRequests,
- reservoir: 30,
- reservoirRefreshAmount: 30,
- reservoirRefreshInterval: maxConcurrentRequests,
- });
- }
-
- public async fetchNumberOfPagesAsync(endpoint: CopperEndpoint, searchParams?: CopperSearchParams): Promise<number> {
- const resp = await this._limiter.schedule(() =>
- fetchAsync(COPPER_URI + endpoint, {
- method: 'POST',
- body: JSON.stringify({ ...DEFAULT_PAGINATION_PARAMS, ...searchParams }),
- headers: this._defaultHeaders,
- }),
- );
-
- httpErrorCheck(resp);
-
- // total number of records that match the request parameters
- if (resp.headers.has('X-Pw-Total')) {
- const totalRecords: number = parseInt(resp.headers.get('X-Pw-Total') as string, 10); // tslint:disable-line:custom-no-magic-numbers
- return Math.ceil(totalRecords / DEFAULT_PAGINATION_PARAMS.page_size);
- } else {
- return 1;
- }
- }
- public async fetchSearchResultsAsync<T extends CopperSearchResponse>(
- endpoint: CopperEndpoint,
- searchParams?: CopperSearchParams,
- ): Promise<T[]> {
- const request = { ...DEFAULT_PAGINATION_PARAMS, ...searchParams };
- const response = await this._limiter.schedule(() =>
- fetchAsync(COPPER_URI + endpoint, {
- method: 'POST',
- body: JSON.stringify(request),
- headers: this._defaultHeaders,
- }),
- );
- httpErrorCheck(response);
- const json: T[] = await response.json();
- return json;
- }
-
- public async fetchActivityTypesAsync(): Promise<Map<CopperActivityTypeCategory, CopperActivityTypeResponse[]>> {
- const response = await this._limiter.schedule(() =>
- fetchAsync(`${COPPER_URI}/activity_types`, {
- method: 'GET',
- headers: this._defaultHeaders,
- }),
- );
- httpErrorCheck(response);
- return response.json();
- }
-
- public async fetchCustomFieldsAsync(): Promise<CopperCustomFieldResponse[]> {
- const response = await this._limiter.schedule(() =>
- fetchAsync(`${COPPER_URI}/custom_field_definitions`, {
- method: 'GET',
- headers: this._defaultHeaders,
- }),
- );
- httpErrorCheck(response);
- return response.json();
- }
-}
diff --git a/packages/pipeline/src/data_sources/ddex/index.ts b/packages/pipeline/src/data_sources/ddex/index.ts
deleted file mode 100644
index 7ef92b90f..000000000
--- a/packages/pipeline/src/data_sources/ddex/index.ts
+++ /dev/null
@@ -1,77 +0,0 @@
-import { fetchAsync, logUtils } from '@0x/utils';
-
-const DDEX_BASE_URL = 'https://api.ddex.io/v3';
-const ACTIVE_MARKETS_URL = `${DDEX_BASE_URL}/markets`;
-const NO_AGGREGATION_LEVEL = 3; // See https://docs.ddex.io/#get-orderbook
-const ORDERBOOK_ENDPOINT = `/orderbook?level=${NO_AGGREGATION_LEVEL}`;
-export const DDEX_SOURCE = 'ddex';
-
-export interface DdexActiveMarketsResponse {
- status: number;
- desc: string;
- data: {
- markets: DdexMarket[];
- };
-}
-
-export interface DdexMarket {
- id: string;
- quoteToken: string;
- quoteTokenDecimals: number;
- quoteTokenAddress: string;
- baseToken: string;
- baseTokenDecimals: number;
- baseTokenAddress: string;
- minOrderSize: string;
- pricePrecision: number;
- priceDecimals: number;
- amountDecimals: number;
-}
-
-export interface DdexOrderbookResponse {
- status: number;
- desc: string;
- data: {
- orderBook: DdexOrderbook;
- };
-}
-
-export interface DdexOrderbook {
- marketId: string;
- bids: DdexOrder[];
- asks: DdexOrder[];
-}
-
-export interface DdexOrder {
- price: string;
- amount: string;
- orderId: string;
-}
-
-// tslint:disable:prefer-function-over-method
-// ^ Keep consistency with other sources and help logical organization
-export class DdexSource {
- /**
- * Call Ddex API to find out which markets they are maintaining orderbooks for.
- */
- public async getActiveMarketsAsync(): Promise<DdexMarket[]> {
- logUtils.log('Getting all active DDEX markets');
- const resp = await fetchAsync(ACTIVE_MARKETS_URL);
- const respJson: DdexActiveMarketsResponse = await resp.json();
- const markets = respJson.data.markets;
- logUtils.log(`Got ${markets.length} markets.`);
- return markets;
- }
-
- /**
- * Retrieve orderbook from Ddex API for a given market.
- * @param marketId String identifying the market we want data for. Eg. 'REP/AUG'
- */
- public async getMarketOrderbookAsync(marketId: string): Promise<DdexOrderbook> {
- logUtils.log(`${marketId}: Retrieving orderbook.`);
- const marketOrderbookUrl = `${ACTIVE_MARKETS_URL}/${marketId}${ORDERBOOK_ENDPOINT}`;
- const resp = await fetchAsync(marketOrderbookUrl);
- const respJson: DdexOrderbookResponse = await resp.json();
- return respJson.data.orderBook;
- }
-}
diff --git a/packages/pipeline/src/data_sources/idex/index.ts b/packages/pipeline/src/data_sources/idex/index.ts
deleted file mode 100644
index c1e53c08d..000000000
--- a/packages/pipeline/src/data_sources/idex/index.ts
+++ /dev/null
@@ -1,82 +0,0 @@
-import { fetchAsync } from '@0x/utils';
-
-const IDEX_BASE_URL = 'https://api.idex.market';
-const MARKETS_URL = `${IDEX_BASE_URL}/returnTicker`;
-const ORDERBOOK_URL = `${IDEX_BASE_URL}/returnOrderBook`;
-const MAX_ORDER_COUNT = 100; // Maximum based on https://github.com/AuroraDAO/idex-api-docs#returnorderbook
-export const IDEX_SOURCE = 'idex';
-
-export interface IdexMarketsResponse {
- [marketName: string]: IdexMarket;
-}
-
-export interface IdexMarket {
- last: string;
- high: string;
- low: string;
- lowestAsk: string;
- highestBid: string;
- percentChange: string;
- baseVolume: string;
- quoteVolume: string;
-}
-
-export interface IdexOrderbook {
- asks: IdexOrder[];
- bids: IdexOrder[];
-}
-
-export interface IdexOrder {
- price: string;
- amount: string;
- total: string;
- orderHash: string;
- params: IdexOrderParam;
-}
-
-export interface IdexOrderParam {
- tokenBuy: string;
- buySymbol: string;
- buyPrecision: number;
- amountBuy: string;
- tokenSell: string;
- sellSymbol: string;
- sellPrecision: number;
- amountSell: string;
- expires: number;
- nonce: number;
- user: string;
-}
-
-// tslint:disable:prefer-function-over-method
-// ^ Keep consistency with other sources and help logical organization
-export class IdexSource {
- /**
- * Call Idex API to find out which markets they are maintaining orderbooks for.
- */
- public async getMarketsAsync(): Promise<string[]> {
- const params = { method: 'POST' };
- const resp = await fetchAsync(MARKETS_URL, params);
- const respJson: IdexMarketsResponse = await resp.json();
- const markets: string[] = Object.keys(respJson);
- return markets;
- }
-
- /**
- * Retrieve orderbook from Idex API for a given market.
- * @param marketId String identifying the market we want data for. Eg. 'REP_AUG'
- */
- public async getMarketOrderbookAsync(marketId: string): Promise<IdexOrderbook> {
- const params = {
- method: 'POST',
- headers: { 'Content-Type': 'application/json' },
- body: JSON.stringify({
- market: marketId,
- count: MAX_ORDER_COUNT,
- }),
- };
- const resp = await fetchAsync(ORDERBOOK_URL, params);
- const respJson: IdexOrderbook = await resp.json();
- return respJson;
- }
-}
diff --git a/packages/pipeline/src/data_sources/oasis/index.ts b/packages/pipeline/src/data_sources/oasis/index.ts
deleted file mode 100644
index 3b30e9dfd..000000000
--- a/packages/pipeline/src/data_sources/oasis/index.ts
+++ /dev/null
@@ -1,103 +0,0 @@
-import { fetchAsync } from '@0x/utils';
-
-const OASIS_BASE_URL = 'https://data.makerdao.com/v1';
-const OASIS_MARKET_QUERY = `query {
- oasisMarkets(period: "1 week") {
- nodes {
- id
- base
- quote
- buyVol
- sellVol
- price
- high
- low
- }
- }
-}`;
-const OASIS_ORDERBOOK_QUERY = `query ($market: String!) {
- allOasisOrders(condition: { market: $market }) {
- totalCount
- nodes {
- market
- offerId
- price
- amount
- act
- }
- }
-}`;
-export const OASIS_SOURCE = 'oasis';
-
-export interface OasisMarket {
- id: string; // market symbol e.g MKRDAI
- base: string; // base symbol e.g MKR
- quote: string; // quote symbol e.g DAI
- buyVol: number; // total buy volume (base)
- sellVol: number; // total sell volume (base)
- price: number; // volume weighted price (quote)
- high: number; // max sell price
- low: number; // min buy price
-}
-
-export interface OasisMarketResponse {
- data: {
- oasisMarkets: {
- nodes: OasisMarket[];
- };
- };
-}
-
-export interface OasisOrder {
- offerId: number; // Offer Id
- market: string; // Market symbol (base/quote)
- price: string; // Offer price (quote)
- amount: string; // Offer amount (base)
- act: string; // Action (ask|bid)
-}
-
-export interface OasisOrderbookResponse {
- data: {
- allOasisOrders: {
- totalCount: number;
- nodes: OasisOrder[];
- };
- };
-}
-
-// tslint:disable:prefer-function-over-method
-// ^ Keep consistency with other sources and help logical organization
-export class OasisSource {
- /**
- * Call Ddex API to find out which markets they are maintaining orderbooks for.
- */
- public async getActiveMarketsAsync(): Promise<OasisMarket[]> {
- const params = {
- method: 'POST',
- headers: { 'Content-Type': 'application/json' },
- body: JSON.stringify({ query: OASIS_MARKET_QUERY }),
- };
- const resp = await fetchAsync(OASIS_BASE_URL, params);
- const respJson: OasisMarketResponse = await resp.json();
- const markets = respJson.data.oasisMarkets.nodes;
- return markets;
- }
-
- /**
- * Retrieve orderbook from Oasis API for a given market.
- * @param marketId String identifying the market we want data for. Eg. 'REPAUG'.
- */
- public async getMarketOrderbookAsync(marketId: string): Promise<OasisOrder[]> {
- const input = {
- market: marketId,
- };
- const params = {
- method: 'POST',
- headers: { 'Content-Type': 'application/json' },
- body: JSON.stringify({ query: OASIS_ORDERBOOK_QUERY, variables: input }),
- };
- const resp = await fetchAsync(OASIS_BASE_URL, params);
- const respJson: OasisOrderbookResponse = await resp.json();
- return respJson.data.allOasisOrders.nodes;
- }
-}
diff --git a/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts b/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts
deleted file mode 100644
index 85042501b..000000000
--- a/packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts
+++ /dev/null
@@ -1,110 +0,0 @@
-// tslint:disable:no-duplicate-imports
-import { fetchAsync } from '@0x/utils';
-import Bottleneck from 'bottleneck';
-import { stringify } from 'querystring';
-import * as R from 'ramda';
-
-import { TradingPair } from '../../utils/get_ohlcv_trading_pairs';
-
-export interface CryptoCompareOHLCVResponse {
- Data: CryptoCompareOHLCVRecord[];
- Response: string;
- Message: string;
- Type: number;
-}
-
-export interface CryptoCompareOHLCVRecord {
- time: number; // in seconds, not milliseconds
- close: number;
- high: number;
- low: number;
- open: number;
- volumefrom: number;
- volumeto: number;
-}
-
-export interface CryptoCompareOHLCVParams {
- fsym: string;
- tsym: string;
- e?: string;
- aggregate?: string;
- aggregatePredictableTimePeriods?: boolean;
- limit?: number;
- toTs?: number;
-}
-
-const ONE_HOUR = 60 * 60 * 1000; // tslint:disable-line:custom-no-magic-numbers
-const ONE_SECOND = 1000;
-const ONE_HOUR_AGO = new Date().getTime() - ONE_HOUR;
-const HTTP_OK_STATUS = 200;
-const CRYPTO_COMPARE_VALID_EMPTY_RESPONSE_TYPE = 96;
-const MAX_PAGE_SIZE = 2000;
-
-export class CryptoCompareOHLCVSource {
- public readonly intervalBetweenRecords = ONE_HOUR;
- public readonly defaultExchange = 'CCCAGG';
- public readonly interval = this.intervalBetweenRecords * MAX_PAGE_SIZE; // the hourly API returns data for one interval at a time
- private readonly _url: string = 'https://min-api.cryptocompare.com/data/histohour?';
-
- // rate-limit for all API calls through this class instance
- private readonly _limiter: Bottleneck;
- constructor(maxReqsPerSecond: number) {
- this._limiter = new Bottleneck({
- minTime: ONE_SECOND / maxReqsPerSecond,
- reservoir: 30,
- reservoirRefreshAmount: 30,
- reservoirRefreshInterval: ONE_SECOND,
- });
- }
-
- // gets OHLCV records starting from pair.latest
- public async getHourlyOHLCVAsync(pair: TradingPair): Promise<CryptoCompareOHLCVRecord[]> {
- const params = {
- e: this.defaultExchange,
- fsym: pair.fromSymbol,
- tsym: pair.toSymbol,
- limit: MAX_PAGE_SIZE,
- toTs: Math.floor((pair.latestSavedTime + this.interval) / ONE_SECOND), // CryptoCompare uses timestamp in seconds. not ms
- };
- const url = this._url + stringify(params);
- const response = await this._limiter.schedule(() => fetchAsync(url));
- if (response.status !== HTTP_OK_STATUS) {
- throw new Error(`HTTP error while scraping Crypto Compare: [${response}]`);
- }
- const json: CryptoCompareOHLCVResponse = await response.json();
- if (
- (json.Response === 'Error' || json.Data.length === 0) &&
- json.Type !== CRYPTO_COMPARE_VALID_EMPTY_RESPONSE_TYPE
- ) {
- throw new Error(JSON.stringify(json));
- }
- return json.Data.filter(rec => {
- return (
- // Crypto Compare takes ~30 mins to finalise records
- rec.time * ONE_SECOND < ONE_HOUR_AGO && rec.time * ONE_SECOND > pair.latestSavedTime && hasData(rec)
- );
- });
- }
- public generateBackfillIntervals(pair: TradingPair): TradingPair[] {
- const now = new Date().getTime();
- const f = (p: TradingPair): false | [TradingPair, TradingPair] => {
- if (p.latestSavedTime > now) {
- return false;
- } else {
- return [p, R.merge(p, { latestSavedTime: p.latestSavedTime + this.interval })];
- }
- };
- return R.unfold(f, pair);
- }
-}
-
-function hasData(record: CryptoCompareOHLCVRecord): boolean {
- return (
- record.close !== 0 ||
- record.open !== 0 ||
- record.high !== 0 ||
- record.low !== 0 ||
- record.volumefrom !== 0 ||
- record.volumeto !== 0
- );
-}
diff --git a/packages/pipeline/src/data_sources/paradex/index.ts b/packages/pipeline/src/data_sources/paradex/index.ts
deleted file mode 100644
index 46d448f4b..000000000
--- a/packages/pipeline/src/data_sources/paradex/index.ts
+++ /dev/null
@@ -1,92 +0,0 @@
-import { fetchAsync, logUtils } from '@0x/utils';
-
-const PARADEX_BASE_URL = 'https://api.paradex.io/consumer/v0';
-const ACTIVE_MARKETS_URL = `${PARADEX_BASE_URL}/markets`;
-const ORDERBOOK_ENDPOINT = `${PARADEX_BASE_URL}/orderbook`;
-const TOKEN_INFO_ENDPOINT = `${PARADEX_BASE_URL}/tokens`;
-export const PARADEX_SOURCE = 'paradex';
-
-export type ParadexActiveMarketsResponse = ParadexMarket[];
-
-export interface ParadexMarket {
- id: string;
- symbol: string;
- baseToken: string;
- quoteToken: string;
- minOrderSize: string;
- maxOrderSize: string;
- priceMaxDecimals: number;
- amountMaxDecimals: number;
- // These are not native to the Paradex API response. We tag them on later
- // by calling the token endpoint and joining on symbol.
- baseTokenAddress?: string;
- quoteTokenAddress?: string;
-}
-
-export interface ParadexOrderbookResponse {
- marketId: number;
- marketSymbol: string;
- bids: ParadexOrder[];
- asks: ParadexOrder[];
-}
-
-export interface ParadexOrder {
- amount: string;
- price: string;
-}
-
-export type ParadexTokenInfoResponse = ParadexTokenInfo[];
-
-export interface ParadexTokenInfo {
- name: string;
- symbol: string;
- address: string;
-}
-
-export class ParadexSource {
- private readonly _apiKey: string;
-
- constructor(apiKey: string) {
- this._apiKey = apiKey;
- }
-
- /**
- * Call Paradex API to find out which markets they are maintaining orderbooks for.
- */
- public async getActiveMarketsAsync(): Promise<ParadexActiveMarketsResponse> {
- logUtils.log('Getting all active Paradex markets.');
- const resp = await fetchAsync(ACTIVE_MARKETS_URL, {
- headers: { 'API-KEY': this._apiKey },
- });
- const markets: ParadexActiveMarketsResponse = await resp.json();
- logUtils.log(`Got ${markets.length} markets.`);
- return markets;
- }
-
- /**
- * Call Paradex API to find out their token information.
- */
- public async getTokenInfoAsync(): Promise<ParadexTokenInfoResponse> {
- logUtils.log('Getting token information from Paradex.');
- const resp = await fetchAsync(TOKEN_INFO_ENDPOINT, {
- headers: { 'API-KEY': this._apiKey },
- });
- const tokens: ParadexTokenInfoResponse = await resp.json();
- logUtils.log(`Got information for ${tokens.length} tokens.`);
- return tokens;
- }
-
- /**
- * Retrieve orderbook from Paradex API for a given market.
- * @param marketSymbol String representing the market we want data for.
- */
- public async getMarketOrderbookAsync(marketSymbol: string): Promise<ParadexOrderbookResponse> {
- logUtils.log(`${marketSymbol}: Retrieving orderbook.`);
- const marketOrderbookUrl = `${ORDERBOOK_ENDPOINT}?market=${marketSymbol}`;
- const resp = await fetchAsync(marketOrderbookUrl, {
- headers: { 'API-KEY': this._apiKey },
- });
- const orderbookResponse: ParadexOrderbookResponse = await resp.json();
- return orderbookResponse;
- }
-}
diff --git a/packages/pipeline/src/data_sources/relayer-registry/index.ts b/packages/pipeline/src/data_sources/relayer-registry/index.ts
deleted file mode 100644
index 8133f5eae..000000000
--- a/packages/pipeline/src/data_sources/relayer-registry/index.ts
+++ /dev/null
@@ -1,33 +0,0 @@
-import axios from 'axios';
-
-export interface RelayerResponse {
- name: string;
- homepage_url: string;
- app_url: string;
- header_img: string;
- logo_img: string;
- networks: RelayerResponseNetwork[];
-}
-
-export interface RelayerResponseNetwork {
- networkId: number;
- sra_http_endpoint?: string;
- sra_ws_endpoint?: string;
- static_order_fields?: {
- fee_recipient_addresses?: string[];
- taker_addresses?: string[];
- };
-}
-
-export class RelayerRegistrySource {
- private readonly _url: string;
-
- constructor(url: string) {
- this._url = url;
- }
-
- public async getRelayerInfoAsync(): Promise<Map<string, RelayerResponse>> {
- const resp = await axios.get<Map<string, RelayerResponse>>(this._url);
- return resp.data;
- }
-}
diff --git a/packages/pipeline/src/data_sources/trusted_tokens/index.ts b/packages/pipeline/src/data_sources/trusted_tokens/index.ts
deleted file mode 100644
index 552739fb9..000000000
--- a/packages/pipeline/src/data_sources/trusted_tokens/index.ts
+++ /dev/null
@@ -1,29 +0,0 @@
-import axios from 'axios';
-
-export interface ZeroExTrustedTokenMeta {
- address: string;
- name: string;
- symbol: string;
- decimals: number;
-}
-
-export interface MetamaskTrustedTokenMeta {
- address: string;
- name: string;
- erc20: boolean;
- symbol: string;
- decimals: number;
-}
-
-export class TrustedTokenSource<T> {
- private readonly _url: string;
-
- constructor(url: string) {
- this._url = url;
- }
-
- public async getTrustedTokenMetaAsync(): Promise<T> {
- const resp = await axios.get<T>(this._url);
- return resp.data;
- }
-}
diff --git a/packages/pipeline/src/data_sources/web3/index.ts b/packages/pipeline/src/data_sources/web3/index.ts
deleted file mode 100644
index 45a9ea161..000000000
--- a/packages/pipeline/src/data_sources/web3/index.ts
+++ /dev/null
@@ -1,22 +0,0 @@
-import { Web3ProviderEngine } from '@0x/subproviders';
-import { Web3Wrapper } from '@0x/web3-wrapper';
-import { BlockWithoutTransactionData, Transaction } from 'ethereum-types';
-
-export class Web3Source {
- private readonly _web3Wrapper: Web3Wrapper;
- constructor(provider: Web3ProviderEngine) {
- this._web3Wrapper = new Web3Wrapper(provider);
- }
-
- public async getBlockInfoAsync(blockNumber: number): Promise<BlockWithoutTransactionData> {
- const block = await this._web3Wrapper.getBlockIfExistsAsync(blockNumber);
- if (block == null) {
- return Promise.reject(new Error(`Could not find block for given block number: ${blockNumber}`));
- }
- return block;
- }
-
- public async getTransactionInfoAsync(txHash: string): Promise<Transaction> {
- return this._web3Wrapper.getTransactionByHashAsync(txHash);
- }
-}