From 7b4a0d4f8aff373b632983400d28dfdce602114a Mon Sep 17 00:00:00 2001 From: fragosti Date: Wed, 30 Jan 2019 10:28:06 -0800 Subject: fix bloxy datasource getDexTradesAsync to allow for tx_hash repetitions in dedupe logic --- packages/pipeline/src/data_sources/bloxy/index.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/pipeline/src/data_sources/bloxy/index.ts b/packages/pipeline/src/data_sources/bloxy/index.ts index bba424f67..68764ca98 100644 --- a/packages/pipeline/src/data_sources/bloxy/index.ts +++ b/packages/pipeline/src/data_sources/bloxy/index.ts @@ -2,6 +2,7 @@ import axios from 'axios'; import * as R from 'ramda'; import { logUtils } from '@0x/utils'; + // URL to use for getting dex trades from Bloxy. export const BLOXY_DEX_TRADES_URL = 'https://bloxy.info/api/dex/trades'; // Number of trades to get at once. Must be less than or equal to MAX_OFFSET. @@ -76,8 +77,11 @@ export class BloxySource { */ public async getDexTradesAsync(lastSeenTimestamp: number): Promise { const allTrades = await this._scrapeAllDexTradesAsync(lastSeenTimestamp); - logUtils.log('Removing duplicate entries'); - const uniqueTrades = R.uniqBy(R.toString, allTrades) as BloxyTrade[]; + logUtils.log(`Removing duplicates from ${allTrades.length} entries`); + const uniqueTrades = R.uniqBy( + (trade: BloxyTrade) => `${trade.tradeIndex}-${trade.tx_hash}`, + allTrades, + ) as BloxyTrade[]; logUtils.log(`Removed ${allTrades.length - uniqueTrades.length} duplicate entries`); return uniqueTrades; } -- cgit v1.2.3