diff options
Diffstat (limited to 'packages/pipeline')
12 files changed, 208 insertions, 2 deletions
diff --git a/packages/pipeline/migrations/1542655823221-NewMetadataAndOHLCVTables.ts b/packages/pipeline/migrations/1542655823221-NewMetadataAndOHLCVTables.ts new file mode 100644 index 000000000..5c7ec6de7 --- /dev/null +++ b/packages/pipeline/migrations/1542655823221-NewMetadataAndOHLCVTables.ts @@ -0,0 +1,46 @@ +import {MigrationInterface, QueryRunner} from 'typeorm'; + +export class NewMetadataAndOHLCVTables1542655823221 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise<any> { + + await queryRunner.query(` + CREATE TABLE raw.trusted_tokens ( + address VARCHAR NOT NULL, + authority VARCHAR NOT NULL, + + PRIMARY KEY (address, authority) + ); + `); + + await queryRunner.query(` + CREATE TABLE raw.ohlcv_external ( + exchange VARCHAR NOT NULL, + from_symbol VARCHAR NOT NULL, + to_symbol VARCHAR NOT NULL, + start_time BIGINT NOT NULL, + end_time BIGINT NOT NULL, + + open DOUBLE PRECISION NOT NULL, + close DOUBLE PRECISION NOT NULL, + low DOUBLE PRECISION NOT NULL, + high DOUBLE PRECISION NOT NULL, + volume_from DOUBLE PRECISION NOT NULL, + volume_to DOUBLE PRECISION NOT NULL, + + source VARCHAR NOT NULL, + observed_timestamp BIGINT NOT NULL, + + PRIMARY KEY (exchange, from_symbol, to_symbol, start_time, end_time, source, observed_timestamp) + ); + `); + } + + public async down(queryRunner: QueryRunner): Promise<any> { + + await queryRunner.dropTable('raw.trusted_tokens'); + + await queryRunner.dropTable('raw.ohlcv_external'); + } + +} diff --git a/packages/pipeline/src/data_sources/trusted_tokens/index.ts b/packages/pipeline/src/data_sources/trusted_tokens/index.ts new file mode 100644 index 000000000..552739fb9 --- /dev/null +++ b/packages/pipeline/src/data_sources/trusted_tokens/index.ts @@ -0,0 +1,29 @@ +import axios from 'axios'; + +export interface ZeroExTrustedTokenMeta { + address: string; + name: string; + symbol: string; + decimals: number; +} + +export interface MetamaskTrustedTokenMeta { + address: string; + name: string; + erc20: boolean; + symbol: string; + decimals: number; +} + +export class TrustedTokenSource<T> { + private readonly _url: string; + + constructor(url: string) { + this._url = url; + } + + public async getTrustedTokenMetaAsync(): Promise<T> { + const resp = await axios.get<T>(this._url); + return resp.data; + } +} diff --git a/packages/pipeline/src/entities/index.ts b/packages/pipeline/src/entities/index.ts index d3c61bcb9..769efb092 100644 --- a/packages/pipeline/src/entities/index.ts +++ b/packages/pipeline/src/entities/index.ts @@ -10,6 +10,8 @@ export { Relayer } from './relayer'; export { SraOrder } from './sra_order'; export { Transaction } from './transaction'; export { TokenOnChainMetadata } from './token_on_chain_metadata'; +export { TrustedToken } from './trusted_tokens'; export { SraOrdersObservedTimeStamp, createObservedTimestampForOrder } from './sra_order_observed_timestamp'; +export { OHLCVExternal } from './ohlcv_external'; export type ExchangeEvent = ExchangeFillEvent | ExchangeCancelEvent | ExchangeCancelUpToEvent; diff --git a/packages/pipeline/src/entities/ohlcv_external.ts b/packages/pipeline/src/entities/ohlcv_external.ts new file mode 100644 index 000000000..3fbb4c499 --- /dev/null +++ b/packages/pipeline/src/entities/ohlcv_external.ts @@ -0,0 +1,20 @@ +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +@Entity({ name: 'ohlcv_external', schema: 'raw' }) +export class OHLCVExternal { + @PrimaryColumn() public exchange!: string; + @PrimaryColumn() public from_symbol!: string; + @PrimaryColumn() public to_symbol!: string; + @PrimaryColumn() public start_time!: number; + @PrimaryColumn() public end_time!: number; + + @Column() public open!: number; + @Column() public close!: number; + @Column() public low!: number; + @Column() public high!: number; + @Column() public volume_from!: number; + @Column() public volume_to!: number; + + @PrimaryColumn() public source!: string; + @PrimaryColumn() public observed_timestamp!: number; +}
\ No newline at end of file diff --git a/packages/pipeline/src/entities/token_on_chain_metadata.ts b/packages/pipeline/src/entities/token_on_chain_metadata.ts index 325d28e91..ca1f1827b 100644 --- a/packages/pipeline/src/entities/token_on_chain_metadata.ts +++ b/packages/pipeline/src/entities/token_on_chain_metadata.ts @@ -14,3 +14,9 @@ export class TokenOnChainMetadata { @Column({ type: 'nvarchar', nullable: false }) public name!: string; } + +@Entity({ name: 'trusted_tokens', schema: 'raw' }) +export class TrustedTokens { + @PrimaryColumn() public address!: string; + @PrimaryColumn() public authority!: string; +} diff --git a/packages/pipeline/src/entities/trusted_tokens.ts b/packages/pipeline/src/entities/trusted_tokens.ts new file mode 100644 index 000000000..6ec27c963 --- /dev/null +++ b/packages/pipeline/src/entities/trusted_tokens.ts @@ -0,0 +1,7 @@ +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +@Entity({ name: 'trusted_tokens', schema: 'raw' }) +export class TrustedToken { + @PrimaryColumn() public address!: string; + @PrimaryColumn() public authority!: string; +} diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts index 64b5b1827..1bd337866 100644 --- a/packages/pipeline/src/ormconfig.ts +++ b/packages/pipeline/src/ormconfig.ts @@ -9,6 +9,7 @@ import { SraOrder, SraOrdersObservedTimeStamp, Transaction, + TrustedToken, } from './entities'; const entities = [ @@ -20,6 +21,7 @@ const entities = [ SraOrder, SraOrdersObservedTimeStamp, Transaction, + TrustedToken, ]; const config: ConnectionOptions = { diff --git a/packages/pipeline/src/parsers/ohlcv_external/index.ts b/packages/pipeline/src/parsers/ohlcv_external/index.ts new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/packages/pipeline/src/parsers/ohlcv_external/index.ts diff --git a/packages/pipeline/src/parsers/trusted_tokens/index.ts b/packages/pipeline/src/parsers/trusted_tokens/index.ts new file mode 100644 index 000000000..a6d32de35 --- /dev/null +++ b/packages/pipeline/src/parsers/trusted_tokens/index.ts @@ -0,0 +1,37 @@ +import * as R from 'ramda'; + +import { MetamaskTrustedTokenMeta, ZeroExTrustedTokenMeta } from '../../data_sources/trusted_tokens'; +import { TrustedToken } from '../../entities'; + +/** + * Parses Metamask's trusted tokens list. + * @param rawResp raw response from the metamask json file. + */ +export function parseMetamaskTrustedTokens(rawResp: Map<string, MetamaskTrustedTokenMeta>): TrustedToken[] { + const parsedAsObject = R.mapObjIndexed(parseMetamaskTrustedToken, rawResp); + return R.values(parsedAsObject); +} + +export function parseZeroExTrustedTokens(rawResp: ZeroExTrustedTokenMeta[]): TrustedToken[] { + return R.map(parseZeroExTrustedToken, rawResp); +} + +function parseMetamaskTrustedToken(resp: MetamaskTrustedTokenMeta, address: string): TrustedToken { + + const trustedToken = new TrustedToken(); + + trustedToken.address = address; + trustedToken.authority = 'metamask'; + + return trustedToken; +} + +function parseZeroExTrustedToken(resp: ZeroExTrustedTokenMeta): TrustedToken { + + const trustedToken = new TrustedToken(); + + trustedToken.address = resp.address; + trustedToken.authority = '0x'; + + return trustedToken; +} diff --git a/packages/pipeline/src/scripts/pull_missing_events.ts b/packages/pipeline/src/scripts/pull_missing_events.ts index 0b7f6287f..68cabe3de 100644 --- a/packages/pipeline/src/scripts/pull_missing_events.ts +++ b/packages/pipeline/src/scripts/pull_missing_events.ts @@ -64,7 +64,7 @@ async function getCancelUpToEventsAsync(eventsSource: ExchangeEventsSource): Pro await saveEventsAsync(startBlock === EXCHANGE_START_BLOCK, repository, events); } -const tabelNameRegex = /^[a-zA-Z_]*$/; +const tableNameRegex = /^[a-zA-Z_]*$/; async function getStartBlockAsync<T extends ExchangeEvent>(repository: Repository<T>): Promise<number> { const fillEventCount = await repository.count(); @@ -73,7 +73,7 @@ async function getStartBlockAsync<T extends ExchangeEvent>(repository: Repositor return EXCHANGE_START_BLOCK; } const tableName = repository.metadata.tableName; - if (!tabelNameRegex.test(tableName)) { + if (!tableNameRegex.test(tableName)) { throw new Error('Unexpected special character in table name: ' + tableName); } const queryResult = await connection.query( diff --git a/packages/pipeline/src/scripts/pull_trusted_tokens.ts b/packages/pipeline/src/scripts/pull_trusted_tokens.ts new file mode 100644 index 000000000..67d9e08d1 --- /dev/null +++ b/packages/pipeline/src/scripts/pull_trusted_tokens.ts @@ -0,0 +1,51 @@ +import 'reflect-metadata'; +import { Connection, ConnectionOptions, createConnection } from 'typeorm'; + +import { MetamaskTrustedTokenMeta, TrustedTokenSource, ZeroExTrustedTokenMeta } from '../data_sources/trusted_tokens'; +import { TrustedToken } from '../entities'; +import * as ormConfig from '../ormconfig'; +import { parseMetamaskTrustedTokens, parseZeroExTrustedTokens } from '../parsers/trusted_tokens'; +import { handleError } from '../utils'; + +const METAMASK_TRUSTED_TOKENS_URL = + 'https://raw.githubusercontent.com/MetaMask/eth-contract-metadata/master/contract-map.json'; + +const ZEROEX_TRUSTED_TOKENS_URL = + 'https://website-api.0xproject.com/tokens'; + +let connection: Connection; + +(async () => { + connection = await createConnection(ormConfig as ConnectionOptions); + await getMetamaskTrustedTokens(); + await getZeroExTrustedTokens(); + process.exit(0); +})().catch(handleError); + +async function getMetamaskTrustedTokens(): Promise<void> { + // tslint:disable-next-line + console.log('Getting latest metamask trusted tokens list ...'); + const trustedTokensRepository = connection.getRepository(TrustedToken); + const trustedTokensSource = new TrustedTokenSource<Map<string, MetamaskTrustedTokenMeta>>(METAMASK_TRUSTED_TOKENS_URL); + const resp = await trustedTokensSource.getTrustedTokenMetaAsync(); + const trustedTokens = parseMetamaskTrustedTokens(resp); + // tslint:disable-next-line + console.log('Saving metamask trusted tokens list'); + await trustedTokensRepository.save(trustedTokens); + // tslint:disable-next-line + console.log('Done saving metamask trusted tokens.') +} + +async function getZeroExTrustedTokens(): Promise<void> { + // tslint:disable-next-line + console.log('Getting latest 0x trusted tokens list ...'); + const trustedTokensRepository = connection.getRepository(TrustedToken); + const trustedTokensSource = new TrustedTokenSource<ZeroExTrustedTokenMeta[]>(ZEROEX_TRUSTED_TOKENS_URL); + const resp = await trustedTokensSource.getTrustedTokenMetaAsync(); + const trustedTokens = parseZeroExTrustedTokens(resp); + // tslint:disable-next-line + console.log('Saving metamask trusted tokens list'); + await trustedTokensRepository.save(trustedTokens); + // tslint:disable-next-line + console.log('Done saving metamask trusted tokens.'); +} diff --git a/packages/pipeline/src/utils/index.ts b/packages/pipeline/src/utils/index.ts index 78aa89374..b66ab40ab 100644 --- a/packages/pipeline/src/utils/index.ts +++ b/packages/pipeline/src/utils/index.ts @@ -49,3 +49,9 @@ class BigNumberTransformer implements ValueTransformer { } export const bigNumberTransformer = new BigNumberTransformer(); + +export function getHourInUnixTime(): number { + const currentTime: number = Date.now(); + // tslint:disable-next-line + return currentTime - (currentTime % (3600 * 1000)); +} |