From dea89c4e221d5b22de97b27573719cd27ce250c7 Mon Sep 17 00:00:00 2001 From: Jake Ellowitz Date: Mon, 19 Nov 2018 19:11:51 -0800 Subject: metadata and trusted sources in same raw table --- .../1542655823221-NewMetadataAndOHLCVTables.ts | 22 +++++++++- packages/pipeline/src/entities/index.ts | 3 +- packages/pipeline/src/entities/ohlcv_external.ts | 18 ++++----- packages/pipeline/src/entities/token_metadata.ts | 19 +++++++++ .../src/entities/token_on_chain_metadata.ts | 22 ---------- packages/pipeline/src/entities/trusted_tokens.ts | 7 ---- packages/pipeline/src/ormconfig.ts | 4 +- .../pipeline/src/parsers/token_metadata/index.ts | 47 ++++++++++++++++++++++ .../pipeline/src/parsers/trusted_tokens/index.ts | 37 ----------------- .../pipeline/src/scripts/pull_trusted_tokens.ts | 8 ++-- packages/pipeline/src/utils/index.ts | 3 ++ 11 files changed, 105 insertions(+), 85 deletions(-) create mode 100644 packages/pipeline/src/entities/token_metadata.ts delete mode 100644 packages/pipeline/src/entities/token_on_chain_metadata.ts delete mode 100644 packages/pipeline/src/entities/trusted_tokens.ts create mode 100644 packages/pipeline/src/parsers/token_metadata/index.ts delete mode 100644 packages/pipeline/src/parsers/trusted_tokens/index.ts diff --git a/packages/pipeline/migrations/1542655823221-NewMetadataAndOHLCVTables.ts b/packages/pipeline/migrations/1542655823221-NewMetadataAndOHLCVTables.ts index 5c7ec6de7..8d131271b 100644 --- a/packages/pipeline/migrations/1542655823221-NewMetadataAndOHLCVTables.ts +++ b/packages/pipeline/migrations/1542655823221-NewMetadataAndOHLCVTables.ts @@ -2,17 +2,23 @@ import {MigrationInterface, QueryRunner} from 'typeorm'; export class NewMetadataAndOHLCVTables1542655823221 implements MigrationInterface { + // tslint:disable-next-line public async up(queryRunner: QueryRunner): Promise { await queryRunner.query(` - CREATE TABLE raw.trusted_tokens ( + CREATE TABLE raw.token_metadata ( address VARCHAR NOT NULL, authority VARCHAR NOT NULL, + decimals INT NULL, + symbol VARCHAR NULL, + name VARCHAR NULL, PRIMARY KEY (address, authority) ); `); + await queryRunner.dropTable('raw.token_on_chain_metadata'); + await queryRunner.query(` CREATE TABLE raw.ohlcv_external ( exchange VARCHAR NOT NULL, @@ -36,9 +42,21 @@ export class NewMetadataAndOHLCVTables1542655823221 implements MigrationInterfac `); } + // tslint:disable-next-line public async down(queryRunner: QueryRunner): Promise { - await queryRunner.dropTable('raw.trusted_tokens'); + await queryRunner.query(` + CREATE TABLE raw.token_on_chain_metadata ( + address VARCHAR NOT NULL, + decimals INT NULL, + symbol VARCHAR NULL, + name VARCHAR NULL, + + PRIMARY KEY (address) + ); + `); + + await queryRunner.dropTable('raw.token_metadata'); await queryRunner.dropTable('raw.ohlcv_external'); } diff --git a/packages/pipeline/src/entities/index.ts b/packages/pipeline/src/entities/index.ts index 769efb092..a8f159a2a 100644 --- a/packages/pipeline/src/entities/index.ts +++ b/packages/pipeline/src/entities/index.ts @@ -9,8 +9,7 @@ export { ExchangeFillEvent } from './exchange_fill_event'; export { Relayer } from './relayer'; export { SraOrder } from './sra_order'; export { Transaction } from './transaction'; -export { TokenOnChainMetadata } from './token_on_chain_metadata'; -export { TrustedToken } from './trusted_tokens'; +export { TokenMetadata } from './token_metadata'; export { SraOrdersObservedTimeStamp, createObservedTimestampForOrder } from './sra_order_observed_timestamp'; export { OHLCVExternal } from './ohlcv_external'; diff --git a/packages/pipeline/src/entities/ohlcv_external.ts b/packages/pipeline/src/entities/ohlcv_external.ts index 3fbb4c499..95cd4f2f5 100644 --- a/packages/pipeline/src/entities/ohlcv_external.ts +++ b/packages/pipeline/src/entities/ohlcv_external.ts @@ -3,18 +3,18 @@ import { Column, Entity, PrimaryColumn } from 'typeorm'; @Entity({ name: 'ohlcv_external', schema: 'raw' }) export class OHLCVExternal { @PrimaryColumn() public exchange!: string; - @PrimaryColumn() public from_symbol!: string; - @PrimaryColumn() public to_symbol!: string; - @PrimaryColumn() public start_time!: number; - @PrimaryColumn() public end_time!: number; - + @PrimaryColumn() public fromSymbol!: string; + @PrimaryColumn() public toSymbol!: string; + @PrimaryColumn() public startTime!: number; + @PrimaryColumn() public endTime!: number; + @Column() public open!: number; @Column() public close!: number; @Column() public low!: number; @Column() public high!: number; - @Column() public volume_from!: number; - @Column() public volume_to!: number; + @Column() public volumeFrom!: number; + @Column() public volumeTo!: number; @PrimaryColumn() public source!: string; - @PrimaryColumn() public observed_timestamp!: number; -} \ No newline at end of file + @PrimaryColumn() public observedTimestamp!: number; +} diff --git a/packages/pipeline/src/entities/token_metadata.ts b/packages/pipeline/src/entities/token_metadata.ts new file mode 100644 index 000000000..4a0bceefe --- /dev/null +++ b/packages/pipeline/src/entities/token_metadata.ts @@ -0,0 +1,19 @@ +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +@Entity({ name: 'token_metadata', schema: 'raw' }) +export class TokenMetadata { + @PrimaryColumn({ type: 'varchar', nullable: false }) + public address!: string; + + @PrimaryColumn({ type: 'varchar', nullable: false }) + public authority!: string; + + @Column({ type: 'integer', nullable: true }) + public decimals!: number; + + @Column({ type: 'varchar', nullable: true }) + public symbol!: string; + + @Column({ type: 'varchar', nullable: true }) + public name!: string; +} diff --git a/packages/pipeline/src/entities/token_on_chain_metadata.ts b/packages/pipeline/src/entities/token_on_chain_metadata.ts deleted file mode 100644 index ca1f1827b..000000000 --- a/packages/pipeline/src/entities/token_on_chain_metadata.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { Column, Entity, PrimaryColumn } from 'typeorm'; - -@Entity({ name: 'token_on_chain_metadata', schema: 'raw' }) -export class TokenOnChainMetadata { - @PrimaryColumn({ type: 'nvarchar', nullable: false }) - public address!: string; - - @Column({ type: 'integer', nullable: false }) - public decimals!: number; - - @Column({ type: 'nvarchar', nullable: false }) - public symbol!: string; - - @Column({ type: 'nvarchar', nullable: false }) - public name!: string; -} - -@Entity({ name: 'trusted_tokens', schema: 'raw' }) -export class TrustedTokens { - @PrimaryColumn() public address!: string; - @PrimaryColumn() public authority!: string; -} diff --git a/packages/pipeline/src/entities/trusted_tokens.ts b/packages/pipeline/src/entities/trusted_tokens.ts deleted file mode 100644 index 6ec27c963..000000000 --- a/packages/pipeline/src/entities/trusted_tokens.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { Column, Entity, PrimaryColumn } from 'typeorm'; - -@Entity({ name: 'trusted_tokens', schema: 'raw' }) -export class TrustedToken { - @PrimaryColumn() public address!: string; - @PrimaryColumn() public authority!: string; -} diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts index 1bd337866..e8277a439 100644 --- a/packages/pipeline/src/ormconfig.ts +++ b/packages/pipeline/src/ormconfig.ts @@ -8,8 +8,8 @@ import { Relayer, SraOrder, SraOrdersObservedTimeStamp, + TokenMetadata, Transaction, - TrustedToken, } from './entities'; const entities = [ @@ -20,8 +20,8 @@ const entities = [ Relayer, SraOrder, SraOrdersObservedTimeStamp, + TokenMetadata, Transaction, - TrustedToken, ]; const config: ConnectionOptions = { diff --git a/packages/pipeline/src/parsers/token_metadata/index.ts b/packages/pipeline/src/parsers/token_metadata/index.ts new file mode 100644 index 000000000..d09def0db --- /dev/null +++ b/packages/pipeline/src/parsers/token_metadata/index.ts @@ -0,0 +1,47 @@ +import * as R from 'ramda'; + +import { MetamaskTrustedTokenMeta, ZeroExTrustedTokenMeta } from '../../data_sources/trusted_tokens'; +import { TokenMetadata } from '../../entities'; + +/** + * Parses Metamask's trusted tokens list. + * @param rawResp raw response from the metamask json file. + */ +export function parseMetamaskTrustedTokens(rawResp: Map): TokenMetadata[] { + const parsedAsObject = R.mapObjIndexed(parseMetamaskTrustedToken, rawResp); + return R.values(parsedAsObject); +} + +/** + * Parses 0x's trusted tokens list. + * @param rawResp raw response from the 0x json file. + */ +export function parseZeroExTrustedTokens(rawResp: ZeroExTrustedTokenMeta[]): TokenMetadata[] { + return R.map(parseZeroExTrustedToken, rawResp); +} + +function parseMetamaskTrustedToken(resp: MetamaskTrustedTokenMeta, address: string): TokenMetadata { + + const trustedToken = new TokenMetadata(); + + trustedToken.address = address; + trustedToken.decimals = resp.decimals; + trustedToken.symbol = resp.symbol; + trustedToken.name = resp.name; + trustedToken.authority = 'metamask'; + + return trustedToken; +} + +function parseZeroExTrustedToken(resp: ZeroExTrustedTokenMeta): TokenMetadata { + + const trustedToken = new TokenMetadata(); + + trustedToken.address = resp.address; + trustedToken.decimals = resp.decimals; + trustedToken.symbol = resp.symbol; + trustedToken.name = resp.name; + trustedToken.authority = '0x'; + + return trustedToken; +} diff --git a/packages/pipeline/src/parsers/trusted_tokens/index.ts b/packages/pipeline/src/parsers/trusted_tokens/index.ts deleted file mode 100644 index a6d32de35..000000000 --- a/packages/pipeline/src/parsers/trusted_tokens/index.ts +++ /dev/null @@ -1,37 +0,0 @@ -import * as R from 'ramda'; - -import { MetamaskTrustedTokenMeta, ZeroExTrustedTokenMeta } from '../../data_sources/trusted_tokens'; -import { TrustedToken } from '../../entities'; - -/** - * Parses Metamask's trusted tokens list. - * @param rawResp raw response from the metamask json file. - */ -export function parseMetamaskTrustedTokens(rawResp: Map): TrustedToken[] { - const parsedAsObject = R.mapObjIndexed(parseMetamaskTrustedToken, rawResp); - return R.values(parsedAsObject); -} - -export function parseZeroExTrustedTokens(rawResp: ZeroExTrustedTokenMeta[]): TrustedToken[] { - return R.map(parseZeroExTrustedToken, rawResp); -} - -function parseMetamaskTrustedToken(resp: MetamaskTrustedTokenMeta, address: string): TrustedToken { - - const trustedToken = new TrustedToken(); - - trustedToken.address = address; - trustedToken.authority = 'metamask'; - - return trustedToken; -} - -function parseZeroExTrustedToken(resp: ZeroExTrustedTokenMeta): TrustedToken { - - const trustedToken = new TrustedToken(); - - trustedToken.address = resp.address; - trustedToken.authority = '0x'; - - return trustedToken; -} diff --git a/packages/pipeline/src/scripts/pull_trusted_tokens.ts b/packages/pipeline/src/scripts/pull_trusted_tokens.ts index 67d9e08d1..48e20bea7 100644 --- a/packages/pipeline/src/scripts/pull_trusted_tokens.ts +++ b/packages/pipeline/src/scripts/pull_trusted_tokens.ts @@ -2,9 +2,9 @@ import 'reflect-metadata'; import { Connection, ConnectionOptions, createConnection } from 'typeorm'; import { MetamaskTrustedTokenMeta, TrustedTokenSource, ZeroExTrustedTokenMeta } from '../data_sources/trusted_tokens'; -import { TrustedToken } from '../entities'; +import { TokenMetadata } from '../entities'; import * as ormConfig from '../ormconfig'; -import { parseMetamaskTrustedTokens, parseZeroExTrustedTokens } from '../parsers/trusted_tokens'; +import { parseMetamaskTrustedTokens, parseZeroExTrustedTokens } from '../parsers/token_metadata'; import { handleError } from '../utils'; const METAMASK_TRUSTED_TOKENS_URL = @@ -25,7 +25,7 @@ let connection: Connection; async function getMetamaskTrustedTokens(): Promise { // tslint:disable-next-line console.log('Getting latest metamask trusted tokens list ...'); - const trustedTokensRepository = connection.getRepository(TrustedToken); + const trustedTokensRepository = connection.getRepository(TokenMetadata); const trustedTokensSource = new TrustedTokenSource>(METAMASK_TRUSTED_TOKENS_URL); const resp = await trustedTokensSource.getTrustedTokenMetaAsync(); const trustedTokens = parseMetamaskTrustedTokens(resp); @@ -39,7 +39,7 @@ async function getMetamaskTrustedTokens(): Promise { async function getZeroExTrustedTokens(): Promise { // tslint:disable-next-line console.log('Getting latest 0x trusted tokens list ...'); - const trustedTokensRepository = connection.getRepository(TrustedToken); + const trustedTokensRepository = connection.getRepository(TokenMetadata); const trustedTokensSource = new TrustedTokenSource(ZEROEX_TRUSTED_TOKENS_URL); const resp = await trustedTokensSource.getTrustedTokenMetaAsync(); const trustedTokens = parseZeroExTrustedTokens(resp); diff --git a/packages/pipeline/src/utils/index.ts b/packages/pipeline/src/utils/index.ts index b66ab40ab..732c259d7 100644 --- a/packages/pipeline/src/utils/index.ts +++ b/packages/pipeline/src/utils/index.ts @@ -50,6 +50,9 @@ class BigNumberTransformer implements ValueTransformer { export const bigNumberTransformer = new BigNumberTransformer(); +/** + * Returns the unix timestamp of the current hour + */ export function getHourInUnixTime(): number { const currentTime: number = Date.now(); // tslint:disable-next-line -- cgit v1.2.3