metadata and trusted sources in same raw table

10 files changed, 75 insertions, 55 deletions

diff --git a/packages/pipeline/migrations/1542655823221-NewMetadataAndOHLCVTables.ts b/packages/pipeline/migrations/1542655823221-NewMetadataAndOHLCVTables.ts
index 5c7ec6de7..8d131271b 100644
--- a/packages/pipeline/migrations/1542655823221-NewMetadataAndOHLCVTables.ts
+++ b/packages/pipeline/migrations/1542655823221-NewMetadataAndOHLCVTables.ts
@@ -2,17 +2,23 @@ import {MigrationInterface, QueryRunner} from 'typeorm';
 
 export class NewMetadataAndOHLCVTables1542655823221 implements MigrationInterface {
 
+    // tslint:disable-next-line
     public async up(queryRunner: QueryRunner): Promise<any> {
 
         await queryRunner.query(`
-            CREATE TABLE raw.trusted_tokens (
+            CREATE TABLE raw.token_metadata (
                 address VARCHAR NOT NULL,
                 authority VARCHAR NOT NULL,
+                decimals INT NULL,
+                symbol VARCHAR NULL,
+                name VARCHAR NULL,
 
                 PRIMARY KEY (address, authority)
             );
         `);
 
+        await queryRunner.dropTable('raw.token_on_chain_metadata');
+
         await queryRunner.query(`
             CREATE TABLE raw.ohlcv_external (
                 exchange VARCHAR NOT NULL,
@@ -36,9 +42,21 @@ export class NewMetadataAndOHLCVTables1542655823221 implements MigrationInterfac
         `);
     }
 
+    // tslint:disable-next-line
     public async down(queryRunner: QueryRunner): Promise<any> {
 
-        await queryRunner.dropTable('raw.trusted_tokens');
+        await queryRunner.query(`
+            CREATE TABLE raw.token_on_chain_metadata (
+                address VARCHAR NOT NULL,
+                decimals INT NULL,
+                symbol VARCHAR NULL,
+                name VARCHAR NULL,
+
+                PRIMARY KEY (address)
+            );
+        `);
+
+        await queryRunner.dropTable('raw.token_metadata');
 
         await queryRunner.dropTable('raw.ohlcv_external');
     }
diff --git a/packages/pipeline/src/entities/index.ts b/packages/pipeline/src/entities/index.ts
index 769efb092..a8f159a2a 100644
--- a/packages/pipeline/src/entities/index.ts
+++ b/packages/pipeline/src/entities/index.ts
@@ -9,8 +9,7 @@ export { ExchangeFillEvent } from './exchange_fill_event';
 export { Relayer } from './relayer';
 export { SraOrder } from './sra_order';
 export { Transaction } from './transaction';
-export { TokenOnChainMetadata } from './token_on_chain_metadata';
-export { TrustedToken } from './trusted_tokens';
+export { TokenMetadata } from './token_metadata';
 export { SraOrdersObservedTimeStamp, createObservedTimestampForOrder } from './sra_order_observed_timestamp';
 export { OHLCVExternal } from './ohlcv_external';
 
diff --git a/packages/pipeline/src/entities/ohlcv_external.ts b/packages/pipeline/src/entities/ohlcv_external.ts
index 3fbb4c499..95cd4f2f5 100644
--- a/packages/pipeline/src/entities/ohlcv_external.ts
+++ b/packages/pipeline/src/entities/ohlcv_external.ts
@@ -3,18 +3,18 @@ import { Column, Entity, PrimaryColumn } from 'typeorm';
 @Entity({ name: 'ohlcv_external', schema: 'raw' })
 export class OHLCVExternal {
     @PrimaryColumn() public exchange!: string;
-    @PrimaryColumn() public from_symbol!: string;
-    @PrimaryColumn() public to_symbol!: string;
-    @PrimaryColumn() public start_time!: number;
-    @PrimaryColumn() public end_time!: number;
-    
+    @PrimaryColumn() public fromSymbol!: string;
+    @PrimaryColumn() public toSymbol!: string;
+    @PrimaryColumn() public startTime!: number;
+    @PrimaryColumn() public endTime!: number;
+
     @Column() public open!: number;
     @Column() public close!: number;
     @Column() public low!: number;
     @Column() public high!: number;
-    @Column() public volume_from!: number;
-    @Column() public volume_to!: number;
+    @Column() public volumeFrom!: number;
+    @Column() public volumeTo!: number;
 
     @PrimaryColumn() public source!: string;
-    @PrimaryColumn() public observed_timestamp!: number;
-}
\ No newline at end of file
+    @PrimaryColumn() public observedTimestamp!: number;
+}
diff --git a/packages/pipeline/src/entities/token_metadata.ts b/packages/pipeline/src/entities/token_metadata.ts
new file mode 100644
index 000000000..4a0bceefe
--- /dev/null
+++ b/packages/pipeline/src/entities/token_metadata.ts
@@ -0,0 +1,19 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+@Entity({ name: 'token_metadata', schema: 'raw' })
+export class TokenMetadata {
+    @PrimaryColumn({ type: 'varchar', nullable: false })
+    public address!: string;
+
+    @PrimaryColumn({ type: 'varchar', nullable: false })
+    public authority!: string;
+
+    @Column({ type: 'integer', nullable: true })
+    public decimals!: number;
+
+    @Column({ type: 'varchar', nullable: true })
+    public symbol!: string;
+
+    @Column({ type: 'varchar', nullable: true })
+    public name!: string;
+}
diff --git a/packages/pipeline/src/entities/token_on_chain_metadata.ts b/packages/pipeline/src/entities/token_on_chain_metadata.ts
deleted file mode 100644
index ca1f1827b..000000000
--- a/packages/pipeline/src/entities/token_on_chain_metadata.ts
+++ /dev/null
@@ -1,22 +0,0 @@
-import { Column, Entity, PrimaryColumn } from 'typeorm';
-
-@Entity({ name: 'token_on_chain_metadata', schema: 'raw' })
-export class TokenOnChainMetadata {
-    @PrimaryColumn({ type: 'nvarchar', nullable: false })
-    public address!: string;
-
-    @Column({ type: 'integer', nullable: false })
-    public decimals!: number;
-
-    @Column({ type: 'nvarchar', nullable: false })
-    public symbol!: string;
-
-    @Column({ type: 'nvarchar', nullable: false })
-    public name!: string;
-}
-
-@Entity({ name: 'trusted_tokens', schema: 'raw' })
-export class TrustedTokens {
-    @PrimaryColumn() public address!: string;
-    @PrimaryColumn() public authority!: string;
-}
diff --git a/packages/pipeline/src/entities/trusted_tokens.ts b/packages/pipeline/src/entities/trusted_tokens.ts
deleted file mode 100644
index 6ec27c963..000000000
--- a/packages/pipeline/src/entities/trusted_tokens.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import { Column, Entity, PrimaryColumn } from 'typeorm';
-
-@Entity({ name: 'trusted_tokens', schema: 'raw' })
-export class TrustedToken {
-    @PrimaryColumn() public address!: string;
-    @PrimaryColumn() public authority!: string;
-}
diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts
index 1bd337866..e8277a439 100644
--- a/packages/pipeline/src/ormconfig.ts
+++ b/packages/pipeline/src/ormconfig.ts
@@ -8,8 +8,8 @@ import {
     Relayer,
     SraOrder,
     SraOrdersObservedTimeStamp,
+    TokenMetadata,
     Transaction,
-    TrustedToken,
 } from './entities';
 
 const entities = [
@@ -20,8 +20,8 @@ const entities = [
     Relayer,
     SraOrder,
     SraOrdersObservedTimeStamp,
+    TokenMetadata,
     Transaction,
-    TrustedToken,
 ];
 
 const config: ConnectionOptions = {
diff --git a/packages/pipeline/src/parsers/trusted_tokens/index.ts b/packages/pipeline/src/parsers/token_metadata/index.ts
index a6d32de35..d09def0db 100644
--- a/packages/pipeline/src/parsers/trusted_tokens/index.ts
+++ b/packages/pipeline/src/parsers/token_metadata/index.ts
@@ -1,36 +1,46 @@
 import * as R from 'ramda';
 
 import { MetamaskTrustedTokenMeta, ZeroExTrustedTokenMeta } from '../../data_sources/trusted_tokens';
-import { TrustedToken } from '../../entities';
+import { TokenMetadata } from '../../entities';
 
 /**
  * Parses Metamask's trusted tokens list.
  * @param rawResp raw response from the metamask json file.
  */
-export function parseMetamaskTrustedTokens(rawResp: Map<string, MetamaskTrustedTokenMeta>): TrustedToken[] {
+export function parseMetamaskTrustedTokens(rawResp: Map<string, MetamaskTrustedTokenMeta>): TokenMetadata[] {
     const parsedAsObject = R.mapObjIndexed(parseMetamaskTrustedToken, rawResp);
     return R.values(parsedAsObject);
 }
 
-export function parseZeroExTrustedTokens(rawResp: ZeroExTrustedTokenMeta[]): TrustedToken[] {
+/**
+ * Parses 0x's trusted tokens list.
+ * @param rawResp raw response from the 0x json file.
+ */
+export function parseZeroExTrustedTokens(rawResp: ZeroExTrustedTokenMeta[]): TokenMetadata[] {
     return R.map(parseZeroExTrustedToken, rawResp);
 }
 
-function parseMetamaskTrustedToken(resp: MetamaskTrustedTokenMeta, address: string): TrustedToken {
+function parseMetamaskTrustedToken(resp: MetamaskTrustedTokenMeta, address: string): TokenMetadata {
 
-    const trustedToken = new TrustedToken();
+    const trustedToken = new TokenMetadata();
 
     trustedToken.address = address;
+    trustedToken.decimals = resp.decimals;
+    trustedToken.symbol = resp.symbol;
+    trustedToken.name = resp.name;
     trustedToken.authority = 'metamask';
 
     return trustedToken;
 }
 
-function parseZeroExTrustedToken(resp: ZeroExTrustedTokenMeta): TrustedToken {
+function parseZeroExTrustedToken(resp: ZeroExTrustedTokenMeta): TokenMetadata {
 
-    const trustedToken = new TrustedToken();
+    const trustedToken = new TokenMetadata();
 
     trustedToken.address = resp.address;
+    trustedToken.decimals = resp.decimals;
+    trustedToken.symbol = resp.symbol;
+    trustedToken.name = resp.name;
     trustedToken.authority = '0x';
 
     return trustedToken;
diff --git a/packages/pipeline/src/scripts/pull_trusted_tokens.ts b/packages/pipeline/src/scripts/pull_trusted_tokens.ts
index 67d9e08d1..48e20bea7 100644
--- a/packages/pipeline/src/scripts/pull_trusted_tokens.ts
+++ b/packages/pipeline/src/scripts/pull_trusted_tokens.ts
@@ -2,9 +2,9 @@ import 'reflect-metadata';
 import { Connection, ConnectionOptions, createConnection } from 'typeorm';
 
 import { MetamaskTrustedTokenMeta, TrustedTokenSource, ZeroExTrustedTokenMeta } from '../data_sources/trusted_tokens';
-import { TrustedToken } from '../entities';
+import { TokenMetadata } from '../entities';
 import * as ormConfig from '../ormconfig';
-import { parseMetamaskTrustedTokens, parseZeroExTrustedTokens } from '../parsers/trusted_tokens';
+import { parseMetamaskTrustedTokens, parseZeroExTrustedTokens } from '../parsers/token_metadata';
 import { handleError } from '../utils';
 
 const METAMASK_TRUSTED_TOKENS_URL =
@@ -25,7 +25,7 @@ let connection: Connection;
 async function getMetamaskTrustedTokens(): Promise<void> {
     // tslint:disable-next-line
     console.log('Getting latest metamask trusted tokens list ...');
-    const trustedTokensRepository = connection.getRepository(TrustedToken);
+    const trustedTokensRepository = connection.getRepository(TokenMetadata);
     const trustedTokensSource = new TrustedTokenSource<Map<string, MetamaskTrustedTokenMeta>>(METAMASK_TRUSTED_TOKENS_URL);
     const resp = await trustedTokensSource.getTrustedTokenMetaAsync();
     const trustedTokens = parseMetamaskTrustedTokens(resp);
@@ -39,7 +39,7 @@ async function getMetamaskTrustedTokens(): Promise<void> {
 async function getZeroExTrustedTokens(): Promise<void> {
     // tslint:disable-next-line
     console.log('Getting latest 0x trusted tokens list ...');
-    const trustedTokensRepository = connection.getRepository(TrustedToken);
+    const trustedTokensRepository = connection.getRepository(TokenMetadata);
     const trustedTokensSource = new TrustedTokenSource<ZeroExTrustedTokenMeta[]>(ZEROEX_TRUSTED_TOKENS_URL);
     const resp = await trustedTokensSource.getTrustedTokenMetaAsync();
     const trustedTokens = parseZeroExTrustedTokens(resp);
diff --git a/packages/pipeline/src/utils/index.ts b/packages/pipeline/src/utils/index.ts
index b66ab40ab..732c259d7 100644
--- a/packages/pipeline/src/utils/index.ts
+++ b/packages/pipeline/src/utils/index.ts
@@ -50,6 +50,9 @@ class BigNumberTransformer implements ValueTransformer {
 
 export const bigNumberTransformer = new BigNumberTransformer();
 
+/**
+ * Returns the unix timestamp of the current hour
+ */
 export function getHourInUnixTime(): number {
     const currentTime: number = Date.now();
     // tslint:disable-next-line