Pull token metadata re trusted tokens

12 files changed, 208 insertions, 2 deletions

diff --git a/packages/pipeline/migrations/1542655823221-NewMetadataAndOHLCVTables.ts b/packages/pipeline/migrations/1542655823221-NewMetadataAndOHLCVTables.ts
new file mode 100644
index 000000000..5c7ec6de7
--- /dev/null
+++ b/packages/pipeline/migrations/1542655823221-NewMetadataAndOHLCVTables.ts
@@ -0,0 +1,46 @@
+import {MigrationInterface, QueryRunner} from 'typeorm';
+
+export class NewMetadataAndOHLCVTables1542655823221 implements MigrationInterface {
+
+    public async up(queryRunner: QueryRunner): Promise<any> {
+
+        await queryRunner.query(`
+            CREATE TABLE raw.trusted_tokens (
+                address VARCHAR NOT NULL,
+                authority VARCHAR NOT NULL,
+
+                PRIMARY KEY (address, authority)
+            );
+        `);
+
+        await queryRunner.query(`
+            CREATE TABLE raw.ohlcv_external (
+                exchange VARCHAR NOT NULL,
+                from_symbol VARCHAR NOT NULL,
+                to_symbol VARCHAR NOT NULL,
+                start_time BIGINT NOT NULL,
+                end_time BIGINT NOT NULL,
+
+                open DOUBLE PRECISION NOT NULL,
+                close DOUBLE PRECISION NOT NULL,
+                low DOUBLE PRECISION NOT NULL,
+                high DOUBLE PRECISION NOT NULL,
+                volume_from DOUBLE PRECISION NOT NULL,
+                volume_to DOUBLE PRECISION NOT NULL,
+
+                source VARCHAR NOT NULL,
+                observed_timestamp BIGINT NOT NULL,
+
+                PRIMARY KEY (exchange, from_symbol, to_symbol, start_time, end_time, source, observed_timestamp)
+            );
+        `);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<any> {
+
+        await queryRunner.dropTable('raw.trusted_tokens');
+
+        await queryRunner.dropTable('raw.ohlcv_external');
+    }
+
+}
diff --git a/packages/pipeline/src/data_sources/trusted_tokens/index.ts b/packages/pipeline/src/data_sources/trusted_tokens/index.ts
new file mode 100644
index 000000000..552739fb9
--- /dev/null
+++ b/packages/pipeline/src/data_sources/trusted_tokens/index.ts
@@ -0,0 +1,29 @@
+import axios from 'axios';
+
+export interface ZeroExTrustedTokenMeta {
+    address: string;
+    name: string;
+    symbol: string;
+    decimals: number;
+}
+
+export interface MetamaskTrustedTokenMeta {
+    address: string;
+    name: string;
+    erc20: boolean;
+    symbol: string;
+    decimals: number;
+}
+
+export class TrustedTokenSource<T> {
+    private readonly _url: string;
+
+    constructor(url: string) {
+        this._url = url;
+    }
+
+    public async getTrustedTokenMetaAsync(): Promise<T> {
+        const resp = await axios.get<T>(this._url);
+        return resp.data;
+    }
+}
diff --git a/packages/pipeline/src/entities/index.ts b/packages/pipeline/src/entities/index.ts
index d3c61bcb9..769efb092 100644
--- a/packages/pipeline/src/entities/index.ts
+++ b/packages/pipeline/src/entities/index.ts
@@ -10,6 +10,8 @@ export { Relayer } from './relayer';
 export { SraOrder } from './sra_order';
 export { Transaction } from './transaction';
 export { TokenOnChainMetadata } from './token_on_chain_metadata';
+export { TrustedToken } from './trusted_tokens';
 export { SraOrdersObservedTimeStamp, createObservedTimestampForOrder } from './sra_order_observed_timestamp';
+export { OHLCVExternal } from './ohlcv_external';
 
 export type ExchangeEvent = ExchangeFillEvent | ExchangeCancelEvent | ExchangeCancelUpToEvent;
diff --git a/packages/pipeline/src/entities/ohlcv_external.ts b/packages/pipeline/src/entities/ohlcv_external.ts
new file mode 100644
index 000000000..3fbb4c499
--- /dev/null
+++ b/packages/pipeline/src/entities/ohlcv_external.ts
@@ -0,0 +1,20 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+@Entity({ name: 'ohlcv_external', schema: 'raw' })
+export class OHLCVExternal {
+    @PrimaryColumn() public exchange!: string;
+    @PrimaryColumn() public from_symbol!: string;
+    @PrimaryColumn() public to_symbol!: string;
+    @PrimaryColumn() public start_time!: number;
+    @PrimaryColumn() public end_time!: number;
+    
+    @Column() public open!: number;
+    @Column() public close!: number;
+    @Column() public low!: number;
+    @Column() public high!: number;
+    @Column() public volume_from!: number;
+    @Column() public volume_to!: number;
+
+    @PrimaryColumn() public source!: string;
+    @PrimaryColumn() public observed_timestamp!: number;
+}
\ No newline at end of file
diff --git a/packages/pipeline/src/entities/token_on_chain_metadata.ts b/packages/pipeline/src/entities/token_on_chain_metadata.ts
index 325d28e91..ca1f1827b 100644
--- a/packages/pipeline/src/entities/token_on_chain_metadata.ts
+++ b/packages/pipeline/src/entities/token_on_chain_metadata.ts
@@ -14,3 +14,9 @@ export class TokenOnChainMetadata {
     @Column({ type: 'nvarchar', nullable: false })
     public name!: string;
 }
+
+@Entity({ name: 'trusted_tokens', schema: 'raw' })
+export class TrustedTokens {
+    @PrimaryColumn() public address!: string;
+    @PrimaryColumn() public authority!: string;
+}
diff --git a/packages/pipeline/src/entities/trusted_tokens.ts b/packages/pipeline/src/entities/trusted_tokens.ts
new file mode 100644
index 000000000..6ec27c963
--- /dev/null
+++ b/packages/pipeline/src/entities/trusted_tokens.ts
@@ -0,0 +1,7 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+@Entity({ name: 'trusted_tokens', schema: 'raw' })
+export class TrustedToken {
+    @PrimaryColumn() public address!: string;
+    @PrimaryColumn() public authority!: string;
+}
diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts
index 64b5b1827..1bd337866 100644
--- a/packages/pipeline/src/ormconfig.ts
+++ b/packages/pipeline/src/ormconfig.ts
@@ -9,6 +9,7 @@ import {
     SraOrder,
     SraOrdersObservedTimeStamp,
     Transaction,
+    TrustedToken,
 } from './entities';
 
 const entities = [
@@ -20,6 +21,7 @@ const entities = [
     SraOrder,
     SraOrdersObservedTimeStamp,
     Transaction,
+    TrustedToken,
 ];
 
 const config: ConnectionOptions = {
diff --git a/packages/pipeline/src/parsers/ohlcv_external/index.ts b/packages/pipeline/src/parsers/ohlcv_external/index.ts
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/packages/pipeline/src/parsers/ohlcv_external/index.ts
diff --git a/packages/pipeline/src/parsers/trusted_tokens/index.ts b/packages/pipeline/src/parsers/trusted_tokens/index.ts
new file mode 100644
index 000000000..a6d32de35
--- /dev/null
+++ b/packages/pipeline/src/parsers/trusted_tokens/index.ts
@@ -0,0 +1,37 @@
+import * as R from 'ramda';
+
+import { MetamaskTrustedTokenMeta, ZeroExTrustedTokenMeta } from '../../data_sources/trusted_tokens';
+import { TrustedToken } from '../../entities';
+
+/**
+ * Parses Metamask's trusted tokens list.
+ * @param rawResp raw response from the metamask json file.
+ */
+export function parseMetamaskTrustedTokens(rawResp: Map<string, MetamaskTrustedTokenMeta>): TrustedToken[] {
+    const parsedAsObject = R.mapObjIndexed(parseMetamaskTrustedToken, rawResp);
+    return R.values(parsedAsObject);
+}
+
+export function parseZeroExTrustedTokens(rawResp: ZeroExTrustedTokenMeta[]): TrustedToken[] {
+    return R.map(parseZeroExTrustedToken, rawResp);
+}
+
+function parseMetamaskTrustedToken(resp: MetamaskTrustedTokenMeta, address: string): TrustedToken {
+
+    const trustedToken = new TrustedToken();
+
+    trustedToken.address = address;
+    trustedToken.authority = 'metamask';
+
+    return trustedToken;
+}
+
+function parseZeroExTrustedToken(resp: ZeroExTrustedTokenMeta): TrustedToken {
+
+    const trustedToken = new TrustedToken();
+
+    trustedToken.address = resp.address;
+    trustedToken.authority = '0x';
+
+    return trustedToken;
+}
diff --git a/packages/pipeline/src/scripts/pull_missing_events.ts b/packages/pipeline/src/scripts/pull_missing_events.ts
index 0b7f6287f..68cabe3de 100644
--- a/packages/pipeline/src/scripts/pull_missing_events.ts
+++ b/packages/pipeline/src/scripts/pull_missing_events.ts
@@ -64,7 +64,7 @@ async function getCancelUpToEventsAsync(eventsSource: ExchangeEventsSource): Pro
     await saveEventsAsync(startBlock === EXCHANGE_START_BLOCK, repository, events);
 }
 
-const tabelNameRegex = /^[a-zA-Z_]*$/;
+const tableNameRegex = /^[a-zA-Z_]*$/;
 
 async function getStartBlockAsync<T extends ExchangeEvent>(repository: Repository<T>): Promise<number> {
     const fillEventCount = await repository.count();
@@ -73,7 +73,7 @@ async function getStartBlockAsync<T extends ExchangeEvent>(repository: Repositor
         return EXCHANGE_START_BLOCK;
     }
     const tableName = repository.metadata.tableName;
-    if (!tabelNameRegex.test(tableName)) {
+    if (!tableNameRegex.test(tableName)) {
         throw new Error('Unexpected special character in table name: ' + tableName);
     }
     const queryResult = await connection.query(
diff --git a/packages/pipeline/src/scripts/pull_trusted_tokens.ts b/packages/pipeline/src/scripts/pull_trusted_tokens.ts
new file mode 100644
index 000000000..67d9e08d1
--- /dev/null
+++ b/packages/pipeline/src/scripts/pull_trusted_tokens.ts
@@ -0,0 +1,51 @@
+import 'reflect-metadata';
+import { Connection, ConnectionOptions, createConnection } from 'typeorm';
+
+import { MetamaskTrustedTokenMeta, TrustedTokenSource, ZeroExTrustedTokenMeta } from '../data_sources/trusted_tokens';
+import { TrustedToken } from '../entities';
+import * as ormConfig from '../ormconfig';
+import { parseMetamaskTrustedTokens, parseZeroExTrustedTokens } from '../parsers/trusted_tokens';
+import { handleError } from '../utils';
+
+const METAMASK_TRUSTED_TOKENS_URL =
+    'https://raw.githubusercontent.com/MetaMask/eth-contract-metadata/master/contract-map.json';
+
+const ZEROEX_TRUSTED_TOKENS_URL =
+    'https://website-api.0xproject.com/tokens';
+
+let connection: Connection;
+
+(async () => {
+    connection = await createConnection(ormConfig as ConnectionOptions);
+    await getMetamaskTrustedTokens();
+    await getZeroExTrustedTokens();
+    process.exit(0);
+})().catch(handleError);
+
+async function getMetamaskTrustedTokens(): Promise<void> {
+    // tslint:disable-next-line
+    console.log('Getting latest metamask trusted tokens list ...');
+    const trustedTokensRepository = connection.getRepository(TrustedToken);
+    const trustedTokensSource = new TrustedTokenSource<Map<string, MetamaskTrustedTokenMeta>>(METAMASK_TRUSTED_TOKENS_URL);
+    const resp = await trustedTokensSource.getTrustedTokenMetaAsync();
+    const trustedTokens = parseMetamaskTrustedTokens(resp);
+    // tslint:disable-next-line
+    console.log('Saving metamask trusted tokens list');
+    await trustedTokensRepository.save(trustedTokens);
+    // tslint:disable-next-line
+    console.log('Done saving metamask trusted tokens.')
+}
+
+async function getZeroExTrustedTokens(): Promise<void> {
+    // tslint:disable-next-line
+    console.log('Getting latest 0x trusted tokens list ...');
+    const trustedTokensRepository = connection.getRepository(TrustedToken);
+    const trustedTokensSource = new TrustedTokenSource<ZeroExTrustedTokenMeta[]>(ZEROEX_TRUSTED_TOKENS_URL);
+    const resp = await trustedTokensSource.getTrustedTokenMetaAsync();
+    const trustedTokens = parseZeroExTrustedTokens(resp);
+    // tslint:disable-next-line
+    console.log('Saving metamask trusted tokens list');
+    await trustedTokensRepository.save(trustedTokens);
+    // tslint:disable-next-line
+    console.log('Done saving metamask trusted tokens.');
+}
diff --git a/packages/pipeline/src/utils/index.ts b/packages/pipeline/src/utils/index.ts
index 78aa89374..b66ab40ab 100644
--- a/packages/pipeline/src/utils/index.ts
+++ b/packages/pipeline/src/utils/index.ts
@@ -49,3 +49,9 @@ class BigNumberTransformer implements ValueTransformer {
 }
 
 export const bigNumberTransformer = new BigNumberTransformer();
+
+export function getHourInUnixTime(): number {
+    const currentTime: number = Date.now();
+    // tslint:disable-next-line
+    return currentTime - (currentTime % (3600 * 1000));
+}