aboutsummaryrefslogtreecommitdiffstats
path: root/packages/pipeline
diff options
context:
space:
mode:
Diffstat (limited to 'packages/pipeline')
-rw-r--r--packages/pipeline/migrations/1542655823221-NewMetadataAndOHLCVTables.ts46
-rw-r--r--packages/pipeline/src/data_sources/trusted_tokens/index.ts29
-rw-r--r--packages/pipeline/src/entities/index.ts2
-rw-r--r--packages/pipeline/src/entities/ohlcv_external.ts20
-rw-r--r--packages/pipeline/src/entities/token_on_chain_metadata.ts6
-rw-r--r--packages/pipeline/src/entities/trusted_tokens.ts7
-rw-r--r--packages/pipeline/src/ormconfig.ts2
-rw-r--r--packages/pipeline/src/parsers/ohlcv_external/index.ts0
-rw-r--r--packages/pipeline/src/parsers/trusted_tokens/index.ts37
-rw-r--r--packages/pipeline/src/scripts/pull_missing_events.ts4
-rw-r--r--packages/pipeline/src/scripts/pull_trusted_tokens.ts51
-rw-r--r--packages/pipeline/src/utils/index.ts6
12 files changed, 208 insertions, 2 deletions
diff --git a/packages/pipeline/migrations/1542655823221-NewMetadataAndOHLCVTables.ts b/packages/pipeline/migrations/1542655823221-NewMetadataAndOHLCVTables.ts
new file mode 100644
index 000000000..5c7ec6de7
--- /dev/null
+++ b/packages/pipeline/migrations/1542655823221-NewMetadataAndOHLCVTables.ts
@@ -0,0 +1,46 @@
+import {MigrationInterface, QueryRunner} from 'typeorm';
+
+export class NewMetadataAndOHLCVTables1542655823221 implements MigrationInterface {
+
+ public async up(queryRunner: QueryRunner): Promise<any> {
+
+ await queryRunner.query(`
+ CREATE TABLE raw.trusted_tokens (
+ address VARCHAR NOT NULL,
+ authority VARCHAR NOT NULL,
+
+ PRIMARY KEY (address, authority)
+ );
+ `);
+
+ await queryRunner.query(`
+ CREATE TABLE raw.ohlcv_external (
+ exchange VARCHAR NOT NULL,
+ from_symbol VARCHAR NOT NULL,
+ to_symbol VARCHAR NOT NULL,
+ start_time BIGINT NOT NULL,
+ end_time BIGINT NOT NULL,
+
+ open DOUBLE PRECISION NOT NULL,
+ close DOUBLE PRECISION NOT NULL,
+ low DOUBLE PRECISION NOT NULL,
+ high DOUBLE PRECISION NOT NULL,
+ volume_from DOUBLE PRECISION NOT NULL,
+ volume_to DOUBLE PRECISION NOT NULL,
+
+ source VARCHAR NOT NULL,
+ observed_timestamp BIGINT NOT NULL,
+
+ PRIMARY KEY (exchange, from_symbol, to_symbol, start_time, end_time, source, observed_timestamp)
+ );
+ `);
+ }
+
+ public async down(queryRunner: QueryRunner): Promise<any> {
+
+ await queryRunner.dropTable('raw.trusted_tokens');
+
+ await queryRunner.dropTable('raw.ohlcv_external');
+ }
+
+}
diff --git a/packages/pipeline/src/data_sources/trusted_tokens/index.ts b/packages/pipeline/src/data_sources/trusted_tokens/index.ts
new file mode 100644
index 000000000..552739fb9
--- /dev/null
+++ b/packages/pipeline/src/data_sources/trusted_tokens/index.ts
@@ -0,0 +1,29 @@
+import axios from 'axios';
+
+export interface ZeroExTrustedTokenMeta {
+ address: string;
+ name: string;
+ symbol: string;
+ decimals: number;
+}
+
+export interface MetamaskTrustedTokenMeta {
+ address: string;
+ name: string;
+ erc20: boolean;
+ symbol: string;
+ decimals: number;
+}
+
+export class TrustedTokenSource<T> {
+ private readonly _url: string;
+
+ constructor(url: string) {
+ this._url = url;
+ }
+
+ public async getTrustedTokenMetaAsync(): Promise<T> {
+ const resp = await axios.get<T>(this._url);
+ return resp.data;
+ }
+}
diff --git a/packages/pipeline/src/entities/index.ts b/packages/pipeline/src/entities/index.ts
index d3c61bcb9..769efb092 100644
--- a/packages/pipeline/src/entities/index.ts
+++ b/packages/pipeline/src/entities/index.ts
@@ -10,6 +10,8 @@ export { Relayer } from './relayer';
export { SraOrder } from './sra_order';
export { Transaction } from './transaction';
export { TokenOnChainMetadata } from './token_on_chain_metadata';
+export { TrustedToken } from './trusted_tokens';
export { SraOrdersObservedTimeStamp, createObservedTimestampForOrder } from './sra_order_observed_timestamp';
+export { OHLCVExternal } from './ohlcv_external';
export type ExchangeEvent = ExchangeFillEvent | ExchangeCancelEvent | ExchangeCancelUpToEvent;
diff --git a/packages/pipeline/src/entities/ohlcv_external.ts b/packages/pipeline/src/entities/ohlcv_external.ts
new file mode 100644
index 000000000..3fbb4c499
--- /dev/null
+++ b/packages/pipeline/src/entities/ohlcv_external.ts
@@ -0,0 +1,20 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+@Entity({ name: 'ohlcv_external', schema: 'raw' })
+export class OHLCVExternal {
+ @PrimaryColumn() public exchange!: string;
+ @PrimaryColumn() public from_symbol!: string;
+ @PrimaryColumn() public to_symbol!: string;
+ @PrimaryColumn() public start_time!: number;
+ @PrimaryColumn() public end_time!: number;
+
+ @Column() public open!: number;
+ @Column() public close!: number;
+ @Column() public low!: number;
+ @Column() public high!: number;
+ @Column() public volume_from!: number;
+ @Column() public volume_to!: number;
+
+ @PrimaryColumn() public source!: string;
+ @PrimaryColumn() public observed_timestamp!: number;
+} \ No newline at end of file
diff --git a/packages/pipeline/src/entities/token_on_chain_metadata.ts b/packages/pipeline/src/entities/token_on_chain_metadata.ts
index 325d28e91..ca1f1827b 100644
--- a/packages/pipeline/src/entities/token_on_chain_metadata.ts
+++ b/packages/pipeline/src/entities/token_on_chain_metadata.ts
@@ -14,3 +14,9 @@ export class TokenOnChainMetadata {
@Column({ type: 'nvarchar', nullable: false })
public name!: string;
}
+
+@Entity({ name: 'trusted_tokens', schema: 'raw' })
+export class TrustedTokens {
+ @PrimaryColumn() public address!: string;
+ @PrimaryColumn() public authority!: string;
+}
diff --git a/packages/pipeline/src/entities/trusted_tokens.ts b/packages/pipeline/src/entities/trusted_tokens.ts
new file mode 100644
index 000000000..6ec27c963
--- /dev/null
+++ b/packages/pipeline/src/entities/trusted_tokens.ts
@@ -0,0 +1,7 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+@Entity({ name: 'trusted_tokens', schema: 'raw' })
+export class TrustedToken {
+ @PrimaryColumn() public address!: string;
+ @PrimaryColumn() public authority!: string;
+}
diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts
index 64b5b1827..1bd337866 100644
--- a/packages/pipeline/src/ormconfig.ts
+++ b/packages/pipeline/src/ormconfig.ts
@@ -9,6 +9,7 @@ import {
SraOrder,
SraOrdersObservedTimeStamp,
Transaction,
+ TrustedToken,
} from './entities';
const entities = [
@@ -20,6 +21,7 @@ const entities = [
SraOrder,
SraOrdersObservedTimeStamp,
Transaction,
+ TrustedToken,
];
const config: ConnectionOptions = {
diff --git a/packages/pipeline/src/parsers/ohlcv_external/index.ts b/packages/pipeline/src/parsers/ohlcv_external/index.ts
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/packages/pipeline/src/parsers/ohlcv_external/index.ts
diff --git a/packages/pipeline/src/parsers/trusted_tokens/index.ts b/packages/pipeline/src/parsers/trusted_tokens/index.ts
new file mode 100644
index 000000000..a6d32de35
--- /dev/null
+++ b/packages/pipeline/src/parsers/trusted_tokens/index.ts
@@ -0,0 +1,37 @@
+import * as R from 'ramda';
+
+import { MetamaskTrustedTokenMeta, ZeroExTrustedTokenMeta } from '../../data_sources/trusted_tokens';
+import { TrustedToken } from '../../entities';
+
+/**
+ * Parses Metamask's trusted tokens list.
+ * @param rawResp raw response from the metamask json file.
+ */
+export function parseMetamaskTrustedTokens(rawResp: Map<string, MetamaskTrustedTokenMeta>): TrustedToken[] {
+ const parsedAsObject = R.mapObjIndexed(parseMetamaskTrustedToken, rawResp);
+ return R.values(parsedAsObject);
+}
+
+export function parseZeroExTrustedTokens(rawResp: ZeroExTrustedTokenMeta[]): TrustedToken[] {
+ return R.map(parseZeroExTrustedToken, rawResp);
+}
+
+function parseMetamaskTrustedToken(resp: MetamaskTrustedTokenMeta, address: string): TrustedToken {
+
+ const trustedToken = new TrustedToken();
+
+ trustedToken.address = address;
+ trustedToken.authority = 'metamask';
+
+ return trustedToken;
+}
+
+function parseZeroExTrustedToken(resp: ZeroExTrustedTokenMeta): TrustedToken {
+
+ const trustedToken = new TrustedToken();
+
+ trustedToken.address = resp.address;
+ trustedToken.authority = '0x';
+
+ return trustedToken;
+}
diff --git a/packages/pipeline/src/scripts/pull_missing_events.ts b/packages/pipeline/src/scripts/pull_missing_events.ts
index 0b7f6287f..68cabe3de 100644
--- a/packages/pipeline/src/scripts/pull_missing_events.ts
+++ b/packages/pipeline/src/scripts/pull_missing_events.ts
@@ -64,7 +64,7 @@ async function getCancelUpToEventsAsync(eventsSource: ExchangeEventsSource): Pro
await saveEventsAsync(startBlock === EXCHANGE_START_BLOCK, repository, events);
}
-const tabelNameRegex = /^[a-zA-Z_]*$/;
+const tableNameRegex = /^[a-zA-Z_]*$/;
async function getStartBlockAsync<T extends ExchangeEvent>(repository: Repository<T>): Promise<number> {
const fillEventCount = await repository.count();
@@ -73,7 +73,7 @@ async function getStartBlockAsync<T extends ExchangeEvent>(repository: Repositor
return EXCHANGE_START_BLOCK;
}
const tableName = repository.metadata.tableName;
- if (!tabelNameRegex.test(tableName)) {
+ if (!tableNameRegex.test(tableName)) {
throw new Error('Unexpected special character in table name: ' + tableName);
}
const queryResult = await connection.query(
diff --git a/packages/pipeline/src/scripts/pull_trusted_tokens.ts b/packages/pipeline/src/scripts/pull_trusted_tokens.ts
new file mode 100644
index 000000000..67d9e08d1
--- /dev/null
+++ b/packages/pipeline/src/scripts/pull_trusted_tokens.ts
@@ -0,0 +1,51 @@
+import 'reflect-metadata';
+import { Connection, ConnectionOptions, createConnection } from 'typeorm';
+
+import { MetamaskTrustedTokenMeta, TrustedTokenSource, ZeroExTrustedTokenMeta } from '../data_sources/trusted_tokens';
+import { TrustedToken } from '../entities';
+import * as ormConfig from '../ormconfig';
+import { parseMetamaskTrustedTokens, parseZeroExTrustedTokens } from '../parsers/trusted_tokens';
+import { handleError } from '../utils';
+
+const METAMASK_TRUSTED_TOKENS_URL =
+ 'https://raw.githubusercontent.com/MetaMask/eth-contract-metadata/master/contract-map.json';
+
+const ZEROEX_TRUSTED_TOKENS_URL =
+ 'https://website-api.0xproject.com/tokens';
+
+let connection: Connection;
+
+(async () => {
+ connection = await createConnection(ormConfig as ConnectionOptions);
+ await getMetamaskTrustedTokens();
+ await getZeroExTrustedTokens();
+ process.exit(0);
+})().catch(handleError);
+
+async function getMetamaskTrustedTokens(): Promise<void> {
+ // tslint:disable-next-line
+ console.log('Getting latest metamask trusted tokens list ...');
+ const trustedTokensRepository = connection.getRepository(TrustedToken);
+ const trustedTokensSource = new TrustedTokenSource<Map<string, MetamaskTrustedTokenMeta>>(METAMASK_TRUSTED_TOKENS_URL);
+ const resp = await trustedTokensSource.getTrustedTokenMetaAsync();
+ const trustedTokens = parseMetamaskTrustedTokens(resp);
+ // tslint:disable-next-line
+ console.log('Saving metamask trusted tokens list');
+ await trustedTokensRepository.save(trustedTokens);
+ // tslint:disable-next-line
+ console.log('Done saving metamask trusted tokens.')
+}
+
+async function getZeroExTrustedTokens(): Promise<void> {
+ // tslint:disable-next-line
+ console.log('Getting latest 0x trusted tokens list ...');
+ const trustedTokensRepository = connection.getRepository(TrustedToken);
+ const trustedTokensSource = new TrustedTokenSource<ZeroExTrustedTokenMeta[]>(ZEROEX_TRUSTED_TOKENS_URL);
+ const resp = await trustedTokensSource.getTrustedTokenMetaAsync();
+ const trustedTokens = parseZeroExTrustedTokens(resp);
+ // tslint:disable-next-line
+ console.log('Saving metamask trusted tokens list');
+ await trustedTokensRepository.save(trustedTokens);
+ // tslint:disable-next-line
+ console.log('Done saving metamask trusted tokens.');
+}
diff --git a/packages/pipeline/src/utils/index.ts b/packages/pipeline/src/utils/index.ts
index 78aa89374..b66ab40ab 100644
--- a/packages/pipeline/src/utils/index.ts
+++ b/packages/pipeline/src/utils/index.ts
@@ -49,3 +49,9 @@ class BigNumberTransformer implements ValueTransformer {
}
export const bigNumberTransformer = new BigNumberTransformer();
+
+export function getHourInUnixTime(): number {
+ const currentTime: number = Date.now();
+ // tslint:disable-next-line
+ return currentTime - (currentTime % (3600 * 1000));
+}