aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJake Ellowitz <jake.ellowitz@gmail.com>2018-11-20 11:11:51 +0800
committerFred Carlsen <fred@sjelfull.no>2018-12-06 19:04:25 +0800
commit72c45d9ea90237d87053fd1ddc751fc699527937 (patch)
tree140a9b62ec450fa2cfb2ac39c90265638ab2ce0c
parent75df8a0e8b375f198555f55bd6ed47af75c2b043 (diff)
downloaddexon-sol-tools-72c45d9ea90237d87053fd1ddc751fc699527937.tar
dexon-sol-tools-72c45d9ea90237d87053fd1ddc751fc699527937.tar.gz
dexon-sol-tools-72c45d9ea90237d87053fd1ddc751fc699527937.tar.bz2
dexon-sol-tools-72c45d9ea90237d87053fd1ddc751fc699527937.tar.lz
dexon-sol-tools-72c45d9ea90237d87053fd1ddc751fc699527937.tar.xz
dexon-sol-tools-72c45d9ea90237d87053fd1ddc751fc699527937.tar.zst
dexon-sol-tools-72c45d9ea90237d87053fd1ddc751fc699527937.zip
metadata and trusted sources in same raw table
-rw-r--r--packages/pipeline/migrations/1542655823221-NewMetadataAndOHLCVTables.ts22
-rw-r--r--packages/pipeline/src/entities/index.ts3
-rw-r--r--packages/pipeline/src/entities/ohlcv_external.ts18
-rw-r--r--packages/pipeline/src/entities/token_metadata.ts19
-rw-r--r--packages/pipeline/src/entities/token_on_chain_metadata.ts22
-rw-r--r--packages/pipeline/src/entities/trusted_tokens.ts7
-rw-r--r--packages/pipeline/src/ormconfig.ts4
-rw-r--r--packages/pipeline/src/parsers/token_metadata/index.ts (renamed from packages/pipeline/src/parsers/trusted_tokens/index.ts)24
-rw-r--r--packages/pipeline/src/scripts/pull_trusted_tokens.ts8
-rw-r--r--packages/pipeline/src/utils/index.ts3
10 files changed, 75 insertions, 55 deletions
diff --git a/packages/pipeline/migrations/1542655823221-NewMetadataAndOHLCVTables.ts b/packages/pipeline/migrations/1542655823221-NewMetadataAndOHLCVTables.ts
index 5c7ec6de7..8d131271b 100644
--- a/packages/pipeline/migrations/1542655823221-NewMetadataAndOHLCVTables.ts
+++ b/packages/pipeline/migrations/1542655823221-NewMetadataAndOHLCVTables.ts
@@ -2,17 +2,23 @@ import {MigrationInterface, QueryRunner} from 'typeorm';
export class NewMetadataAndOHLCVTables1542655823221 implements MigrationInterface {
+ // tslint:disable-next-line
public async up(queryRunner: QueryRunner): Promise<any> {
await queryRunner.query(`
- CREATE TABLE raw.trusted_tokens (
+ CREATE TABLE raw.token_metadata (
address VARCHAR NOT NULL,
authority VARCHAR NOT NULL,
+ decimals INT NULL,
+ symbol VARCHAR NULL,
+ name VARCHAR NULL,
PRIMARY KEY (address, authority)
);
`);
+ await queryRunner.dropTable('raw.token_on_chain_metadata');
+
await queryRunner.query(`
CREATE TABLE raw.ohlcv_external (
exchange VARCHAR NOT NULL,
@@ -36,9 +42,21 @@ export class NewMetadataAndOHLCVTables1542655823221 implements MigrationInterfac
`);
}
+ // tslint:disable-next-line
public async down(queryRunner: QueryRunner): Promise<any> {
- await queryRunner.dropTable('raw.trusted_tokens');
+ await queryRunner.query(`
+ CREATE TABLE raw.token_on_chain_metadata (
+ address VARCHAR NOT NULL,
+ decimals INT NULL,
+ symbol VARCHAR NULL,
+ name VARCHAR NULL,
+
+ PRIMARY KEY (address)
+ );
+ `);
+
+ await queryRunner.dropTable('raw.token_metadata');
await queryRunner.dropTable('raw.ohlcv_external');
}
diff --git a/packages/pipeline/src/entities/index.ts b/packages/pipeline/src/entities/index.ts
index 769efb092..a8f159a2a 100644
--- a/packages/pipeline/src/entities/index.ts
+++ b/packages/pipeline/src/entities/index.ts
@@ -9,8 +9,7 @@ export { ExchangeFillEvent } from './exchange_fill_event';
export { Relayer } from './relayer';
export { SraOrder } from './sra_order';
export { Transaction } from './transaction';
-export { TokenOnChainMetadata } from './token_on_chain_metadata';
-export { TrustedToken } from './trusted_tokens';
+export { TokenMetadata } from './token_metadata';
export { SraOrdersObservedTimeStamp, createObservedTimestampForOrder } from './sra_order_observed_timestamp';
export { OHLCVExternal } from './ohlcv_external';
diff --git a/packages/pipeline/src/entities/ohlcv_external.ts b/packages/pipeline/src/entities/ohlcv_external.ts
index 3fbb4c499..95cd4f2f5 100644
--- a/packages/pipeline/src/entities/ohlcv_external.ts
+++ b/packages/pipeline/src/entities/ohlcv_external.ts
@@ -3,18 +3,18 @@ import { Column, Entity, PrimaryColumn } from 'typeorm';
@Entity({ name: 'ohlcv_external', schema: 'raw' })
export class OHLCVExternal {
@PrimaryColumn() public exchange!: string;
- @PrimaryColumn() public from_symbol!: string;
- @PrimaryColumn() public to_symbol!: string;
- @PrimaryColumn() public start_time!: number;
- @PrimaryColumn() public end_time!: number;
-
+ @PrimaryColumn() public fromSymbol!: string;
+ @PrimaryColumn() public toSymbol!: string;
+ @PrimaryColumn() public startTime!: number;
+ @PrimaryColumn() public endTime!: number;
+
@Column() public open!: number;
@Column() public close!: number;
@Column() public low!: number;
@Column() public high!: number;
- @Column() public volume_from!: number;
- @Column() public volume_to!: number;
+ @Column() public volumeFrom!: number;
+ @Column() public volumeTo!: number;
@PrimaryColumn() public source!: string;
- @PrimaryColumn() public observed_timestamp!: number;
-} \ No newline at end of file
+ @PrimaryColumn() public observedTimestamp!: number;
+}
diff --git a/packages/pipeline/src/entities/token_metadata.ts b/packages/pipeline/src/entities/token_metadata.ts
new file mode 100644
index 000000000..4a0bceefe
--- /dev/null
+++ b/packages/pipeline/src/entities/token_metadata.ts
@@ -0,0 +1,19 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+@Entity({ name: 'token_metadata', schema: 'raw' })
+export class TokenMetadata {
+ @PrimaryColumn({ type: 'varchar', nullable: false })
+ public address!: string;
+
+ @PrimaryColumn({ type: 'varchar', nullable: false })
+ public authority!: string;
+
+ @Column({ type: 'integer', nullable: true })
+ public decimals!: number;
+
+ @Column({ type: 'varchar', nullable: true })
+ public symbol!: string;
+
+ @Column({ type: 'varchar', nullable: true })
+ public name!: string;
+}
diff --git a/packages/pipeline/src/entities/token_on_chain_metadata.ts b/packages/pipeline/src/entities/token_on_chain_metadata.ts
deleted file mode 100644
index ca1f1827b..000000000
--- a/packages/pipeline/src/entities/token_on_chain_metadata.ts
+++ /dev/null
@@ -1,22 +0,0 @@
-import { Column, Entity, PrimaryColumn } from 'typeorm';
-
-@Entity({ name: 'token_on_chain_metadata', schema: 'raw' })
-export class TokenOnChainMetadata {
- @PrimaryColumn({ type: 'nvarchar', nullable: false })
- public address!: string;
-
- @Column({ type: 'integer', nullable: false })
- public decimals!: number;
-
- @Column({ type: 'nvarchar', nullable: false })
- public symbol!: string;
-
- @Column({ type: 'nvarchar', nullable: false })
- public name!: string;
-}
-
-@Entity({ name: 'trusted_tokens', schema: 'raw' })
-export class TrustedTokens {
- @PrimaryColumn() public address!: string;
- @PrimaryColumn() public authority!: string;
-}
diff --git a/packages/pipeline/src/entities/trusted_tokens.ts b/packages/pipeline/src/entities/trusted_tokens.ts
deleted file mode 100644
index 6ec27c963..000000000
--- a/packages/pipeline/src/entities/trusted_tokens.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import { Column, Entity, PrimaryColumn } from 'typeorm';
-
-@Entity({ name: 'trusted_tokens', schema: 'raw' })
-export class TrustedToken {
- @PrimaryColumn() public address!: string;
- @PrimaryColumn() public authority!: string;
-}
diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts
index 1bd337866..e8277a439 100644
--- a/packages/pipeline/src/ormconfig.ts
+++ b/packages/pipeline/src/ormconfig.ts
@@ -8,8 +8,8 @@ import {
Relayer,
SraOrder,
SraOrdersObservedTimeStamp,
+ TokenMetadata,
Transaction,
- TrustedToken,
} from './entities';
const entities = [
@@ -20,8 +20,8 @@ const entities = [
Relayer,
SraOrder,
SraOrdersObservedTimeStamp,
+ TokenMetadata,
Transaction,
- TrustedToken,
];
const config: ConnectionOptions = {
diff --git a/packages/pipeline/src/parsers/trusted_tokens/index.ts b/packages/pipeline/src/parsers/token_metadata/index.ts
index a6d32de35..d09def0db 100644
--- a/packages/pipeline/src/parsers/trusted_tokens/index.ts
+++ b/packages/pipeline/src/parsers/token_metadata/index.ts
@@ -1,36 +1,46 @@
import * as R from 'ramda';
import { MetamaskTrustedTokenMeta, ZeroExTrustedTokenMeta } from '../../data_sources/trusted_tokens';
-import { TrustedToken } from '../../entities';
+import { TokenMetadata } from '../../entities';
/**
* Parses Metamask's trusted tokens list.
* @param rawResp raw response from the metamask json file.
*/
-export function parseMetamaskTrustedTokens(rawResp: Map<string, MetamaskTrustedTokenMeta>): TrustedToken[] {
+export function parseMetamaskTrustedTokens(rawResp: Map<string, MetamaskTrustedTokenMeta>): TokenMetadata[] {
const parsedAsObject = R.mapObjIndexed(parseMetamaskTrustedToken, rawResp);
return R.values(parsedAsObject);
}
-export function parseZeroExTrustedTokens(rawResp: ZeroExTrustedTokenMeta[]): TrustedToken[] {
+/**
+ * Parses 0x's trusted tokens list.
+ * @param rawResp raw response from the 0x json file.
+ */
+export function parseZeroExTrustedTokens(rawResp: ZeroExTrustedTokenMeta[]): TokenMetadata[] {
return R.map(parseZeroExTrustedToken, rawResp);
}
-function parseMetamaskTrustedToken(resp: MetamaskTrustedTokenMeta, address: string): TrustedToken {
+function parseMetamaskTrustedToken(resp: MetamaskTrustedTokenMeta, address: string): TokenMetadata {
- const trustedToken = new TrustedToken();
+ const trustedToken = new TokenMetadata();
trustedToken.address = address;
+ trustedToken.decimals = resp.decimals;
+ trustedToken.symbol = resp.symbol;
+ trustedToken.name = resp.name;
trustedToken.authority = 'metamask';
return trustedToken;
}
-function parseZeroExTrustedToken(resp: ZeroExTrustedTokenMeta): TrustedToken {
+function parseZeroExTrustedToken(resp: ZeroExTrustedTokenMeta): TokenMetadata {
- const trustedToken = new TrustedToken();
+ const trustedToken = new TokenMetadata();
trustedToken.address = resp.address;
+ trustedToken.decimals = resp.decimals;
+ trustedToken.symbol = resp.symbol;
+ trustedToken.name = resp.name;
trustedToken.authority = '0x';
return trustedToken;
diff --git a/packages/pipeline/src/scripts/pull_trusted_tokens.ts b/packages/pipeline/src/scripts/pull_trusted_tokens.ts
index 67d9e08d1..48e20bea7 100644
--- a/packages/pipeline/src/scripts/pull_trusted_tokens.ts
+++ b/packages/pipeline/src/scripts/pull_trusted_tokens.ts
@@ -2,9 +2,9 @@ import 'reflect-metadata';
import { Connection, ConnectionOptions, createConnection } from 'typeorm';
import { MetamaskTrustedTokenMeta, TrustedTokenSource, ZeroExTrustedTokenMeta } from '../data_sources/trusted_tokens';
-import { TrustedToken } from '../entities';
+import { TokenMetadata } from '../entities';
import * as ormConfig from '../ormconfig';
-import { parseMetamaskTrustedTokens, parseZeroExTrustedTokens } from '../parsers/trusted_tokens';
+import { parseMetamaskTrustedTokens, parseZeroExTrustedTokens } from '../parsers/token_metadata';
import { handleError } from '../utils';
const METAMASK_TRUSTED_TOKENS_URL =
@@ -25,7 +25,7 @@ let connection: Connection;
async function getMetamaskTrustedTokens(): Promise<void> {
// tslint:disable-next-line
console.log('Getting latest metamask trusted tokens list ...');
- const trustedTokensRepository = connection.getRepository(TrustedToken);
+ const trustedTokensRepository = connection.getRepository(TokenMetadata);
const trustedTokensSource = new TrustedTokenSource<Map<string, MetamaskTrustedTokenMeta>>(METAMASK_TRUSTED_TOKENS_URL);
const resp = await trustedTokensSource.getTrustedTokenMetaAsync();
const trustedTokens = parseMetamaskTrustedTokens(resp);
@@ -39,7 +39,7 @@ async function getMetamaskTrustedTokens(): Promise<void> {
async function getZeroExTrustedTokens(): Promise<void> {
// tslint:disable-next-line
console.log('Getting latest 0x trusted tokens list ...');
- const trustedTokensRepository = connection.getRepository(TrustedToken);
+ const trustedTokensRepository = connection.getRepository(TokenMetadata);
const trustedTokensSource = new TrustedTokenSource<ZeroExTrustedTokenMeta[]>(ZEROEX_TRUSTED_TOKENS_URL);
const resp = await trustedTokensSource.getTrustedTokenMetaAsync();
const trustedTokens = parseZeroExTrustedTokens(resp);
diff --git a/packages/pipeline/src/utils/index.ts b/packages/pipeline/src/utils/index.ts
index b66ab40ab..732c259d7 100644
--- a/packages/pipeline/src/utils/index.ts
+++ b/packages/pipeline/src/utils/index.ts
@@ -50,6 +50,9 @@ class BigNumberTransformer implements ValueTransformer {
export const bigNumberTransformer = new BigNumberTransformer();
+/**
+ * Returns the unix timestamp of the current hour
+ */
export function getHourInUnixTime(): number {
const currentTime: number = Date.now();
// tslint:disable-next-line