Merge branch 'development' into feature/contracts-monorepo-7

13 files changed, 441 insertions, 0 deletions

diff --git a/packages/pipeline/src/entities/block.ts b/packages/pipeline/src/entities/block.ts
new file mode 100644
index 000000000..398946622
--- /dev/null
+++ b/packages/pipeline/src/entities/block.ts
@@ -0,0 +1,13 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+import { numberToBigIntTransformer } from '../utils';
+
+@Entity({ name: 'blocks', schema: 'raw' })
+export class Block {
+    @PrimaryColumn() public hash!: string;
+    @PrimaryColumn({ transformer: numberToBigIntTransformer })
+    public number!: number;
+
+    @Column({ name: 'timestamp', transformer: numberToBigIntTransformer })
+    public timestamp!: number;
+}
diff --git a/packages/pipeline/src/entities/dex_trade.ts b/packages/pipeline/src/entities/dex_trade.ts
new file mode 100644
index 000000000..9d288cb51
--- /dev/null
+++ b/packages/pipeline/src/entities/dex_trade.ts
@@ -0,0 +1,54 @@
+import { BigNumber } from '@0x/utils';
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+import { bigNumberTransformer, numberToBigIntTransformer } from '../utils';
+
+@Entity({ name: 'dex_trades', schema: 'raw' })
+export class DexTrade {
+    @PrimaryColumn({ name: 'source_url' })
+    public sourceUrl!: string;
+    @PrimaryColumn({ name: 'tx_hash' })
+    public txHash!: string;
+
+    @Column({ name: 'tx_timestamp', type: 'bigint', transformer: numberToBigIntTransformer })
+    public txTimestamp!: number;
+    @Column({ name: 'tx_date' })
+    public txDate!: string;
+    @Column({ name: 'tx_sender' })
+    public txSender!: string;
+    @Column({ name: 'smart_contract_id', type: 'bigint', transformer: numberToBigIntTransformer })
+    public smartContractId!: number;
+    @Column({ name: 'smart_contract_address' })
+    public smartContractAddress!: string;
+    @Column({ name: 'contract_type' })
+    public contractType!: string;
+    @Column({ type: 'varchar' })
+    public maker!: string;
+    @Column({ type: 'varchar' })
+    public taker!: string;
+    @Column({ name: 'amount_buy', type: 'numeric', transformer: bigNumberTransformer })
+    public amountBuy!: BigNumber;
+    @Column({ name: 'maker_fee_amount', type: 'numeric', transformer: bigNumberTransformer })
+    public makerFeeAmount!: BigNumber;
+    @Column({ name: 'buy_currency_id', type: 'bigint', transformer: numberToBigIntTransformer })
+    public buyCurrencyId!: number;
+    @Column({ name: 'buy_symbol' })
+    public buySymbol!: string;
+    @Column({ name: 'amount_sell', type: 'numeric', transformer: bigNumberTransformer })
+    public amountSell!: BigNumber;
+    @Column({ name: 'taker_fee_amount', type: 'numeric', transformer: bigNumberTransformer })
+    public takerFeeAmount!: BigNumber;
+    @Column({ name: 'sell_currency_id', type: 'bigint', transformer: numberToBigIntTransformer })
+    public sellCurrencyId!: number;
+    @Column({ name: 'sell_symbol' })
+    public sellSymbol!: string;
+    @Column({ name: 'maker_annotation' })
+    public makerAnnotation!: string;
+    @Column({ name: 'taker_annotation' })
+    public takerAnnotation!: string;
+    @Column() public protocol!: string;
+    @Column({ name: 'buy_address', type: 'varchar', nullable: true })
+    public buyAddress!: string | null;
+    @Column({ name: 'sell_address', type: 'varchar', nullable: true })
+    public sellAddress!: string | null;
+}
diff --git a/packages/pipeline/src/entities/exchange_cancel_event.ts b/packages/pipeline/src/entities/exchange_cancel_event.ts
new file mode 100644
index 000000000..38f99c903
--- /dev/null
+++ b/packages/pipeline/src/entities/exchange_cancel_event.ts
@@ -0,0 +1,51 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+import { AssetType } from '../types';
+import { numberToBigIntTransformer } from '../utils';
+
+@Entity({ name: 'exchange_cancel_events', schema: 'raw' })
+export class ExchangeCancelEvent {
+    @PrimaryColumn({ name: 'contract_address' })
+    public contractAddress!: string;
+    @PrimaryColumn({ name: 'log_index' })
+    public logIndex!: number;
+    @PrimaryColumn({ name: 'block_number', transformer: numberToBigIntTransformer })
+    public blockNumber!: number;
+
+    @Column({ name: 'raw_data' })
+    public rawData!: string;
+
+    @Column({ name: 'transaction_hash' })
+    public transactionHash!: string;
+    @Column({ name: 'maker_address' })
+    public makerAddress!: string;
+    @Column({ nullable: true, type: String, name: 'taker_address' })
+    public takerAddress!: string;
+    @Column({ name: 'fee_recipient_address' })
+    public feeRecipientAddress!: string;
+    @Column({ name: 'sender_address' })
+    public senderAddress!: string;
+    @Column({ name: 'order_hash' })
+    public orderHash!: string;
+
+    @Column({ name: 'raw_maker_asset_data' })
+    public rawMakerAssetData!: string;
+    @Column({ name: 'maker_asset_type' })
+    public makerAssetType!: AssetType;
+    @Column({ name: 'maker_asset_proxy_id' })
+    public makerAssetProxyId!: string;
+    @Column({ name: 'maker_token_address' })
+    public makerTokenAddress!: string;
+    @Column({ nullable: true, type: String, name: 'maker_token_id' })
+    public makerTokenId!: string | null;
+    @Column({ name: 'raw_taker_asset_data' })
+    public rawTakerAssetData!: string;
+    @Column({ name: 'taker_asset_type' })
+    public takerAssetType!: AssetType;
+    @Column({ name: 'taker_asset_proxy_id' })
+    public takerAssetProxyId!: string;
+    @Column({ name: 'taker_token_address' })
+    public takerTokenAddress!: string;
+    @Column({ nullable: true, type: String, name: 'taker_token_id' })
+    public takerTokenId!: string | null;
+}
diff --git a/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts b/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts
new file mode 100644
index 000000000..27580305e
--- /dev/null
+++ b/packages/pipeline/src/entities/exchange_cancel_up_to_event.ts
@@ -0,0 +1,26 @@
+import { BigNumber } from '@0x/utils';
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+import { bigNumberTransformer, numberToBigIntTransformer } from '../utils';
+
+@Entity({ name: 'exchange_cancel_up_to_events', schema: 'raw' })
+export class ExchangeCancelUpToEvent {
+    @PrimaryColumn({ name: 'contract_address' })
+    public contractAddress!: string;
+    @PrimaryColumn({ name: 'log_index' })
+    public logIndex!: number;
+    @PrimaryColumn({ name: 'block_number', transformer: numberToBigIntTransformer })
+    public blockNumber!: number;
+
+    @Column({ name: 'raw_data' })
+    public rawData!: string;
+
+    @Column({ name: 'transaction_hash' })
+    public transactionHash!: string;
+    @Column({ name: 'maker_address' })
+    public makerAddress!: string;
+    @Column({ name: 'sender_address' })
+    public senderAddress!: string;
+    @Column({ name: 'order_epoch', type: 'numeric', transformer: bigNumberTransformer })
+    public orderEpoch!: BigNumber;
+}
diff --git a/packages/pipeline/src/entities/exchange_fill_event.ts b/packages/pipeline/src/entities/exchange_fill_event.ts
new file mode 100644
index 000000000..9b7727615
--- /dev/null
+++ b/packages/pipeline/src/entities/exchange_fill_event.ts
@@ -0,0 +1,60 @@
+import { BigNumber } from '@0x/utils';
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+import { AssetType } from '../types';
+import { bigNumberTransformer, numberToBigIntTransformer } from '../utils';
+
+@Entity({ name: 'exchange_fill_events', schema: 'raw' })
+export class ExchangeFillEvent {
+    @PrimaryColumn({ name: 'contract_address' })
+    public contractAddress!: string;
+    @PrimaryColumn({ name: 'log_index' })
+    public logIndex!: number;
+    @PrimaryColumn({ name: 'block_number', transformer: numberToBigIntTransformer })
+    public blockNumber!: number;
+
+    @Column({ name: 'raw_data' })
+    public rawData!: string;
+
+    @Column({ name: 'transaction_hash' })
+    public transactionHash!: string;
+    @Column({ name: 'maker_address' })
+    public makerAddress!: string;
+    @Column({ name: 'taker_address' })
+    public takerAddress!: string;
+    @Column({ name: 'fee_recipient_address' })
+    public feeRecipientAddress!: string;
+    @Column({ name: 'sender_address' })
+    public senderAddress!: string;
+    @Column({ name: 'maker_asset_filled_amount', type: 'numeric', transformer: bigNumberTransformer })
+    public makerAssetFilledAmount!: BigNumber;
+    @Column({ name: 'taker_asset_filled_amount', type: 'numeric', transformer: bigNumberTransformer })
+    public takerAssetFilledAmount!: BigNumber;
+    @Column({ name: 'maker_fee_paid', type: 'numeric', transformer: bigNumberTransformer })
+    public makerFeePaid!: BigNumber;
+    @Column({ name: 'taker_fee_paid', type: 'numeric', transformer: bigNumberTransformer })
+    public takerFeePaid!: BigNumber;
+    @Column({ name: 'order_hash' })
+    public orderHash!: string;
+
+    @Column({ name: 'raw_maker_asset_data' })
+    public rawMakerAssetData!: string;
+    @Column({ name: 'maker_asset_type' })
+    public makerAssetType!: AssetType;
+    @Column({ name: 'maker_asset_proxy_id' })
+    public makerAssetProxyId!: string;
+    @Column({ name: 'maker_token_address' })
+    public makerTokenAddress!: string;
+    @Column({ nullable: true, type: String, name: 'maker_token_id' })
+    public makerTokenId!: string | null;
+    @Column({ name: 'raw_taker_asset_data' })
+    public rawTakerAssetData!: string;
+    @Column({ name: 'taker_asset_type' })
+    public takerAssetType!: AssetType;
+    @Column({ name: 'taker_asset_proxy_id' })
+    public takerAssetProxyId!: string;
+    @Column({ name: 'taker_token_address' })
+    public takerTokenAddress!: string;
+    @Column({ nullable: true, type: String, name: 'taker_token_id' })
+    public takerTokenId!: string | null;
+}
diff --git a/packages/pipeline/src/entities/index.ts b/packages/pipeline/src/entities/index.ts
new file mode 100644
index 000000000..db0814e38
--- /dev/null
+++ b/packages/pipeline/src/entities/index.ts
@@ -0,0 +1,18 @@
+import { ExchangeCancelEvent } from './exchange_cancel_event';
+import { ExchangeCancelUpToEvent } from './exchange_cancel_up_to_event';
+import { ExchangeFillEvent } from './exchange_fill_event';
+
+export { Block } from './block';
+export { DexTrade } from './dex_trade';
+export { ExchangeCancelEvent } from './exchange_cancel_event';
+export { ExchangeCancelUpToEvent } from './exchange_cancel_up_to_event';
+export { ExchangeFillEvent } from './exchange_fill_event';
+export { OHLCVExternal } from './ohlcv_external';
+export { Relayer } from './relayer';
+export { SraOrder } from './sra_order';
+export { SraOrdersObservedTimeStamp, createObservedTimestampForOrder } from './sra_order_observed_timestamp';
+export { TokenMetadata } from './token_metadata';
+export { TokenOrderbookSnapshot } from './token_order';
+export { Transaction } from './transaction';
+
+export type ExchangeEvent = ExchangeFillEvent | ExchangeCancelEvent | ExchangeCancelUpToEvent;
diff --git a/packages/pipeline/src/entities/ohlcv_external.ts b/packages/pipeline/src/entities/ohlcv_external.ts
new file mode 100644
index 000000000..4f55dd930
--- /dev/null
+++ b/packages/pipeline/src/entities/ohlcv_external.ts
@@ -0,0 +1,30 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+import { numberToBigIntTransformer } from '../utils';
+
+@Entity({ name: 'ohlcv_external', schema: 'raw' })
+export class OHLCVExternal {
+    @PrimaryColumn() public exchange!: string;
+
+    @PrimaryColumn({ name: 'from_symbol', type: 'varchar' })
+    public fromSymbol!: string;
+    @PrimaryColumn({ name: 'to_symbol', type: 'varchar' })
+    public toSymbol!: string;
+    @PrimaryColumn({ name: 'start_time', transformer: numberToBigIntTransformer })
+    public startTime!: number;
+    @PrimaryColumn({ name: 'end_time', transformer: numberToBigIntTransformer })
+    public endTime!: number;
+
+    @Column() public open!: number;
+    @Column() public close!: number;
+    @Column() public low!: number;
+    @Column() public high!: number;
+    @Column({ name: 'volume_from' })
+    public volumeFrom!: number;
+    @Column({ name: 'volume_to' })
+    public volumeTo!: number;
+
+    @PrimaryColumn() public source!: string;
+    @PrimaryColumn({ name: 'observed_timestamp', transformer: numberToBigIntTransformer })
+    public observedTimestamp!: number;
+}
diff --git a/packages/pipeline/src/entities/relayer.ts b/packages/pipeline/src/entities/relayer.ts
new file mode 100644
index 000000000..5af8578b4
--- /dev/null
+++ b/packages/pipeline/src/entities/relayer.ts
@@ -0,0 +1,21 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+@Entity({ name: 'relayers', schema: 'raw' })
+export class Relayer {
+    @PrimaryColumn() public uuid!: string;
+
+    @Column() public name!: string;
+    @Column({ name: 'homepage_url', type: 'varchar' })
+    public homepageUrl!: string;
+    @Column({ name: 'sra_http_endpoint', type: 'varchar', nullable: true })
+    public sraHttpEndpoint!: string | null;
+    @Column({ name: 'sra_ws_endpoint', type: 'varchar', nullable: true })
+    public sraWsEndpoint!: string | null;
+    @Column({ name: 'app_url', type: 'varchar', nullable: true })
+    public appUrl!: string | null;
+
+    @Column({ name: 'fee_recipient_addresses', type: 'varchar', array: true })
+    public feeRecipientAddresses!: string[];
+    @Column({ name: 'taker_addresses', type: 'varchar', array: true })
+    public takerAddresses!: string[];
+}
diff --git a/packages/pipeline/src/entities/sra_order.ts b/packages/pipeline/src/entities/sra_order.ts
new file mode 100644
index 000000000..9c730a0bb
--- /dev/null
+++ b/packages/pipeline/src/entities/sra_order.ts
@@ -0,0 +1,63 @@
+import { BigNumber } from '@0x/utils';
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+import { AssetType } from '../types';
+import { bigNumberTransformer } from '../utils';
+
+@Entity({ name: 'sra_orders', schema: 'raw' })
+export class SraOrder {
+    @PrimaryColumn({ name: 'exchange_address' })
+    public exchangeAddress!: string;
+    @PrimaryColumn({ name: 'order_hash_hex' })
+    public orderHashHex!: string;
+    @PrimaryColumn({ name: 'source_url' })
+    public sourceUrl!: string;
+
+    @Column({ name: 'maker_address' })
+    public makerAddress!: string;
+    @Column({ name: 'taker_address' })
+    public takerAddress!: string;
+    @Column({ name: 'fee_recipient_address' })
+    public feeRecipientAddress!: string;
+    @Column({ name: 'sender_address' })
+    public senderAddress!: string;
+    @Column({ name: 'maker_asset_amount', type: 'numeric', transformer: bigNumberTransformer })
+    public makerAssetAmount!: BigNumber;
+    @Column({ name: 'taker_asset_amount', type: 'numeric', transformer: bigNumberTransformer })
+    public takerAssetAmount!: BigNumber;
+    @Column({ name: 'maker_fee', type: 'numeric', transformer: bigNumberTransformer })
+    public makerFee!: BigNumber;
+    @Column({ name: 'taker_fee', type: 'numeric', transformer: bigNumberTransformer })
+    public takerFee!: BigNumber;
+    @Column({ name: 'expiration_time_seconds', type: 'numeric', transformer: bigNumberTransformer })
+    public expirationTimeSeconds!: BigNumber;
+    @Column({ name: 'salt', type: 'numeric', transformer: bigNumberTransformer })
+    public salt!: BigNumber;
+    @Column({ name: 'signature' })
+    public signature!: string;
+
+    @Column({ name: 'raw_maker_asset_data' })
+    public rawMakerAssetData!: string;
+    @Column({ name: 'maker_asset_type' })
+    public makerAssetType!: AssetType;
+    @Column({ name: 'maker_asset_proxy_id' })
+    public makerAssetProxyId!: string;
+    @Column({ name: 'maker_token_address' })
+    public makerTokenAddress!: string;
+    @Column({ nullable: true, type: String, name: 'maker_token_id' })
+    public makerTokenId!: string | null;
+    @Column({ name: 'raw_taker_asset_data' })
+    public rawTakerAssetData!: string;
+    @Column({ name: 'taker_asset_type' })
+    public takerAssetType!: AssetType;
+    @Column({ name: 'taker_asset_proxy_id' })
+    public takerAssetProxyId!: string;
+    @Column({ name: 'taker_token_address' })
+    public takerTokenAddress!: string;
+    @Column({ nullable: true, type: String, name: 'taker_token_id' })
+    public takerTokenId!: string | null;
+
+    // TODO(albrow): Make this optional?
+    @Column({ name: 'metadata_json' })
+    public metadataJson!: string;
+}
diff --git a/packages/pipeline/src/entities/sra_order_observed_timestamp.ts b/packages/pipeline/src/entities/sra_order_observed_timestamp.ts
new file mode 100644
index 000000000..cbec1c6d0
--- /dev/null
+++ b/packages/pipeline/src/entities/sra_order_observed_timestamp.ts
@@ -0,0 +1,35 @@
+import { Entity, PrimaryColumn } from 'typeorm';
+
+import { numberToBigIntTransformer } from '../utils';
+
+import { SraOrder } from './sra_order';
+
+@Entity({ name: 'sra_orders_observed_timestamps', schema: 'raw' })
+export class SraOrdersObservedTimeStamp {
+    @PrimaryColumn({ name: 'exchange_address' })
+    public exchangeAddress!: string;
+    @PrimaryColumn({ name: 'order_hash_hex' })
+    public orderHashHex!: string;
+    @PrimaryColumn({ name: 'source_url' })
+    public sourceUrl!: string;
+
+    @PrimaryColumn({ name: 'observed_timestamp', transformer: numberToBigIntTransformer })
+    public observedTimestamp!: number;
+}
+
+/**
+ * Returns a new SraOrdersObservedTimeStamp for the given order based on the
+ * current time.
+ * @param order The order to generate a timestamp for.
+ */
+export function createObservedTimestampForOrder(
+    order: SraOrder,
+    observedTimestamp: number,
+): SraOrdersObservedTimeStamp {
+    const observed = new SraOrdersObservedTimeStamp();
+    observed.exchangeAddress = order.exchangeAddress;
+    observed.orderHashHex = order.orderHashHex;
+    observed.sourceUrl = order.sourceUrl;
+    observed.observedTimestamp = observedTimestamp;
+    return observed;
+}
diff --git a/packages/pipeline/src/entities/token_metadata.ts b/packages/pipeline/src/entities/token_metadata.ts
new file mode 100644
index 000000000..911b53972
--- /dev/null
+++ b/packages/pipeline/src/entities/token_metadata.ts
@@ -0,0 +1,22 @@
+import { BigNumber } from '@0x/utils';
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+import { bigNumberTransformer } from '../utils/transformers';
+
+@Entity({ name: 'token_metadata', schema: 'raw' })
+export class TokenMetadata {
+    @PrimaryColumn({ type: 'varchar', nullable: false })
+    public address!: string;
+
+    @PrimaryColumn({ type: 'varchar', nullable: false })
+    public authority!: string;
+
+    @Column({ type: 'numeric', transformer: bigNumberTransformer, nullable: true })
+    public decimals!: BigNumber | null;
+
+    @Column({ type: 'varchar', nullable: true })
+    public symbol!: string | null;
+
+    @Column({ type: 'varchar', nullable: true })
+    public name!: string | null;
+}
diff --git a/packages/pipeline/src/entities/token_order.ts b/packages/pipeline/src/entities/token_order.ts
new file mode 100644
index 000000000..557705767
--- /dev/null
+++ b/packages/pipeline/src/entities/token_order.ts
@@ -0,0 +1,29 @@
+import { BigNumber } from '@0x/utils';
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+import { OrderType } from '../types';
+import { bigNumberTransformer, numberToBigIntTransformer } from '../utils';
+
+@Entity({ name: 'token_orderbook_snapshots', schema: 'raw' })
+export class TokenOrderbookSnapshot {
+    @PrimaryColumn({ name: 'observed_timestamp', type: 'bigint', transformer: numberToBigIntTransformer })
+    public observedTimestamp!: number;
+    @PrimaryColumn({ name: 'source' })
+    public source!: string;
+    @Column({ name: 'order_type' })
+    public orderType!: OrderType;
+    @PrimaryColumn({ name: 'price', type: 'numeric', transformer: bigNumberTransformer })
+    public price!: BigNumber;
+    @PrimaryColumn({ name: 'base_asset_symbol' })
+    public baseAssetSymbol!: string;
+    @Column({ name: 'base_asset_address' })
+    public baseAssetAddress!: string;
+    @Column({ name: 'base_volume', type: 'numeric', transformer: bigNumberTransformer })
+    public baseVolume!: BigNumber;
+    @PrimaryColumn({ name: 'quote_asset_symbol' })
+    public quoteAssetSymbol!: string;
+    @Column({ name: 'quote_asset_address' })
+    public quoteAssetAddress!: string;
+    @Column({ name: 'quote_volume', type: 'numeric', transformer: bigNumberTransformer })
+    public quoteVolume!: BigNumber;
+}
diff --git a/packages/pipeline/src/entities/transaction.ts b/packages/pipeline/src/entities/transaction.ts
new file mode 100644
index 000000000..742050177
--- /dev/null
+++ b/packages/pipeline/src/entities/transaction.ts
@@ -0,0 +1,19 @@
+import { BigNumber } from '@0x/utils';
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+import { bigNumberTransformer, numberToBigIntTransformer } from '../utils';
+
+@Entity({ name: 'transactions', schema: 'raw' })
+export class Transaction {
+    @PrimaryColumn({ name: 'transaction_hash' })
+    public transactionHash!: string;
+    @PrimaryColumn({ name: 'block_hash' })
+    public blockHash!: string;
+    @PrimaryColumn({ name: 'block_number', transformer: numberToBigIntTransformer })
+    public blockNumber!: number;
+
+    @Column({ type: 'numeric', name: 'gas_used', transformer: bigNumberTransformer })
+    public gasUsed!: BigNumber;
+    @Column({ type: 'numeric', name: 'gas_price', transformer: bigNumberTransformer })
+    public gasPrice!: BigNumber;
+}