From 1402a3dfae1be32a346f8b75a4209950bda00b74 Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Tue, 23 Oct 2018 16:03:52 -0700 Subject: Implement support for getting and parsing blocks and transactions --- packages/pipeline/src/data_sources/web3/index.ts | 22 ++++++++ packages/pipeline/src/entities/Block.ts | 9 ++++ packages/pipeline/src/entities/Transaction.ts | 11 ++++ packages/pipeline/src/index.ts | 68 ++++++++++++++++++++++++ packages/pipeline/src/parsers/web3/index.ts | 39 ++++++++++++++ 5 files changed, 149 insertions(+) create mode 100644 packages/pipeline/src/data_sources/web3/index.ts create mode 100644 packages/pipeline/src/entities/Block.ts create mode 100644 packages/pipeline/src/entities/Transaction.ts create mode 100644 packages/pipeline/src/index.ts create mode 100644 packages/pipeline/src/parsers/web3/index.ts (limited to 'packages/pipeline') diff --git a/packages/pipeline/src/data_sources/web3/index.ts b/packages/pipeline/src/data_sources/web3/index.ts new file mode 100644 index 000000000..64e909939 --- /dev/null +++ b/packages/pipeline/src/data_sources/web3/index.ts @@ -0,0 +1,22 @@ +import { Web3ProviderEngine } from '@0x/subproviders'; +import { Web3Wrapper } from '@0x/web3-wrapper'; +import { BlockWithoutTransactionData, Transaction } from 'ethereum-types'; + +export class Web3Source { + private _web3Wrapper: Web3Wrapper; + constructor(provider: Web3ProviderEngine) { + this._web3Wrapper = new Web3Wrapper(provider); + } + + public async getBlockInfoAsync(blockNumber: number): Promise { + const block = await this._web3Wrapper.getBlockIfExistsAsync(blockNumber); + if (block == null) { + return Promise.reject(new Error('Could not find block for given block number: ' + blockNumber)); + } + return block; + } + + public async getTransactionInfoAsync(txHash: string): Promise { + return this._web3Wrapper.getTransactionByHashAsync(txHash); + } +} diff --git a/packages/pipeline/src/entities/Block.ts b/packages/pipeline/src/entities/Block.ts new file mode 100644 index 000000000..49e0ef840 --- /dev/null +++ b/packages/pipeline/src/entities/Block.ts @@ -0,0 +1,9 @@ +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +@Entity() +export class Block { + @PrimaryColumn() public hash!: string; + @PrimaryColumn() public number!: number; + + @Column() public unixTimestampSeconds!: number; +} diff --git a/packages/pipeline/src/entities/Transaction.ts b/packages/pipeline/src/entities/Transaction.ts new file mode 100644 index 000000000..d89d44746 --- /dev/null +++ b/packages/pipeline/src/entities/Transaction.ts @@ -0,0 +1,11 @@ +import { Column, Entity, PrimaryColumn } from 'typeorm'; + +@Entity() +export class Transaction { + @PrimaryColumn() public transactionHash!: string; + @PrimaryColumn() public blockHash!: string; + @PrimaryColumn() public blockNumber!: number; + + @Column() public gasUsed!: number; + @Column() public gasPrice!: number; +} diff --git a/packages/pipeline/src/index.ts b/packages/pipeline/src/index.ts new file mode 100644 index 000000000..ad0e4c68f --- /dev/null +++ b/packages/pipeline/src/index.ts @@ -0,0 +1,68 @@ +import { web3Factory } from '@0x/dev-utils'; +import { Web3ProviderEngine } from '@0x/subproviders'; +import 'reflect-metadata'; +import { Connection, createConnection } from 'typeorm'; + +import { ExchangeEventsSource } from './data_sources/contract-wrappers/exchange_events'; +import { Web3Source } from './data_sources/web3'; +import { Block } from './entities/Block'; +import { ExchangeFillEvent } from './entities/ExchangeFillEvent'; +import { Transaction } from './entities/Transaction'; +import { testConfig } from './ormconfig'; +import { parseExchangeEvents } from './parsers/events'; +import { parseBlock, parseTransaction } from './parsers/web3'; + +const EXCHANGE_START_BLOCK = 6271590; // Block number when the Exchange contract was deployed to mainnet. + +let connection: Connection; + +(async () => { + connection = await createConnection(testConfig); + const provider = web3Factory.getRpcProvider({ + rpcUrl: 'https://mainnet.infura.io', + }); + await getExchangeEventsAsync(provider); + await getBlockAsync(provider); + await getTransactionAsync(provider); + console.log('Exiting process'); + process.exit(0); +})(); + +async function getExchangeEventsAsync(provider: Web3ProviderEngine): Promise { + console.log('Getting event logs...'); + const eventsRepository = connection.getRepository(ExchangeFillEvent); + const exchangeEvents = new ExchangeEventsSource(provider, 1); + const eventLogs = await exchangeEvents.getFillEventsAsync(EXCHANGE_START_BLOCK, EXCHANGE_START_BLOCK + 100000); + console.log('Parsing events...'); + const events = parseExchangeEvents(eventLogs); + console.log(`Retrieved and parsed ${events.length} total events.`); + console.log('Saving events...'); + for (const event of events) { + await eventsRepository.save(event); + } + console.log('Saved events.'); +} + +async function getBlockAsync(provider: Web3ProviderEngine): Promise { + console.log('Getting block info...'); + const blocksRepository = connection.getRepository(Block); + const web3Source = new Web3Source(provider); + const rawBlock = await web3Source.getBlockInfoAsync(EXCHANGE_START_BLOCK); + const block = parseBlock(rawBlock); + console.log('Saving block info...'); + await blocksRepository.save(block); + console.log('Done saving block.'); +} + +async function getTransactionAsync(provider: Web3ProviderEngine): Promise { + console.log('Getting tx info...'); + const txsRepository = connection.getRepository(Transaction); + const web3Source = new Web3Source(provider); + const rawTx = await web3Source.getTransactionInfoAsync( + '0x6dd106d002873746072fc5e496dd0fb2541b68c77bcf9184ae19a42fd33657fe', + ); + const tx = parseTransaction(rawTx); + console.log('Saving tx info...'); + await txsRepository.save(tx); + console.log('Done saving tx.'); +} diff --git a/packages/pipeline/src/parsers/web3/index.ts b/packages/pipeline/src/parsers/web3/index.ts new file mode 100644 index 000000000..c6647c966 --- /dev/null +++ b/packages/pipeline/src/parsers/web3/index.ts @@ -0,0 +1,39 @@ +import { BlockWithoutTransactionData, Transaction as EthTransaction } from 'ethereum-types'; + +import { Block } from '../../entities/Block'; +import { Transaction } from '../../entities/Transaction'; + +export function parseBlock(rawBlock: BlockWithoutTransactionData): Block { + if (rawBlock.hash == null) { + throw new Error('Tried to parse raw block but hash was null'); + } + if (rawBlock.number == null) { + throw new Error('Tried to parse raw block but number was null'); + } + + const block = new Block(); + block.hash = rawBlock.hash; + block.number = rawBlock.number; + block.unixTimestampSeconds = rawBlock.timestamp; + return block; +} + +export function parseTransaction(rawTransaction: EthTransaction): Transaction { + if (rawTransaction.blockHash == null) { + throw new Error('Tried to parse raw transaction but blockHash was null'); + } + if (rawTransaction.blockNumber == null) { + throw new Error('Tried to parse raw transaction but blockNumber was null'); + } + + const tx = new Transaction(); + tx.transactionHash = rawTransaction.hash; + tx.blockHash = rawTransaction.blockHash; + tx.blockNumber = rawTransaction.blockNumber; + + tx.gasUsed = rawTransaction.gas; + // TODO(albrow) figure out bignum solution. + tx.gasPrice = rawTransaction.gasPrice.toNumber(); + + return tx; +} -- cgit v1.2.3