aboutsummaryrefslogtreecommitdiffstats
path: root/packages/pipeline
diff options
context:
space:
mode:
authorAlex Browne <stephenalexbrowne@gmail.com>2018-10-24 07:03:52 +0800
committerFred Carlsen <fred@sjelfull.no>2018-12-06 19:04:24 +0800
commit1402a3dfae1be32a346f8b75a4209950bda00b74 (patch)
treecdd899665634759da68b51a6d4b57e6a7c47a10e /packages/pipeline
parent64865bc10f88f97c3e406bbc1d39e68ca45380ef (diff)
downloaddexon-sol-tools-1402a3dfae1be32a346f8b75a4209950bda00b74.tar
dexon-sol-tools-1402a3dfae1be32a346f8b75a4209950bda00b74.tar.gz
dexon-sol-tools-1402a3dfae1be32a346f8b75a4209950bda00b74.tar.bz2
dexon-sol-tools-1402a3dfae1be32a346f8b75a4209950bda00b74.tar.lz
dexon-sol-tools-1402a3dfae1be32a346f8b75a4209950bda00b74.tar.xz
dexon-sol-tools-1402a3dfae1be32a346f8b75a4209950bda00b74.tar.zst
dexon-sol-tools-1402a3dfae1be32a346f8b75a4209950bda00b74.zip
Implement support for getting and parsing blocks and transactions
Diffstat (limited to 'packages/pipeline')
-rw-r--r--packages/pipeline/src/data_sources/web3/index.ts22
-rw-r--r--packages/pipeline/src/entities/Block.ts9
-rw-r--r--packages/pipeline/src/entities/Transaction.ts11
-rw-r--r--packages/pipeline/src/index.ts68
-rw-r--r--packages/pipeline/src/parsers/web3/index.ts39
5 files changed, 149 insertions, 0 deletions
diff --git a/packages/pipeline/src/data_sources/web3/index.ts b/packages/pipeline/src/data_sources/web3/index.ts
new file mode 100644
index 000000000..64e909939
--- /dev/null
+++ b/packages/pipeline/src/data_sources/web3/index.ts
@@ -0,0 +1,22 @@
+import { Web3ProviderEngine } from '@0x/subproviders';
+import { Web3Wrapper } from '@0x/web3-wrapper';
+import { BlockWithoutTransactionData, Transaction } from 'ethereum-types';
+
+export class Web3Source {
+ private _web3Wrapper: Web3Wrapper;
+ constructor(provider: Web3ProviderEngine) {
+ this._web3Wrapper = new Web3Wrapper(provider);
+ }
+
+ public async getBlockInfoAsync(blockNumber: number): Promise<BlockWithoutTransactionData> {
+ const block = await this._web3Wrapper.getBlockIfExistsAsync(blockNumber);
+ if (block == null) {
+ return Promise.reject(new Error('Could not find block for given block number: ' + blockNumber));
+ }
+ return block;
+ }
+
+ public async getTransactionInfoAsync(txHash: string): Promise<Transaction> {
+ return this._web3Wrapper.getTransactionByHashAsync(txHash);
+ }
+}
diff --git a/packages/pipeline/src/entities/Block.ts b/packages/pipeline/src/entities/Block.ts
new file mode 100644
index 000000000..49e0ef840
--- /dev/null
+++ b/packages/pipeline/src/entities/Block.ts
@@ -0,0 +1,9 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+@Entity()
+export class Block {
+ @PrimaryColumn() public hash!: string;
+ @PrimaryColumn() public number!: number;
+
+ @Column() public unixTimestampSeconds!: number;
+}
diff --git a/packages/pipeline/src/entities/Transaction.ts b/packages/pipeline/src/entities/Transaction.ts
new file mode 100644
index 000000000..d89d44746
--- /dev/null
+++ b/packages/pipeline/src/entities/Transaction.ts
@@ -0,0 +1,11 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+@Entity()
+export class Transaction {
+ @PrimaryColumn() public transactionHash!: string;
+ @PrimaryColumn() public blockHash!: string;
+ @PrimaryColumn() public blockNumber!: number;
+
+ @Column() public gasUsed!: number;
+ @Column() public gasPrice!: number;
+}
diff --git a/packages/pipeline/src/index.ts b/packages/pipeline/src/index.ts
new file mode 100644
index 000000000..ad0e4c68f
--- /dev/null
+++ b/packages/pipeline/src/index.ts
@@ -0,0 +1,68 @@
+import { web3Factory } from '@0x/dev-utils';
+import { Web3ProviderEngine } from '@0x/subproviders';
+import 'reflect-metadata';
+import { Connection, createConnection } from 'typeorm';
+
+import { ExchangeEventsSource } from './data_sources/contract-wrappers/exchange_events';
+import { Web3Source } from './data_sources/web3';
+import { Block } from './entities/Block';
+import { ExchangeFillEvent } from './entities/ExchangeFillEvent';
+import { Transaction } from './entities/Transaction';
+import { testConfig } from './ormconfig';
+import { parseExchangeEvents } from './parsers/events';
+import { parseBlock, parseTransaction } from './parsers/web3';
+
+const EXCHANGE_START_BLOCK = 6271590; // Block number when the Exchange contract was deployed to mainnet.
+
+let connection: Connection;
+
+(async () => {
+ connection = await createConnection(testConfig);
+ const provider = web3Factory.getRpcProvider({
+ rpcUrl: 'https://mainnet.infura.io',
+ });
+ await getExchangeEventsAsync(provider);
+ await getBlockAsync(provider);
+ await getTransactionAsync(provider);
+ console.log('Exiting process');
+ process.exit(0);
+})();
+
+async function getExchangeEventsAsync(provider: Web3ProviderEngine): Promise<void> {
+ console.log('Getting event logs...');
+ const eventsRepository = connection.getRepository(ExchangeFillEvent);
+ const exchangeEvents = new ExchangeEventsSource(provider, 1);
+ const eventLogs = await exchangeEvents.getFillEventsAsync(EXCHANGE_START_BLOCK, EXCHANGE_START_BLOCK + 100000);
+ console.log('Parsing events...');
+ const events = parseExchangeEvents(eventLogs);
+ console.log(`Retrieved and parsed ${events.length} total events.`);
+ console.log('Saving events...');
+ for (const event of events) {
+ await eventsRepository.save(event);
+ }
+ console.log('Saved events.');
+}
+
+async function getBlockAsync(provider: Web3ProviderEngine): Promise<void> {
+ console.log('Getting block info...');
+ const blocksRepository = connection.getRepository(Block);
+ const web3Source = new Web3Source(provider);
+ const rawBlock = await web3Source.getBlockInfoAsync(EXCHANGE_START_BLOCK);
+ const block = parseBlock(rawBlock);
+ console.log('Saving block info...');
+ await blocksRepository.save(block);
+ console.log('Done saving block.');
+}
+
+async function getTransactionAsync(provider: Web3ProviderEngine): Promise<void> {
+ console.log('Getting tx info...');
+ const txsRepository = connection.getRepository(Transaction);
+ const web3Source = new Web3Source(provider);
+ const rawTx = await web3Source.getTransactionInfoAsync(
+ '0x6dd106d002873746072fc5e496dd0fb2541b68c77bcf9184ae19a42fd33657fe',
+ );
+ const tx = parseTransaction(rawTx);
+ console.log('Saving tx info...');
+ await txsRepository.save(tx);
+ console.log('Done saving tx.');
+}
diff --git a/packages/pipeline/src/parsers/web3/index.ts b/packages/pipeline/src/parsers/web3/index.ts
new file mode 100644
index 000000000..c6647c966
--- /dev/null
+++ b/packages/pipeline/src/parsers/web3/index.ts
@@ -0,0 +1,39 @@
+import { BlockWithoutTransactionData, Transaction as EthTransaction } from 'ethereum-types';
+
+import { Block } from '../../entities/Block';
+import { Transaction } from '../../entities/Transaction';
+
+export function parseBlock(rawBlock: BlockWithoutTransactionData): Block {
+ if (rawBlock.hash == null) {
+ throw new Error('Tried to parse raw block but hash was null');
+ }
+ if (rawBlock.number == null) {
+ throw new Error('Tried to parse raw block but number was null');
+ }
+
+ const block = new Block();
+ block.hash = rawBlock.hash;
+ block.number = rawBlock.number;
+ block.unixTimestampSeconds = rawBlock.timestamp;
+ return block;
+}
+
+export function parseTransaction(rawTransaction: EthTransaction): Transaction {
+ if (rawTransaction.blockHash == null) {
+ throw new Error('Tried to parse raw transaction but blockHash was null');
+ }
+ if (rawTransaction.blockNumber == null) {
+ throw new Error('Tried to parse raw transaction but blockNumber was null');
+ }
+
+ const tx = new Transaction();
+ tx.transactionHash = rawTransaction.hash;
+ tx.blockHash = rawTransaction.blockHash;
+ tx.blockNumber = rawTransaction.blockNumber;
+
+ tx.gasUsed = rawTransaction.gas;
+ // TODO(albrow) figure out bignum solution.
+ tx.gasPrice = rawTransaction.gasPrice.toNumber();
+
+ return tx;
+}