Implement fetching and parsing relayer info

6 files changed, 114 insertions, 7 deletions

diff --git a/packages/pipeline/package.json b/packages/pipeline/package.json
index 47033bf33..253da3f19 100644
--- a/packages/pipeline/package.json
+++ b/packages/pipeline/package.json
@@ -27,6 +27,7 @@
     "devDependencies": {
         "@0x/tslint-config": "^1.0.9",
         "@types/ramda": "^0.25.38",
+        "@types/axios": "^0.14.0",
         "chai": "^4.1.2",
         "chai-as-promised": "^7.1.1",
         "chai-bignumber": "^2.0.2",
diff --git a/packages/pipeline/src/data_sources/relayer-registry/index.ts b/packages/pipeline/src/data_sources/relayer-registry/index.ts
new file mode 100644
index 000000000..c97b50d27
--- /dev/null
+++ b/packages/pipeline/src/data_sources/relayer-registry/index.ts
@@ -0,0 +1,33 @@
+import axios from 'axios';
+
+export interface RelayerResponse {
+    name: string;
+    homepage_url: string;
+    app_url: string;
+    header_img: string;
+    logo_img: string;
+    networks: RelayerResponseNetwork[];
+}
+
+export interface RelayerResponseNetwork {
+    networkId: number;
+    sra_http_endpoint?: string;
+    sra_ws_endpoint?: string;
+    static_order_fields?: {
+        fee_recipient_addresses?: string[];
+        taker_addresses?: string[];
+    };
+}
+
+export class RelayerRegistrySource {
+    private _url: string;
+
+    constructor(url: string) {
+        this._url = url;
+    }
+
+    public async getRelayerInfoAsync(): Promise<RelayerResponse[]> {
+        const resp = await axios.get<RelayerResponse[]>(this._url);
+        return resp.data;
+    }
+}
diff --git a/packages/pipeline/src/entities/Relayer.ts b/packages/pipeline/src/entities/Relayer.ts
new file mode 100644
index 000000000..ebdcbf345
--- /dev/null
+++ b/packages/pipeline/src/entities/Relayer.ts
@@ -0,0 +1,22 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm';
+
+@Entity()
+export class Relayer {
+    @PrimaryColumn() public name!: string;
+
+    @Column() public url!: string;
+    @Column({ nullable: true, type: String })
+    public sraHttpEndpoint!: string | null;
+    @Column({ nullable: true, type: String })
+    public sraWsEndpoint!: string | null;
+    @Column({ nullable: true, type: String })
+    public appUrl!: string | null;
+
+    // TODO(albrow): Add exchange contract or protocol version?
+    // TODO(albrow): Add network ids for addresses?
+
+    @Column({ type: 'varchar', array: true })
+    public feeRecipientAddresses!: string[];
+    @Column({ type: 'varchar', array: true })
+    public takerAddresses!: string[];
+}
diff --git a/packages/pipeline/src/index.ts b/packages/pipeline/src/index.ts
index ad0e4c68f..9483eb257 100644
--- a/packages/pipeline/src/index.ts
+++ b/packages/pipeline/src/index.ts
@@ -4,27 +4,31 @@ import 'reflect-metadata';
 import { Connection, createConnection } from 'typeorm';
 
 import { ExchangeEventsSource } from './data_sources/contract-wrappers/exchange_events';
+import { RelayerRegistrySource } from './data_sources/relayer-registry';
 import { Web3Source } from './data_sources/web3';
 import { Block } from './entities/Block';
 import { ExchangeFillEvent } from './entities/ExchangeFillEvent';
+import { Relayer } from './entities/Relayer';
 import { Transaction } from './entities/Transaction';
-import { testConfig } from './ormconfig';
+import { deployConfig } from './ormconfig';
 import { parseExchangeEvents } from './parsers/events';
+import { parseRelayers } from './parsers/relayer_registry';
 import { parseBlock, parseTransaction } from './parsers/web3';
 
 const EXCHANGE_START_BLOCK = 6271590; // Block number when the Exchange contract was deployed to mainnet.
+const RELAYER_REGISTRY_URL = 'https://raw.githubusercontent.com/0xProject/0x-relayer-registry/master/relayers.json';
 
 let connection: Connection;
 
 (async () => {
-    connection = await createConnection(testConfig);
+    connection = await createConnection(deployConfig);
     const provider = web3Factory.getRpcProvider({
         rpcUrl: 'https://mainnet.infura.io',
     });
-    await getExchangeEventsAsync(provider);
-    await getBlockAsync(provider);
-    await getTransactionAsync(provider);
-    console.log('Exiting process');
+    // await getExchangeEventsAsync(provider);
+    // await getBlockAsync(provider);
+    // await getTransactionAsync(provider);
+    await getRelayers(RELAYER_REGISTRY_URL);
     process.exit(0);
 })();
 
@@ -66,3 +70,16 @@ async function getTransactionAsync(provider: Web3ProviderEngine): Promise<void>
     await txsRepository.save(tx);
     console.log('Done saving tx.');
 }
+
+async function getRelayers(url: string): Promise<void> {
+    console.log('Getting relayer info...');
+    const relayerRepository = connection.getRepository(Relayer);
+    const relayerSource = new RelayerRegistrySource(RELAYER_REGISTRY_URL);
+    const relayersResp = await relayerSource.getRelayerInfoAsync();
+    const relayers = parseRelayers(relayersResp);
+    console.log('Saving relayer info...');
+    // for (const relayer of relayers) {
+    await relayerRepository.save(relayers);
+    // }
+    console.log('Done saving relayers.');
+}
diff --git a/packages/pipeline/src/parsers/relayer_registry/index.ts b/packages/pipeline/src/parsers/relayer_registry/index.ts
new file mode 100644
index 000000000..6eca10167
--- /dev/null
+++ b/packages/pipeline/src/parsers/relayer_registry/index.ts
@@ -0,0 +1,28 @@
+import * as R from 'ramda';
+
+import { RelayerResponse, RelayerResponseNetwork } from '../../data_sources/relayer-registry';
+import { Relayer } from '../../entities/Relayer';
+
+export const parseRelayers = R.map(parseRelayer);
+
+function parseRelayer(relayerResp: RelayerResponse): Relayer {
+    const relayer = new Relayer();
+    relayer.name = relayerResp.name;
+    relayer.url = relayerResp.homepage_url;
+    relayer.appUrl = relayerResp.app_url;
+    const mainnet = getMainNetwork(relayerResp);
+    if (mainnet !== undefined) {
+        relayer.sraHttpEndpoint = mainnet.sra_http_endpoint || null;
+        relayer.sraWsEndpoint = mainnet.sra_ws_endpoint || null;
+        relayer.feeRecipientAddresses = R.path(['static_order_fields', 'fee_recipient_addresses'], mainnet) || [];
+        relayer.takerAddresses = R.path(['static_order_fields', 'taker_addresses'], mainnet) || [];
+    } else {
+        relayer.feeRecipientAddresses = [];
+        relayer.takerAddresses = [];
+    }
+    return relayer;
+}
+
+function getMainNetwork(relayerResp: RelayerResponse): RelayerResponseNetwork | undefined {
+    return R.find(network => network.networkId === 1, relayerResp.networks);
+}
diff --git a/yarn.lock b/yarn.lock
index 195c31a55..b1de13e68 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1231,6 +1231,12 @@
   version "0.4.1"
   resolved "https://registry.yarnpkg.com/@types/accounting/-/accounting-0.4.1.tgz#865d9f5694fd7c438fba34eb4bc82eec6f34cdd5"
 
+"@types/axios@^0.14.0":
+  version "0.14.0"
+  resolved "https://registry.yarnpkg.com/@types/axios/-/axios-0.14.0.tgz#ec2300fbe7d7dddd7eb9d3abf87999964cafce46"
+  dependencies:
+    axios "*"
+
 "@types/bintrees@^1.0.2":
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/@types/bintrees/-/bintrees-1.0.2.tgz#0dfdce4eeebdf90427bd35b0e79dc248b3d157a6"
@@ -2427,7 +2433,7 @@ aws4@^1.8.0:
   version "1.8.0"
   resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.8.0.tgz#f0e003d9ca9e7f59c7a508945d7b2ef9a04a542f"
 
-axios@^0.18.0:
+axios@*, axios@^0.18.0:
   version "0.18.0"
   resolved "http://registry.npmjs.org/axios/-/axios-0.18.0.tgz#32d53e4851efdc0a11993b6cd000789d70c05102"
   dependencies: