From 4e6919a6e58d76766098262664c78f92d888337d Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Wed, 19 Sep 2018 17:30:34 -0700 Subject: Add typeorm and begin inserting database records --- packages/pipeline/src/ormconfig.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 packages/pipeline/src/ormconfig.ts (limited to 'packages/pipeline/src/ormconfig.ts') diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts new file mode 100644 index 000000000..48e316078 --- /dev/null +++ b/packages/pipeline/src/ormconfig.ts @@ -0,0 +1,14 @@ +import { ConnectionOptions } from 'typeorm'; + +export const config: ConnectionOptions = { + type: 'sqlite', + database: 'database.sqlite', + synchronize: true, + logging: false, + entities: ['./lib/src/entities/**/*.js'], + migrations: ['./lib/src/migrations/**/*.js'], + cli: { + entitiesDir: 'lib/src/entities', + migrationsDir: 'lib/src/migrations', + }, +}; -- cgit v1.2.3 From 3a19faa5ff8a66395c5eb8d1400338cefaa21187 Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Thu, 18 Oct 2018 17:44:48 -0700 Subject: Update script to work with existing v1 pipeline data --- packages/pipeline/src/ormconfig.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'packages/pipeline/src/ormconfig.ts') diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts index 48e316078..eaafeb7d5 100644 --- a/packages/pipeline/src/ormconfig.ts +++ b/packages/pipeline/src/ormconfig.ts @@ -1,6 +1,6 @@ import { ConnectionOptions } from 'typeorm'; -export const config: ConnectionOptions = { +export const testConfig: ConnectionOptions = { type: 'sqlite', database: 'database.sqlite', synchronize: true, @@ -12,3 +12,16 @@ export const config: ConnectionOptions = { migrationsDir: 'lib/src/migrations', }, }; + +export const deployConfig: ConnectionOptions = { + type: 'postgres', + url: process.env.ZEROEX_DATA_PIPELINE_DB_URL, + synchronize: true, + logging: false, + entities: ['./lib/src/entities/**/*.js'], + migrations: ['./lib/src/migrations/**/*.js'], + cli: { + entitiesDir: 'lib/src/entities', + migrationsDir: 'lib/src/migrations', + }, +}; -- cgit v1.2.3 From 7945d2ea62f3a2bca363f9a37b471e02531490c7 Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Fri, 19 Oct 2018 13:50:48 -0700 Subject: Enable verbose logging --- packages/pipeline/src/ormconfig.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'packages/pipeline/src/ormconfig.ts') diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts index eaafeb7d5..6201d4f37 100644 --- a/packages/pipeline/src/ormconfig.ts +++ b/packages/pipeline/src/ormconfig.ts @@ -17,7 +17,7 @@ export const deployConfig: ConnectionOptions = { type: 'postgres', url: process.env.ZEROEX_DATA_PIPELINE_DB_URL, synchronize: true, - logging: false, + logging: true, entities: ['./lib/src/entities/**/*.js'], migrations: ['./lib/src/migrations/**/*.js'], cli: { -- cgit v1.2.3 From aeff948c9a89b07117a714cc7af1c1a3c810f0b7 Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Fri, 19 Oct 2018 14:05:35 -0700 Subject: Fix bugs in merge_v2_events script and disable verbose logging --- packages/pipeline/src/ormconfig.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'packages/pipeline/src/ormconfig.ts') diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts index 6201d4f37..eaafeb7d5 100644 --- a/packages/pipeline/src/ormconfig.ts +++ b/packages/pipeline/src/ormconfig.ts @@ -17,7 +17,7 @@ export const deployConfig: ConnectionOptions = { type: 'postgres', url: process.env.ZEROEX_DATA_PIPELINE_DB_URL, synchronize: true, - logging: true, + logging: false, entities: ['./lib/src/entities/**/*.js'], migrations: ['./lib/src/migrations/**/*.js'], cli: { -- cgit v1.2.3 From ccad046eb649a60fdf7319a075fa41490d593ae8 Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Thu, 8 Nov 2018 10:15:09 -0800 Subject: Reorganize entities. Make scripts work from any directory. --- packages/pipeline/src/ormconfig.ts | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) (limited to 'packages/pipeline/src/ormconfig.ts') diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts index eaafeb7d5..39b496ace 100644 --- a/packages/pipeline/src/ormconfig.ts +++ b/packages/pipeline/src/ormconfig.ts @@ -1,11 +1,31 @@ import { ConnectionOptions } from 'typeorm'; +import { + Block, + ExchangeCancelEvent, + ExchangeCancelUpToEvent, + ExchangeFillEvent, + Relayer, + SraOrder, + Transaction, +} from './entities'; + +const entities = [ + Block, + ExchangeCancelEvent, + ExchangeCancelUpToEvent, + ExchangeFillEvent, + Relayer, + SraOrder, + Transaction, +]; + export const testConfig: ConnectionOptions = { type: 'sqlite', database: 'database.sqlite', synchronize: true, logging: false, - entities: ['./lib/src/entities/**/*.js'], + entities, migrations: ['./lib/src/migrations/**/*.js'], cli: { entitiesDir: 'lib/src/entities', @@ -18,7 +38,7 @@ export const deployConfig: ConnectionOptions = { url: process.env.ZEROEX_DATA_PIPELINE_DB_URL, synchronize: true, logging: false, - entities: ['./lib/src/entities/**/*.js'], + entities, migrations: ['./lib/src/migrations/**/*.js'], cli: { entitiesDir: 'lib/src/entities', -- cgit v1.2.3 From 329c68f610843ebded9ca31fc9cd6f3eed744a8e Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Mon, 12 Nov 2018 16:40:20 -0800 Subject: Configure TypeORM for migrations. Add new package.json scripts. --- packages/pipeline/src/ormconfig.ts | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) (limited to 'packages/pipeline/src/ormconfig.ts') diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts index 39b496ace..95c27eeba 100644 --- a/packages/pipeline/src/ormconfig.ts +++ b/packages/pipeline/src/ormconfig.ts @@ -20,28 +20,13 @@ const entities = [ Transaction, ]; -export const testConfig: ConnectionOptions = { - type: 'sqlite', - database: 'database.sqlite', - synchronize: true, - logging: false, - entities, - migrations: ['./lib/src/migrations/**/*.js'], - cli: { - entitiesDir: 'lib/src/entities', - migrationsDir: 'lib/src/migrations', - }, -}; - -export const deployConfig: ConnectionOptions = { +const config: ConnectionOptions = { type: 'postgres', url: process.env.ZEROEX_DATA_PIPELINE_DB_URL, - synchronize: true, + synchronize: false, logging: false, entities, - migrations: ['./lib/src/migrations/**/*.js'], - cli: { - entitiesDir: 'lib/src/entities', - migrationsDir: 'lib/src/migrations', - }, + migrations: ['./lib/migrations/**/*.js'], }; + +module.exports = config as ConnectionOptions; -- cgit v1.2.3 From 688d277b30b287f66f0dbd49f2a23cab8b256219 Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Mon, 12 Nov 2018 17:36:33 -0800 Subject: Configure linter with --format stylish and fix linter errors --- packages/pipeline/src/ormconfig.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'packages/pipeline/src/ormconfig.ts') diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts index 95c27eeba..2fb6b3d3c 100644 --- a/packages/pipeline/src/ormconfig.ts +++ b/packages/pipeline/src/ormconfig.ts @@ -29,4 +29,4 @@ const config: ConnectionOptions = { migrations: ['./lib/migrations/**/*.js'], }; -module.exports = config as ConnectionOptions; +module.exports = config; -- cgit v1.2.3 From 26280e4aba147ad6000b9df309e64db84b6932fc Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Tue, 13 Nov 2018 15:33:43 -0800 Subject: Implement scraping sra orders from radar relay --- packages/pipeline/src/ormconfig.ts | 2 ++ 1 file changed, 2 insertions(+) (limited to 'packages/pipeline/src/ormconfig.ts') diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts index 2fb6b3d3c..2f5f7df33 100644 --- a/packages/pipeline/src/ormconfig.ts +++ b/packages/pipeline/src/ormconfig.ts @@ -7,6 +7,7 @@ import { ExchangeFillEvent, Relayer, SraOrder, + SraOrdersObservedTimeStamp, Transaction, } from './entities'; @@ -17,6 +18,7 @@ const entities = [ ExchangeFillEvent, Relayer, SraOrder, + SraOrdersObservedTimeStamp, Transaction, ]; -- cgit v1.2.3 From 303bbc42f4322448998f3fde202574335d1190e6 Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Wed, 14 Nov 2018 15:58:36 -0800 Subject: Change some column types from varchar to numeric --- packages/pipeline/src/ormconfig.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'packages/pipeline/src/ormconfig.ts') diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts index 2f5f7df33..0e489e560 100644 --- a/packages/pipeline/src/ormconfig.ts +++ b/packages/pipeline/src/ormconfig.ts @@ -26,7 +26,7 @@ const config: ConnectionOptions = { type: 'postgres', url: process.env.ZEROEX_DATA_PIPELINE_DB_URL, synchronize: false, - logging: false, + logging: ['error', 'warn'], entities, migrations: ['./lib/migrations/**/*.js'], }; -- cgit v1.2.3 From b42cf0c797661bfc94eb8162817eb8fa118a69a2 Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Wed, 14 Nov 2018 18:39:06 -0800 Subject: Add workaround for broken save method --- packages/pipeline/src/ormconfig.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'packages/pipeline/src/ormconfig.ts') diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts index 0e489e560..64b5b1827 100644 --- a/packages/pipeline/src/ormconfig.ts +++ b/packages/pipeline/src/ormconfig.ts @@ -26,7 +26,7 @@ const config: ConnectionOptions = { type: 'postgres', url: process.env.ZEROEX_DATA_PIPELINE_DB_URL, synchronize: false, - logging: ['error', 'warn'], + // logging: ['error'], entities, migrations: ['./lib/migrations/**/*.js'], }; -- cgit v1.2.3 From c6af5131b0b06433d6294260274e187ad61f4ef7 Mon Sep 17 00:00:00 2001 From: Jake Ellowitz Date: Mon, 19 Nov 2018 16:24:07 -0800 Subject: Pull token metadata re trusted tokens --- packages/pipeline/src/ormconfig.ts | 2 ++ 1 file changed, 2 insertions(+) (limited to 'packages/pipeline/src/ormconfig.ts') diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts index 64b5b1827..1bd337866 100644 --- a/packages/pipeline/src/ormconfig.ts +++ b/packages/pipeline/src/ormconfig.ts @@ -9,6 +9,7 @@ import { SraOrder, SraOrdersObservedTimeStamp, Transaction, + TrustedToken, } from './entities'; const entities = [ @@ -20,6 +21,7 @@ const entities = [ SraOrder, SraOrdersObservedTimeStamp, Transaction, + TrustedToken, ]; const config: ConnectionOptions = { -- cgit v1.2.3 From dea89c4e221d5b22de97b27573719cd27ce250c7 Mon Sep 17 00:00:00 2001 From: Jake Ellowitz Date: Mon, 19 Nov 2018 19:11:51 -0800 Subject: metadata and trusted sources in same raw table --- packages/pipeline/src/ormconfig.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'packages/pipeline/src/ormconfig.ts') diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts index 1bd337866..e8277a439 100644 --- a/packages/pipeline/src/ormconfig.ts +++ b/packages/pipeline/src/ormconfig.ts @@ -8,8 +8,8 @@ import { Relayer, SraOrder, SraOrdersObservedTimeStamp, + TokenMetadata, Transaction, - TrustedToken, } from './entities'; const entities = [ @@ -20,8 +20,8 @@ const entities = [ Relayer, SraOrder, SraOrdersObservedTimeStamp, + TokenMetadata, Transaction, - TrustedToken, ]; const config: ConnectionOptions = { -- cgit v1.2.3 From 7198b441e0d85785eec7244dd60bcd92269d954e Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Thu, 29 Nov 2018 11:40:09 -0800 Subject: Add script for parsing competing dex trades from Bloxy (#1355) --- packages/pipeline/src/ormconfig.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'packages/pipeline/src/ormconfig.ts') diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts index e8277a439..fd6c7c39b 100644 --- a/packages/pipeline/src/ormconfig.ts +++ b/packages/pipeline/src/ormconfig.ts @@ -2,6 +2,7 @@ import { ConnectionOptions } from 'typeorm'; import { Block, + DexTrade, ExchangeCancelEvent, ExchangeCancelUpToEvent, ExchangeFillEvent, @@ -14,6 +15,7 @@ import { const entities = [ Block, + DexTrade, ExchangeCancelEvent, ExchangeCancelUpToEvent, ExchangeFillEvent, @@ -28,7 +30,7 @@ const config: ConnectionOptions = { type: 'postgres', url: process.env.ZEROEX_DATA_PIPELINE_DB_URL, synchronize: false, - // logging: ['error'], + logging: ['error'], entities, migrations: ['./lib/migrations/**/*.js'], }; -- cgit v1.2.3 From 87ffa5d7ab19d2288bf68131a7e7ec77578c564c Mon Sep 17 00:00:00 2001 From: zkao Date: Tue, 4 Dec 2018 13:21:46 -0800 Subject: Token_orderbook_snapshots for Ddex and Paradex(#1354) * Implements the TokenOrderbookSnapshot Table * Scripts, Data Sources and Entities to pull Ddex and Paradex API data. --- packages/pipeline/src/ormconfig.ts | 2 ++ 1 file changed, 2 insertions(+) (limited to 'packages/pipeline/src/ormconfig.ts') diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts index fd6c7c39b..c135c399b 100644 --- a/packages/pipeline/src/ormconfig.ts +++ b/packages/pipeline/src/ormconfig.ts @@ -10,6 +10,7 @@ import { SraOrder, SraOrdersObservedTimeStamp, TokenMetadata, + TokenOrderbookSnapshot, Transaction, } from './entities'; @@ -23,6 +24,7 @@ const entities = [ SraOrder, SraOrdersObservedTimeStamp, TokenMetadata, + TokenOrderbookSnapshot, Transaction, ]; -- cgit v1.2.3 From 8c21a700bae0c751f7f9ca47f9a47628a4478911 Mon Sep 17 00:00:00 2001 From: Xianny <8582774+xianny@users.noreply.github.com> Date: Tue, 4 Dec 2018 13:36:18 -0800 Subject: pull OHLCV records from Crypto Compare (#1349) * [WIP] pull OHLCV records from Crypto Compare * lint * refactor to pull logic out of script and into modules * add entity test for ohlcv_external entity * implement rate limit and chronological backfill for ohlcv * add unit tests; cleanup variable names * Fetch OHLCV pairs params from events table * better method names * fix outdated test * lint * Clean up after review * oops * fix failing test * better filtering of most recent records * fix bug when generating pairs * fix default earliest backfill date * fix bug with retrieving backfill time * prettier --- packages/pipeline/src/ormconfig.ts | 2 ++ 1 file changed, 2 insertions(+) (limited to 'packages/pipeline/src/ormconfig.ts') diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts index c135c399b..9f7815b4e 100644 --- a/packages/pipeline/src/ormconfig.ts +++ b/packages/pipeline/src/ormconfig.ts @@ -6,6 +6,7 @@ import { ExchangeCancelEvent, ExchangeCancelUpToEvent, ExchangeFillEvent, + OHLCVExternal, Relayer, SraOrder, SraOrdersObservedTimeStamp, @@ -20,6 +21,7 @@ const entities = [ ExchangeCancelEvent, ExchangeCancelUpToEvent, ExchangeFillEvent, + OHLCVExternal, Relayer, SraOrder, SraOrdersObservedTimeStamp, -- cgit v1.2.3 From b4cdb14b9b79589d7b24fd7655406c15b6bb00f6 Mon Sep 17 00:00:00 2001 From: Alex Browne Date: Tue, 11 Dec 2018 15:16:05 -0800 Subject: Refactor event scraping and add support for scraping ERC20 approval events (#1401) * Refactor event scraping and add support for scraping ERC20 approval events * Add tests for data_sources/contract-wrappers/utils --- packages/pipeline/src/ormconfig.ts | 2 ++ 1 file changed, 2 insertions(+) (limited to 'packages/pipeline/src/ormconfig.ts') diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts index 9f7815b4e..fe11d81d5 100644 --- a/packages/pipeline/src/ormconfig.ts +++ b/packages/pipeline/src/ormconfig.ts @@ -3,6 +3,7 @@ import { ConnectionOptions } from 'typeorm'; import { Block, DexTrade, + ERC20ApprovalEvent, ExchangeCancelEvent, ExchangeCancelUpToEvent, ExchangeFillEvent, @@ -21,6 +22,7 @@ const entities = [ ExchangeCancelEvent, ExchangeCancelUpToEvent, ExchangeFillEvent, + ERC20ApprovalEvent, OHLCVExternal, Relayer, SraOrder, -- cgit v1.2.3 From 27fc640a9eae70813e9f02cda2c27a4509a04591 Mon Sep 17 00:00:00 2001 From: Xianny <8582774+xianny@users.noreply.github.com> Date: Tue, 8 Jan 2019 13:50:51 -0800 Subject: fetch and save copper (#1472) Fetch and save Copper CRM --- packages/pipeline/src/ormconfig.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'packages/pipeline/src/ormconfig.ts') diff --git a/packages/pipeline/src/ormconfig.ts b/packages/pipeline/src/ormconfig.ts index fe11d81d5..2700714cd 100644 --- a/packages/pipeline/src/ormconfig.ts +++ b/packages/pipeline/src/ormconfig.ts @@ -2,6 +2,11 @@ import { ConnectionOptions } from 'typeorm'; import { Block, + CopperActivity, + CopperActivityType, + CopperCustomField, + CopperLead, + CopperOpportunity, DexTrade, ERC20ApprovalEvent, ExchangeCancelEvent, @@ -18,6 +23,11 @@ import { const entities = [ Block, + CopperOpportunity, + CopperActivity, + CopperActivityType, + CopperCustomField, + CopperLead, DexTrade, ExchangeCancelEvent, ExchangeCancelUpToEvent, -- cgit v1.2.3