pull OHLCV records from Crypto Compare (#1349)

* [WIP] pull OHLCV records from Crypto Compare

* lint

* refactor to pull logic out of script and into modules

* add entity test for ohlcv_external entity

* implement rate limit and chronological backfill for ohlcv

* add unit tests; cleanup variable names

* Fetch OHLCV pairs params from events table

* better method names

* fix outdated test

* lint

* Clean up after review

* oops

* fix failing test

* better filtering of most recent records

* fix bug when generating pairs

* fix default earliest backfill date

* fix bug with retrieving backfill time

* prettier

1 files changed, 17 insertions, 7 deletions

diff --git a/packages/pipeline/src/entities/ohlcv_external.ts b/packages/pipeline/src/entities/ohlcv_external.ts
index 95cd4f2f5..4f55dd930 100644
--- a/packages/pipeline/src/entities/ohlcv_external.ts
+++ b/packages/pipeline/src/entities/ohlcv_external.ts
@@ -1,20 +1,30 @@
 import { Column, Entity, PrimaryColumn } from 'typeorm';
 
+import { numberToBigIntTransformer } from '../utils';
+
 @Entity({ name: 'ohlcv_external', schema: 'raw' })
 export class OHLCVExternal {
     @PrimaryColumn() public exchange!: string;
-    @PrimaryColumn() public fromSymbol!: string;
-    @PrimaryColumn() public toSymbol!: string;
-    @PrimaryColumn() public startTime!: number;
-    @PrimaryColumn() public endTime!: number;
+
+    @PrimaryColumn({ name: 'from_symbol', type: 'varchar' })
+    public fromSymbol!: string;
+    @PrimaryColumn({ name: 'to_symbol', type: 'varchar' })
+    public toSymbol!: string;
+    @PrimaryColumn({ name: 'start_time', transformer: numberToBigIntTransformer })
+    public startTime!: number;
+    @PrimaryColumn({ name: 'end_time', transformer: numberToBigIntTransformer })
+    public endTime!: number;
 
     @Column() public open!: number;
     @Column() public close!: number;
     @Column() public low!: number;
     @Column() public high!: number;
-    @Column() public volumeFrom!: number;
-    @Column() public volumeTo!: number;
+    @Column({ name: 'volume_from' })
+    public volumeFrom!: number;
+    @Column({ name: 'volume_to' })
+    public volumeTo!: number;
 
     @PrimaryColumn() public source!: string;
-    @PrimaryColumn() public observedTimestamp!: number;
+    @PrimaryColumn({ name: 'observed_timestamp', transformer: numberToBigIntTransformer })
+    public observedTimestamp!: number;
 }