packages/pipeline/src/run_jobs.ts


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87

import { exec } from 'child_process';

import { postgresClient } from './postgres.js';
import { dataFetchingQueries } from './scripts/query_data.js';
import { web3, zrx } from './zrx.js';
const CUR_BLOCK_OFFSET = 20;
postgresClient.query(dataFetchingQueries.get_max_block, []).then((data: any) => {
    const maxBlockNumber = data.rows[0].max;
    const safeCurBlockNumber = web3.eth.blockNumber - CUR_BLOCK_OFFSET;
    console.log('Scraping ' + maxBlockNumber + ' to ' + safeCurBlockNumber);
    exec(
        'node ./lib/scripts/scrape_data --type events --from ' + maxBlockNumber + ' --to ' + safeCurBlockNumber,
        (error, stdout, stderr) => {
            if (error) {
                console.log(error);
                return;
            }
            console.log('Scraped events');
            console.log('Scraping blocks');
            exec(
                'node ./lib/scripts/scrape_data --type blocks --from ' + maxBlockNumber + ' --to ' + safeCurBlockNumber,
                (error, stdout, stderr) => {
                    if (error) {
                        console.log(error);
                        return;
                    }
                    console.log('Scraped blocks');
                    console.log('Scraping transactions');
                    exec(
                        'node ./lib/scripts/scrape_data --type transactions --from ' +
                            maxBlockNumber +
                            ' --to ' +
                            safeCurBlockNumber,
                        (error, stdout, stderr) => {
                            if (error) {
                                console.log(error);
                                return;
                            }
                            console.log('Scraped transactions');
                            console.log('Joining events_staging');
                            exec(
                                'node ./lib/scripts/join_tables --name events_staging --from ' +
                                    maxBlockNumber +
                                    ' --to ' +
                                    safeCurBlockNumber,
                                (error, stdout, stderr) => {
                                    if (error) {
                                        console.log(error);
                                        return;
                                    }
                                    console.log('Joined events_staging');
                                    console.log('Joining events');
                                    exec(
                                        'node ./lib/scripts/join_tables --name events --from ' +
                                            maxBlockNumber +
                                            ' --to ' +
                                            safeCurBlockNumber,
                                        (error, stdout, stderr) => {
                                            if (error) {
                                                console.log(error);
                                                return;
                                            }
                                            console.log('Joined events');
                                            console.log('Joining events_full');
                                            exec(
                                                'node ./lib/scripts/join_tables --name events_full --from ' +
                                                    maxBlockNumber +
                                                    ' --to ' +
                                                    safeCurBlockNumber,
                                                (error, stdout, stderr) => {
                                                    if (error) {
                                                        console.log(error);
                                                        return;
                                                    }
                                                },
                                            );
                                        },
                                    );
                                },
                            );
                        },
                    );
                },
            );
        },
    );
});