aboutsummaryrefslogtreecommitdiffstats
path: root/cmd/geth/monitorcmd.go
blob: e4ba96a7a681de5c2b891b7edce1683ae82a67ec (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
// Copyright 2015 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.

package main

import (
    "fmt"
    "math"
    "reflect"
    "runtime"
    "sort"
    "strings"
    "time"

    "github.com/ethereum/go-ethereum/cmd/utils"
    "github.com/ethereum/go-ethereum/node"
    "github.com/ethereum/go-ethereum/rpc"
    "github.com/gizak/termui"
    "gopkg.in/urfave/cli.v1"
)

var (
    monitorCommandAttachFlag = cli.StringFlag{
        Name:  "attach",
        Value: node.DefaultIPCEndpoint(clientIdentifier),
        Usage: "API endpoint to attach to",
    }
    monitorCommandRowsFlag = cli.IntFlag{
        Name:  "rows",
        Value: 5,
        Usage: "Maximum rows in the chart grid",
    }
    monitorCommandRefreshFlag = cli.IntFlag{
        Name:  "refresh",
        Value: 3,
        Usage: "Refresh interval in seconds",
    }
    monitorCommand = cli.Command{
        Action:    utils.MigrateFlags(monitor), // keep track of migration progress
        Name:      "monitor",
        Usage:     "Monitor and visualize node metrics",
        ArgsUsage: " ",
        Category:  "MONITOR COMMANDS",
        Description: `
The Geth monitor is a tool to collect and visualize various internal metrics
gathered by the node, supporting different chart types as well as the capacity
to display multiple metrics simultaneously.
`,
        Flags: []cli.Flag{
            monitorCommandAttachFlag,
            monitorCommandRowsFlag,
            monitorCommandRefreshFlag,
        },
    }
)

// monitor starts a terminal UI based monitoring tool for the requested metrics.
func monitor(ctx *cli.Context) error {
    var (
        client *rpc.Client
        err    error
    )
    // Attach to an Ethereum node over IPC or RPC
    endpoint := ctx.String(monitorCommandAttachFlag.Name)
    if client, err = dialRPC(endpoint); err != nil {
        utils.Fatalf("Unable to attach to geth node: %v", err)
    }
    defer client.Close()

    // Retrieve all the available metrics and resolve the user pattens
    metrics, err := retrieveMetrics(client)
    if err != nil {
        utils.Fatalf("Failed to retrieve system metrics: %v", err)
    }
    monitored := resolveMetrics(metrics, ctx.Args())
    if len(monitored) == 0 {
        list := expandMetrics(metrics, "")
        sort.Strings(list)

        if len(list) > 0 {
            utils.Fatalf("No metrics specified.\n\nAvailable:\n - %s", strings.Join(list, "\n - "))
        } else {
            utils.Fatalf("No metrics collected by geth (--%s).\n", utils.MetricsEnabledFlag.Name)
        }
    }
    sort.Strings(monitored)
    if cols := len(monitored) / ctx.Int(monitorCommandRowsFlag.Name); cols > 6 {
        utils.Fatalf("Requested metrics (%d) spans more that 6 columns:\n - %s", len(monitored), strings.Join(monitored, "\n - "))
    }
    // Create and configure the chart UI defaults
    if err := termui.Init(); err != nil {
        utils.Fatalf("Unable to initialize terminal UI: %v", err)
    }
    defer termui.Close()

    rows := len(monitored)
    if max := ctx.Int(monitorCommandRowsFlag.Name); rows > max {
        rows = max
    }
    cols := (len(monitored) + rows - 1) / rows
    for i := 0; i < rows; i++ {
        termui.Body.AddRows(termui.NewRow())
    }
    // Create each individual data chart
    footer := termui.NewPar("")
    footer.Block.Border = true
    footer.Height = 3

    charts := make([]*termui.LineChart, len(monitored))
    units := make([]int, len(monitored))
    data := make([][]float64, len(monitored))
    for i := 0; i < len(monitored); i++ {
        charts[i] = createChart((termui.TermHeight() - footer.Height) / rows)
        row := termui.Body.Rows[i%rows]
        row.Cols = append(row.Cols, termui.NewCol(12/cols, 0, charts[i]))
    }
    termui.Body.AddRows(termui.NewRow(termui.NewCol(12, 0, footer)))

    refreshCharts(client, monitored, data, units, charts, ctx, footer)
    termui.Body.Align()
    termui.Render(termui.Body)

    // Watch for various system events, and periodically refresh the charts
    termui.Handle("/sys/kbd/C-c", func(termui.Event) {
        termui.StopLoop()
    })
    termui.Handle("/sys/wnd/resize", func(termui.Event) {
        termui.Body.Width = termui.TermWidth()
        for _, chart := range charts {
            chart.Height = (termui.TermHeight() - footer.Height) / rows
        }
        termui.Body.Align()
        termui.Render(termui.Body)
    })
    go func() {
        tick := time.NewTicker(time.Duration(ctx.Int(monitorCommandRefreshFlag.Name)) * time.Second)
        for range tick.C {
            if refreshCharts(client, monitored, data, units, charts, ctx, footer) {
                termui.Body.Align()
            }
            termui.Render(termui.Body)
        }
    }()
    termui.Loop()
    return nil
}

// retrieveMetrics contacts the attached geth node and retrieves the entire set
// of collected system metrics.
func retrieveMetrics(client *rpc.Client) (map[string]interface{}, error) {
    var metrics map[string]interface{}
    err := client.Call(&metrics, "debug_metrics", true)
    return metrics, err
}

// resolveMetrics takes a list of input metric patterns, and resolves each to one
// or more canonical metric names.
func resolveMetrics(metrics map[string]interface{}, patterns []string) []string {
    res := []string{}
    for _, pattern := range patterns {
        res = append(res, resolveMetric(metrics, pattern, "")...)
    }
    return res
}

// resolveMetrics takes a single of input metric pattern, and resolves it to one
// or more canonical metric names.
func resolveMetric(metrics map[string]interface{}, pattern string, path string) []string {
    results := []string{}

    // If a nested metric was requested, recurse optionally branching (via comma)
    parts := strings.SplitN(pattern, "/", 2)
    if len(parts) > 1 {
        for _, variation := range strings.Split(parts[0], ",") {
            submetrics, ok := metrics[variation].(map[string]interface{})
            if !ok {
                utils.Fatalf("Failed to retrieve system metrics: %s", path+variation)
                return nil
            }
            results = append(results, resolveMetric(submetrics, parts[1], path+variation+"/")...)
        }
        return results
    }
    // Depending what the last link is, return or expand
    for _, variation := range strings.Split(pattern, ",") {
        switch metric := metrics[variation].(type) {
        case float64:
            // Final metric value found, return as singleton
            results = append(results, path+variation)

        case map[string]interface{}:
            results = append(results, expandMetrics(metric, path+variation+"/")...)

        default:
            utils.Fatalf("Metric pattern resolved to unexpected type: %v", reflect.TypeOf(metric))
            return nil
        }
    }
    return results
}

// expandMetrics expands the entire tree of metrics into a flat list of paths.
func expandMetrics(metrics map[string]interface{}, path string) []string {
    // Iterate over all fields and expand individually
    list := []string{}
    for name, metric := range metrics {
        switch metric := metric.(type) {
        case float64:
            // Final metric value found, append to list
            list = append(list, path+name)

        case map[string]interface{}:
            // Tree of metrics found, expand recursively
            list = append(list, expandMetrics(metric, path+name+"/")...)

        default:
            utils.Fatalf("Metric pattern %s resolved to unexpected type: %v", path+name, reflect.TypeOf(metric))
            return nil
        }
    }
    return list
}

// fetchMetric iterates over the metrics map and retrieves a specific one.
func fetchMetric(metrics map[string]interface{}, metric string) float64 {
    parts := strings.Split(metric, "/")
    for _, part := range parts[:len(parts)-1] {
        var found bool
        metrics, found = metrics[part].(map[string]interface{})
        if !found {
            return 0
        }
    }
    if v, ok := metrics[parts[len(parts)-1]].(float64); ok {
        return v
    }
    return 0
}

// refreshCharts retrieves a next batch of metrics, and inserts all the new
// values into the active datasets and charts
func refreshCharts(client *rpc.Client, metrics []string, data [][]float64, units []int, charts []*termui.LineChart, ctx *cli.Context, footer *termui.Par) (realign bool) {
    values, err := retrieveMetrics(client)
    for i, metric := range metrics {
        if len(data) < 512 {
            data[i] = append([]float64{fetchMetric(values, metric)}, data[i]...)
        } else {
            data[i] = append([]float64{fetchMetric(values, metric)}, data[i][:len(data[i])-1]...)
        }
        if updateChart(metric, data[i], &units[i], charts[i], err) {
            realign = true
        }
    }
    updateFooter(ctx, err, footer)
    return
}

// updateChart inserts a dataset into a line chart, scaling appropriately as to
// not display weird labels, also updating the chart label accordingly.
func updateChart(metric string, data []float64, base *int, chart *termui.LineChart, err error) (realign bool) {
    dataUnits := []string{"", "K", "M", "G", "T", "E"}
    timeUnits := []string{"ns", "µs", "ms", "s", "ks", "ms"}
    colors := []termui.Attribute{termui.ColorBlue, termui.ColorCyan, termui.ColorGreen, termui.ColorYellow, termui.ColorRed, termui.ColorRed}

    // Extract only part of the data that's actually visible
    if chart.Width*2 < len(data) {
        data = data[:chart.Width*2]
    }
    // Find the maximum value and scale under 1K
    high := 0.0
    if len(data) > 0 {
        high = data[0]
        for _, value := range data[1:] {
            high = math.Max(high, value)
        }
    }
    unit, scale := 0, 1.0
    for high >= 1000 && unit+1 < len(dataUnits) {
        high, unit, scale = high/1000, unit+1, scale*1000
    }
    // If the unit changes, re-create the chart (hack to set max height...)
    if unit != *base {
        realign, *base, *chart = true, unit, *createChart(chart.Height)
    }
    // Update the chart's data points with the scaled values
    if cap(chart.Data) < len(data) {
        chart.Data = make([]float64, len(data))
    }
    chart.Data = chart.Data[:len(data)]
    for i, value := range data {
        chart.Data[i] = value / scale
    }
    // Update the chart's label with the scale units
    units := dataUnits
    if strings.Contains(metric, "/Percentiles/") || strings.Contains(metric, "/pauses/") || strings.Contains(metric, "/time/") {
        units = timeUnits
    }
    chart.BorderLabel = metric
    if len(units[unit]) > 0 {
        chart.BorderLabel += " [" + units[unit] + "]"
    }
    chart.LineColor = colors[unit] | termui.AttrBold
    if err != nil {
        chart.LineColor = termui.ColorRed | termui.AttrBold
    }
    return
}

// createChart creates an empty line chart with the default configs.
func createChart(height int) *termui.LineChart {
    chart := termui.NewLineChart()
    if runtime.GOOS == "windows" {
        chart.Mode = "dot"
    }
    chart.DataLabels = []string{""}
    chart.Height = height
    chart.AxesColor = termui.ColorWhite
    chart.PaddingBottom = -2

    chart.BorderLabelFg = chart.BorderFg | termui.AttrBold
    chart.BorderFg = chart.BorderBg

    return chart
}

// updateFooter updates the footer contents based on any encountered errors.
func updateFooter(ctx *cli.Context, err error, footer *termui.Par) {
    // Generate the basic footer
    refresh := time.Duration(ctx.Int(monitorCommandRefreshFlag.Name)) * time.Second
    footer.Text = fmt.Sprintf("Press Ctrl+C to quit. Refresh interval: %v.", refresh)
    footer.TextFgColor = termui.ThemeAttr("par.fg") | termui.AttrBold

    // Append any encountered errors
    if err != nil {
        footer.Text = fmt.Sprintf("Error: %v.", err)
        footer.TextFgColor = termui.ColorRed | termui.AttrBold
    }
}