From 43e4a6501beb42bc0f4fffcbcb1e3f5fe58b6062 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 22 Jun 2015 12:01:27 +0300 Subject: core, ethdb: instrument the block and state db Conflicts: ethdb/database.go --- ethdb/database.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) (limited to 'ethdb') diff --git a/ethdb/database.go b/ethdb/database.go index 5faf8c4e0..48cb37f7c 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -4,6 +4,7 @@ import ( "github.com/ethereum/go-ethereum/compression/rle" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" + "github.com/rcrowley/go-metrics" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/iterator" @@ -13,10 +14,11 @@ import ( var OpenFileLimit = 64 type LDBDatabase struct { - // filename for reporting - fn string - // LevelDB instance - db *leveldb.DB + fn string // filename for reporting + db *leveldb.DB // LevelDB instance + + GetMeter metrics.Meter // Meter for measuring the database get requests + PutMeter metrics.Meter // Meter for measuring the database put requests } // NewLDBDatabase returns a LevelDB wrapped object. LDBDatabase does not persist data by @@ -43,7 +45,11 @@ func NewLDBDatabase(file string) (*LDBDatabase, error) { // Put puts the given key / value to the queue func (self *LDBDatabase) Put(key []byte, value []byte) error { - return self.db.Put(key, rle.Compress(value), nil) + dat := rle.Compress(value) + if self.PutMeter != nil { + self.PutMeter.Mark(int64(len(dat))) + } + return self.db.Put(key, dat, nil) } // Get returns the given key if it's present. @@ -52,6 +58,9 @@ func (self *LDBDatabase) Get(key []byte) ([]byte, error) { if err != nil { return nil, err } + if self.GetMeter != nil { + self.GetMeter.Mark(int64(len(dat))) + } return rle.Decompress(dat) } -- cgit v1.2.3 From 792b0ddccdf135a8d9c33298bcb20d7bafb79a0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 23 Jun 2015 13:03:33 +0300 Subject: core, eth, eth/fetcher, ethdb: polish metrics gathering a bit --- ethdb/database.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) (limited to 'ethdb') diff --git a/ethdb/database.go b/ethdb/database.go index 48cb37f7c..13b4ef449 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -17,8 +17,11 @@ type LDBDatabase struct { fn string // filename for reporting db *leveldb.DB // LevelDB instance - GetMeter metrics.Meter // Meter for measuring the database get requests - PutMeter metrics.Meter // Meter for measuring the database put requests + GetMeter metrics.Meter // Meter for measuring the database get request counts + PutMeter metrics.Meter // Meter for measuring the database put request counts + DelMeter metrics.Meter // Meter for measuring the database delete request counts + ReadMeter metrics.Meter // Meter for measuring the database get request data usage + WriteMeter metrics.Meter // Meter for measuring the database put request data usage } // NewLDBDatabase returns a LevelDB wrapped object. LDBDatabase does not persist data by @@ -47,7 +50,10 @@ func NewLDBDatabase(file string) (*LDBDatabase, error) { func (self *LDBDatabase) Put(key []byte, value []byte) error { dat := rle.Compress(value) if self.PutMeter != nil { - self.PutMeter.Mark(int64(len(dat))) + self.PutMeter.Mark(1) + } + if self.WriteMeter != nil { + self.WriteMeter.Mark(int64(len(dat))) } return self.db.Put(key, dat, nil) } @@ -59,13 +65,19 @@ func (self *LDBDatabase) Get(key []byte) ([]byte, error) { return nil, err } if self.GetMeter != nil { - self.GetMeter.Mark(int64(len(dat))) + self.GetMeter.Mark(1) + } + if self.ReadMeter != nil { + self.ReadMeter.Mark(int64(len(dat))) } return rle.Decompress(dat) } // Delete deletes the key from the queue and database func (self *LDBDatabase) Delete(key []byte) error { + if self.DelMeter != nil { + self.DelMeter.Mark(1) + } return self.db.Delete(key, nil) } -- cgit v1.2.3 From 803b3c4a825ed0ac5d22c93fc3159651b8c590b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 23 Jun 2015 18:36:08 +0300 Subject: eth, ethdb: measure database operation latencies too --- ethdb/database.go | 39 ++++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) (limited to 'ethdb') diff --git a/ethdb/database.go b/ethdb/database.go index 13b4ef449..32ce845a0 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -1,6 +1,8 @@ package ethdb import ( + "time" + "github.com/ethereum/go-ethereum/compression/rle" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" @@ -17,9 +19,10 @@ type LDBDatabase struct { fn string // filename for reporting db *leveldb.DB // LevelDB instance - GetMeter metrics.Meter // Meter for measuring the database get request counts - PutMeter metrics.Meter // Meter for measuring the database put request counts - DelMeter metrics.Meter // Meter for measuring the database delete request counts + GetTimer metrics.Timer // Timer for measuring the database get request counts and latencies + PutTimer metrics.Timer // Timer for measuring the database put request counts and latencies + DelTimer metrics.Timer // Timer for measuring the database delete request counts and latencies + MissMeter metrics.Meter // MEter for measuring the missed database get requests ReadMeter metrics.Meter // Meter for measuring the database get request data usage WriteMeter metrics.Meter // Meter for measuring the database put request data usage } @@ -48,10 +51,14 @@ func NewLDBDatabase(file string) (*LDBDatabase, error) { // Put puts the given key / value to the queue func (self *LDBDatabase) Put(key []byte, value []byte) error { - dat := rle.Compress(value) - if self.PutMeter != nil { - self.PutMeter.Mark(1) + // Measure the database put latency, if requested + if self.PutTimer != nil { + start := time.Now() + defer self.PutTimer.UpdateSince(start) } + // Generate the data to write to disk, update the meter and write + dat := rle.Compress(value) + if self.WriteMeter != nil { self.WriteMeter.Mark(int64(len(dat))) } @@ -60,13 +67,20 @@ func (self *LDBDatabase) Put(key []byte, value []byte) error { // Get returns the given key if it's present. func (self *LDBDatabase) Get(key []byte) ([]byte, error) { + // Measure the database get latency, if requested + if self.GetTimer != nil { + start := time.Now() + defer self.GetTimer.UpdateSince(start) + } + // Retrieve the key and increment the miss counter if not found dat, err := self.db.Get(key, nil) if err != nil { + if self.MissMeter != nil { + self.MissMeter.Mark(1) + } return nil, err } - if self.GetMeter != nil { - self.GetMeter.Mark(1) - } + // Otherwise update the actually retrieved amount of data if self.ReadMeter != nil { self.ReadMeter.Mark(int64(len(dat))) } @@ -75,9 +89,12 @@ func (self *LDBDatabase) Get(key []byte) ([]byte, error) { // Delete deletes the key from the queue and database func (self *LDBDatabase) Delete(key []byte) error { - if self.DelMeter != nil { - self.DelMeter.Mark(1) + // Measure the database delete latency, if requested + if self.DelTimer != nil { + start := time.Now() + defer self.DelTimer.UpdateSince(start) } + // Execute the actual operation return self.db.Delete(key, nil) } -- cgit v1.2.3 From 2a5a55efafa39a21a5c70afcc85dfb00c6002028 Mon Sep 17 00:00:00 2001 From: obscuren Date: Fri, 19 Jun 2015 16:20:49 +0200 Subject: ethdb: accessor for LDB. TODO remove this interface --- ethdb/database.go | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'ethdb') diff --git a/ethdb/database.go b/ethdb/database.go index 32ce845a0..8f55d43c3 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -115,3 +115,7 @@ func (self *LDBDatabase) Close() { self.db.Close() glog.V(logger.Error).Infoln("flushed and closed db:", self.fn) } + +func (self *LDBDatabase) LDB() *leveldb.DB { + return self.db +} -- cgit v1.2.3 From ccbb56b4f2cdba352eaa859ce3e34f999287f5c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Sat, 27 Jun 2015 20:03:31 +0300 Subject: cmd/geth, eth, ethdb: monitor database compactions --- ethdb/database.go | 97 +++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 84 insertions(+), 13 deletions(-) (limited to 'ethdb') diff --git a/ethdb/database.go b/ethdb/database.go index 8f55d43c3..9ff90d167 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -1,6 +1,8 @@ package ethdb import ( + "strconv" + "strings" "time" "github.com/ethereum/go-ethereum/compression/rle" @@ -19,12 +21,15 @@ type LDBDatabase struct { fn string // filename for reporting db *leveldb.DB // LevelDB instance - GetTimer metrics.Timer // Timer for measuring the database get request counts and latencies - PutTimer metrics.Timer // Timer for measuring the database put request counts and latencies - DelTimer metrics.Timer // Timer for measuring the database delete request counts and latencies - MissMeter metrics.Meter // MEter for measuring the missed database get requests - ReadMeter metrics.Meter // Meter for measuring the database get request data usage - WriteMeter metrics.Meter // Meter for measuring the database put request data usage + GetTimer metrics.Timer // Timer for measuring the database get request counts and latencies + PutTimer metrics.Timer // Timer for measuring the database put request counts and latencies + DelTimer metrics.Timer // Timer for measuring the database delete request counts and latencies + MissMeter metrics.Meter // Meter for measuring the missed database get requests + ReadMeter metrics.Meter // Meter for measuring the database get request data usage + WriteMeter metrics.Meter // Meter for measuring the database put request data usage + CompTimeMeter metrics.Meter // Meter for measuring the total time spent in database compaction + CompReadMeter metrics.Meter // Meter for measuring the data read during compaction + CompWriteMeter metrics.Meter // Meter for measuring the data written during compaction } // NewLDBDatabase returns a LevelDB wrapped object. LDBDatabase does not persist data by @@ -45,6 +50,7 @@ func NewLDBDatabase(file string) (*LDBDatabase, error) { fn: file, db: db, } + go database.meter(3 * time.Second) return database, nil } @@ -53,8 +59,7 @@ func NewLDBDatabase(file string) (*LDBDatabase, error) { func (self *LDBDatabase) Put(key []byte, value []byte) error { // Measure the database put latency, if requested if self.PutTimer != nil { - start := time.Now() - defer self.PutTimer.UpdateSince(start) + defer self.PutTimer.UpdateSince(time.Now()) } // Generate the data to write to disk, update the meter and write dat := rle.Compress(value) @@ -69,8 +74,7 @@ func (self *LDBDatabase) Put(key []byte, value []byte) error { func (self *LDBDatabase) Get(key []byte) ([]byte, error) { // Measure the database get latency, if requested if self.GetTimer != nil { - start := time.Now() - defer self.GetTimer.UpdateSince(start) + defer self.GetTimer.UpdateSince(time.Now()) } // Retrieve the key and increment the miss counter if not found dat, err := self.db.Get(key, nil) @@ -91,8 +95,7 @@ func (self *LDBDatabase) Get(key []byte) ([]byte, error) { func (self *LDBDatabase) Delete(key []byte) error { // Measure the database delete latency, if requested if self.DelTimer != nil { - start := time.Now() - defer self.DelTimer.UpdateSince(start) + defer self.DelTimer.UpdateSince(time.Now()) } // Execute the actual operation return self.db.Delete(key, nil) @@ -111,7 +114,6 @@ func (self *LDBDatabase) Close() { if err := self.Flush(); err != nil { glog.V(logger.Error).Infof("error: flush '%s': %v\n", self.fn, err) } - self.db.Close() glog.V(logger.Error).Infoln("flushed and closed db:", self.fn) } @@ -119,3 +121,72 @@ func (self *LDBDatabase) Close() { func (self *LDBDatabase) LDB() *leveldb.DB { return self.db } + +// meter periodically retrieves internal leveldb counters and reports them to +// the metrics subsystem. +// +// This is how a stats table look like (currently): +// Compactions +// Level | Tables | Size(MB) | Time(sec) | Read(MB) | Write(MB) +// -------+------------+---------------+---------------+---------------+--------------- +// 0 | 0 | 0.00000 | 1.27969 | 0.00000 | 12.31098 +// 1 | 85 | 109.27913 | 28.09293 | 213.92493 | 214.26294 +// 2 | 523 | 1000.37159 | 7.26059 | 66.86342 | 66.77884 +// 3 | 570 | 1113.18458 | 0.00000 | 0.00000 | 0.00000 +func (self *LDBDatabase) meter(refresh time.Duration) { + // Create the counters to store current and previous values + counters := make([][]float64, 2) + for i := 0; i < 2; i++ { + counters[i] = make([]float64, 3) + } + // Iterate ad infinitum and collect the stats + for i := 1; ; i++ { + // Retrieve the database stats + stats, err := self.db.GetProperty("leveldb.stats") + if err != nil { + glog.V(logger.Error).Infof("failed to read database stats: %v", err) + return + } + // Find the compaction table, skip the header + lines := strings.Split(stats, "\n") + for len(lines) > 0 && strings.TrimSpace(lines[0]) != "Compactions" { + lines = lines[1:] + } + if len(lines) <= 3 { + glog.V(logger.Error).Infof("compaction table not found") + return + } + lines = lines[3:] + + // Iterate over all the table rows, and accumulate the entries + for j := 0; j < len(counters[i%2]); j++ { + counters[i%2][j] = 0 + } + for _, line := range lines { + parts := strings.Split(line, "|") + if len(parts) != 6 { + break + } + for idx, counter := range parts[3:] { + if value, err := strconv.ParseFloat(strings.TrimSpace(counter), 64); err != nil { + glog.V(logger.Error).Infof("compaction entry parsing failed: %v", err) + return + } else { + counters[i%2][idx] += value + } + } + } + // Update all the requested meters + if self.CompTimeMeter != nil { + self.CompTimeMeter.Mark(int64((counters[i%2][0] - counters[(i-1)%2][0]) * 1000 * 1000 * 1000)) + } + if self.CompReadMeter != nil { + self.CompReadMeter.Mark(int64((counters[i%2][1] - counters[(i-1)%2][1]) * 1024 * 1024)) + } + if self.CompWriteMeter != nil { + self.CompWriteMeter.Mark(int64((counters[i%2][2] - counters[(i-1)%2][2]) * 1024 * 1024)) + } + // Sleep a bit, then repeat the stats collection + time.Sleep(refresh) + } +} -- cgit v1.2.3