From 43e4a6501beb42bc0f4fffcbcb1e3f5fe58b6062 Mon Sep 17 00:00:00 2001 From: Péter Szilágyi Date: Mon, 22 Jun 2015 12:01:27 +0300 Subject: core, ethdb: instrument the block and state db Conflicts: ethdb/database.go --- ethdb/database.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) (limited to 'ethdb') diff --git a/ethdb/database.go b/ethdb/database.go index 5faf8c4e0..48cb37f7c 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -4,6 +4,7 @@ import ( "github.com/ethereum/go-ethereum/compression/rle" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" + "github.com/rcrowley/go-metrics" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/errors" "github.com/syndtr/goleveldb/leveldb/iterator" @@ -13,10 +14,11 @@ import ( var OpenFileLimit = 64 type LDBDatabase struct { - // filename for reporting - fn string - // LevelDB instance - db *leveldb.DB + fn string // filename for reporting + db *leveldb.DB // LevelDB instance + + GetMeter metrics.Meter // Meter for measuring the database get requests + PutMeter metrics.Meter // Meter for measuring the database put requests } // NewLDBDatabase returns a LevelDB wrapped object. LDBDatabase does not persist data by @@ -43,7 +45,11 @@ func NewLDBDatabase(file string) (*LDBDatabase, error) { // Put puts the given key / value to the queue func (self *LDBDatabase) Put(key []byte, value []byte) error { - return self.db.Put(key, rle.Compress(value), nil) + dat := rle.Compress(value) + if self.PutMeter != nil { + self.PutMeter.Mark(int64(len(dat))) + } + return self.db.Put(key, dat, nil) } // Get returns the given key if it's present. @@ -52,6 +58,9 @@ func (self *LDBDatabase) Get(key []byte) ([]byte, error) { if err != nil { return nil, err } + if self.GetMeter != nil { + self.GetMeter.Mark(int64(len(dat))) + } return rle.Decompress(dat) } -- cgit v1.2.3 From 792b0ddccdf135a8d9c33298bcb20d7bafb79a0b Mon Sep 17 00:00:00 2001 From: Péter Szilágyi Date: Tue, 23 Jun 2015 13:03:33 +0300 Subject: core, eth, eth/fetcher, ethdb: polish metrics gathering a bit --- core/chain_manager.go | 14 -------------- eth/backend.go | 19 +++++++++++++++++++ eth/fetcher/fetcher.go | 10 +++++----- ethdb/database.go | 20 ++++++++++++++++---- 4 files changed, 40 insertions(+), 23 deletions(-) (limited to 'ethdb') diff --git a/core/chain_manager.go b/core/chain_manager.go index b87f65893..3b9b7517b 100644 --- a/core/chain_manager.go +++ b/core/chain_manager.go @@ -13,7 +13,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" @@ -31,10 +30,6 @@ var ( blockNumPre = []byte("block-num-") blockInsertTimer = metrics.GetOrRegisterTimer("core/BlockInsertions", metrics.DefaultRegistry) - blockdbGetMeter = metrics.GetOrRegisterMeter("core/blockdb/Gets", metrics.DefaultRegistry) - blockdbPutMeter = metrics.GetOrRegisterMeter("core/blockdb/Puts", metrics.DefaultRegistry) - statedbGetMeter = metrics.GetOrRegisterMeter("core/statedb/Gets", metrics.DefaultRegistry) - statedbPutMeter = metrics.GetOrRegisterMeter("core/statedb/Puts", metrics.DefaultRegistry) ) const ( @@ -126,15 +121,6 @@ func NewChainManager(genesis *types.Block, blockDb, stateDb common.Database, pow cache: NewBlockCache(blockCacheLimit), pow: pow, } - // Instrument the block and state databases - if db, ok := blockDb.(*ethdb.LDBDatabase); ok { - db.GetMeter = blockdbGetMeter - db.PutMeter = blockdbPutMeter - } - if db, ok := stateDb.(*ethdb.LDBDatabase); ok { - db.GetMeter = statedbGetMeter - db.PutMeter = statedbPutMeter - } // Check the genesis block given to the chain manager. If the genesis block mismatches block number 0 // throw an error. If no block or the same block's found continue. if g := bc.GetBlockByNumber(0); g != nil && g.Hash() != genesis.Hash() { diff --git a/eth/backend.go b/eth/backend.go index 37fe66abf..5969c238c 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/p2p/nat" "github.com/ethereum/go-ethereum/whisper" + "github.com/rcrowley/go-metrics" ) const ( @@ -248,14 +249,32 @@ func New(config *Config) (*Ethereum, error) { if err != nil { return nil, fmt.Errorf("blockchain db err: %v", err) } + blockDb.(*ethdb.LDBDatabase).GetMeter = metrics.GetOrRegisterMeter("eth/db/block/Gets", metrics.DefaultRegistry) + blockDb.(*ethdb.LDBDatabase).PutMeter = metrics.GetOrRegisterMeter("eth/db/block/Puts", metrics.DefaultRegistry) + blockDb.(*ethdb.LDBDatabase).DelMeter = metrics.GetOrRegisterMeter("eth/db/block/Dels", metrics.DefaultRegistry) + blockDb.(*ethdb.LDBDatabase).ReadMeter = metrics.GetOrRegisterMeter("eth/db/block/Reads", metrics.DefaultRegistry) + blockDb.(*ethdb.LDBDatabase).WriteMeter = metrics.GetOrRegisterMeter("eth/db/block/Writes", metrics.DefaultRegistry) + stateDb, err := newdb(filepath.Join(config.DataDir, "state")) if err != nil { return nil, fmt.Errorf("state db err: %v", err) } + stateDb.(*ethdb.LDBDatabase).GetMeter = metrics.GetOrRegisterMeter("eth/db/state/Gets", metrics.DefaultRegistry) + stateDb.(*ethdb.LDBDatabase).PutMeter = metrics.GetOrRegisterMeter("eth/db/state/Puts", metrics.DefaultRegistry) + stateDb.(*ethdb.LDBDatabase).DelMeter = metrics.GetOrRegisterMeter("eth/db/state/Dels", metrics.DefaultRegistry) + stateDb.(*ethdb.LDBDatabase).ReadMeter = metrics.GetOrRegisterMeter("eth/db/state/Reads", metrics.DefaultRegistry) + stateDb.(*ethdb.LDBDatabase).WriteMeter = metrics.GetOrRegisterMeter("eth/db/state/Writes", metrics.DefaultRegistry) + extraDb, err := newdb(filepath.Join(config.DataDir, "extra")) if err != nil { return nil, fmt.Errorf("extra db err: %v", err) } + extraDb.(*ethdb.LDBDatabase).GetMeter = metrics.GetOrRegisterMeter("eth/db/extra/Gets", metrics.DefaultRegistry) + extraDb.(*ethdb.LDBDatabase).PutMeter = metrics.GetOrRegisterMeter("eth/db/extra/Puts", metrics.DefaultRegistry) + extraDb.(*ethdb.LDBDatabase).DelMeter = metrics.GetOrRegisterMeter("eth/db/extra/Dels", metrics.DefaultRegistry) + extraDb.(*ethdb.LDBDatabase).ReadMeter = metrics.GetOrRegisterMeter("eth/db/extra/Reads", metrics.DefaultRegistry) + extraDb.(*ethdb.LDBDatabase).WriteMeter = metrics.GetOrRegisterMeter("eth/db/extra/Writes", metrics.DefaultRegistry) + nodeDb := filepath.Join(config.DataDir, "nodes") // Perform database sanity checks diff --git a/eth/fetcher/fetcher.go b/eth/fetcher/fetcher.go index 13c65121b..07a32b9f1 100644 --- a/eth/fetcher/fetcher.go +++ b/eth/fetcher/fetcher.go @@ -126,11 +126,11 @@ func New(getBlock blockRetrievalFn, validateBlock blockValidatorFn, broadcastBlo chainHeight: chainHeight, insertChain: insertChain, dropPeer: dropPeer, - announceMeter: metrics.GetOrRegisterMeter("eth/RemoteAnnounces", metrics.DefaultRegistry), - announceTimer: metrics.GetOrRegisterTimer("eth/LocalAnnounces", metrics.DefaultRegistry), - broadcastMeter: metrics.GetOrRegisterMeter("eth/RemoteBroadcasts", metrics.DefaultRegistry), - broadcastTimer: metrics.GetOrRegisterTimer("eth/LocalBroadcasts", metrics.DefaultRegistry), - discardMeter: metrics.GetOrRegisterMeter("eth/DiscardedBlocks", metrics.DefaultRegistry), + announceMeter: metrics.GetOrRegisterMeter("eth/sync/RemoteAnnounces", metrics.DefaultRegistry), + announceTimer: metrics.GetOrRegisterTimer("eth/sync/LocalAnnounces", metrics.DefaultRegistry), + broadcastMeter: metrics.GetOrRegisterMeter("eth/sync/RemoteBroadcasts", metrics.DefaultRegistry), + broadcastTimer: metrics.GetOrRegisterTimer("eth/sync/LocalBroadcasts", metrics.DefaultRegistry), + discardMeter: metrics.GetOrRegisterMeter("eth/sync/DiscardedBlocks", metrics.DefaultRegistry), } } diff --git a/ethdb/database.go b/ethdb/database.go index 48cb37f7c..13b4ef449 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -17,8 +17,11 @@ type LDBDatabase struct { fn string // filename for reporting db *leveldb.DB // LevelDB instance - GetMeter metrics.Meter // Meter for measuring the database get requests - PutMeter metrics.Meter // Meter for measuring the database put requests + GetMeter metrics.Meter // Meter for measuring the database get request counts + PutMeter metrics.Meter // Meter for measuring the database put request counts + DelMeter metrics.Meter // Meter for measuring the database delete request counts + ReadMeter metrics.Meter // Meter for measuring the database get request data usage + WriteMeter metrics.Meter // Meter for measuring the database put request data usage } // NewLDBDatabase returns a LevelDB wrapped object. LDBDatabase does not persist data by @@ -47,7 +50,10 @@ func NewLDBDatabase(file string) (*LDBDatabase, error) { func (self *LDBDatabase) Put(key []byte, value []byte) error { dat := rle.Compress(value) if self.PutMeter != nil { - self.PutMeter.Mark(int64(len(dat))) + self.PutMeter.Mark(1) + } + if self.WriteMeter != nil { + self.WriteMeter.Mark(int64(len(dat))) } return self.db.Put(key, dat, nil) } @@ -59,13 +65,19 @@ func (self *LDBDatabase) Get(key []byte) ([]byte, error) { return nil, err } if self.GetMeter != nil { - self.GetMeter.Mark(int64(len(dat))) + self.GetMeter.Mark(1) + } + if self.ReadMeter != nil { + self.ReadMeter.Mark(int64(len(dat))) } return rle.Decompress(dat) } // Delete deletes the key from the queue and database func (self *LDBDatabase) Delete(key []byte) error { + if self.DelMeter != nil { + self.DelMeter.Mark(1) + } return self.db.Delete(key, nil) } -- cgit v1.2.3 From 803b3c4a825ed0ac5d22c93fc3159651b8c590b2 Mon Sep 17 00:00:00 2001 From: Péter Szilágyi Date: Tue, 23 Jun 2015 18:36:08 +0300 Subject: eth, ethdb: measure database operation latencies too --- eth/backend.go | 18 +++++++++--------- ethdb/database.go | 39 ++++++++++++++++++++++++++++----------- 2 files changed, 37 insertions(+), 20 deletions(-) (limited to 'ethdb') diff --git a/eth/backend.go b/eth/backend.go index cef0fe3ab..0da6f128c 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -250,9 +250,9 @@ func New(config *Config) (*Ethereum, error) { return nil, fmt.Errorf("blockchain db err: %v", err) } if db, ok := blockDb.(*ethdb.LDBDatabase); ok { - db.GetMeter = metrics.GetOrRegisterMeter("eth/db/block/Gets", metrics.DefaultRegistry) - db.PutMeter = metrics.GetOrRegisterMeter("eth/db/block/Puts", metrics.DefaultRegistry) - db.DelMeter = metrics.GetOrRegisterMeter("eth/db/block/Dels", metrics.DefaultRegistry) + db.GetTimer = metrics.GetOrRegisterTimer("eth/db/block/Gets", metrics.DefaultRegistry) + db.PutTimer = metrics.GetOrRegisterTimer("eth/db/block/Puts", metrics.DefaultRegistry) + db.MissMeter = metrics.GetOrRegisterMeter("eth/db/block/Misses", metrics.DefaultRegistry) db.ReadMeter = metrics.GetOrRegisterMeter("eth/db/block/Reads", metrics.DefaultRegistry) db.WriteMeter = metrics.GetOrRegisterMeter("eth/db/block/Writes", metrics.DefaultRegistry) } @@ -261,9 +261,9 @@ func New(config *Config) (*Ethereum, error) { return nil, fmt.Errorf("state db err: %v", err) } if db, ok := stateDb.(*ethdb.LDBDatabase); ok { - db.GetMeter = metrics.GetOrRegisterMeter("eth/db/state/Gets", metrics.DefaultRegistry) - db.PutMeter = metrics.GetOrRegisterMeter("eth/db/state/Puts", metrics.DefaultRegistry) - db.DelMeter = metrics.GetOrRegisterMeter("eth/db/state/Dels", metrics.DefaultRegistry) + db.GetTimer = metrics.GetOrRegisterTimer("eth/db/state/Gets", metrics.DefaultRegistry) + db.PutTimer = metrics.GetOrRegisterTimer("eth/db/state/Puts", metrics.DefaultRegistry) + db.MissMeter = metrics.GetOrRegisterMeter("eth/db/state/Misses", metrics.DefaultRegistry) db.ReadMeter = metrics.GetOrRegisterMeter("eth/db/state/Reads", metrics.DefaultRegistry) db.WriteMeter = metrics.GetOrRegisterMeter("eth/db/state/Writes", metrics.DefaultRegistry) } @@ -272,9 +272,9 @@ func New(config *Config) (*Ethereum, error) { return nil, fmt.Errorf("extra db err: %v", err) } if db, ok := extraDb.(*ethdb.LDBDatabase); ok { - db.GetMeter = metrics.GetOrRegisterMeter("eth/db/extra/Gets", metrics.DefaultRegistry) - db.PutMeter = metrics.GetOrRegisterMeter("eth/db/extra/Puts", metrics.DefaultRegistry) - db.DelMeter = metrics.GetOrRegisterMeter("eth/db/extra/Dels", metrics.DefaultRegistry) + db.GetTimer = metrics.GetOrRegisterTimer("eth/db/extra/Gets", metrics.DefaultRegistry) + db.PutTimer = metrics.GetOrRegisterTimer("eth/db/extra/Puts", metrics.DefaultRegistry) + db.MissMeter = metrics.GetOrRegisterMeter("eth/db/extra/Misses", metrics.DefaultRegistry) db.ReadMeter = metrics.GetOrRegisterMeter("eth/db/extra/Reads", metrics.DefaultRegistry) db.WriteMeter = metrics.GetOrRegisterMeter("eth/db/extra/Writes", metrics.DefaultRegistry) } diff --git a/ethdb/database.go b/ethdb/database.go index 13b4ef449..32ce845a0 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -1,6 +1,8 @@ package ethdb import ( + "time" + "github.com/ethereum/go-ethereum/compression/rle" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" @@ -17,9 +19,10 @@ type LDBDatabase struct { fn string // filename for reporting db *leveldb.DB // LevelDB instance - GetMeter metrics.Meter // Meter for measuring the database get request counts - PutMeter metrics.Meter // Meter for measuring the database put request counts - DelMeter metrics.Meter // Meter for measuring the database delete request counts + GetTimer metrics.Timer // Timer for measuring the database get request counts and latencies + PutTimer metrics.Timer // Timer for measuring the database put request counts and latencies + DelTimer metrics.Timer // Timer for measuring the database delete request counts and latencies + MissMeter metrics.Meter // MEter for measuring the missed database get requests ReadMeter metrics.Meter // Meter for measuring the database get request data usage WriteMeter metrics.Meter // Meter for measuring the database put request data usage } @@ -48,10 +51,14 @@ func NewLDBDatabase(file string) (*LDBDatabase, error) { // Put puts the given key / value to the queue func (self *LDBDatabase) Put(key []byte, value []byte) error { - dat := rle.Compress(value) - if self.PutMeter != nil { - self.PutMeter.Mark(1) + // Measure the database put latency, if requested + if self.PutTimer != nil { + start := time.Now() + defer self.PutTimer.UpdateSince(start) } + // Generate the data to write to disk, update the meter and write + dat := rle.Compress(value) + if self.WriteMeter != nil { self.WriteMeter.Mark(int64(len(dat))) } @@ -60,13 +67,20 @@ func (self *LDBDatabase) Put(key []byte, value []byte) error { // Get returns the given key if it's present. func (self *LDBDatabase) Get(key []byte) ([]byte, error) { + // Measure the database get latency, if requested + if self.GetTimer != nil { + start := time.Now() + defer self.GetTimer.UpdateSince(start) + } + // Retrieve the key and increment the miss counter if not found dat, err := self.db.Get(key, nil) if err != nil { + if self.MissMeter != nil { + self.MissMeter.Mark(1) + } return nil, err } - if self.GetMeter != nil { - self.GetMeter.Mark(1) - } + // Otherwise update the actually retrieved amount of data if self.ReadMeter != nil { self.ReadMeter.Mark(int64(len(dat))) } @@ -75,9 +89,12 @@ func (self *LDBDatabase) Get(key []byte) ([]byte, error) { // Delete deletes the key from the queue and database func (self *LDBDatabase) Delete(key []byte) error { - if self.DelMeter != nil { - self.DelMeter.Mark(1) + // Measure the database delete latency, if requested + if self.DelTimer != nil { + start := time.Now() + defer self.DelTimer.UpdateSince(start) } + // Execute the actual operation return self.db.Delete(key, nil) } -- cgit v1.2.3