From c7e7778f2a7d80fa12643db546db98fa70f2e384 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 22 Jul 2015 13:46:20 +0300 Subject: cmd, core, eth, ethdb: cache flag to allocate memory for db internal use --- ethdb/database.go | 31 +++++++++++++++++++++++++------ ethdb/database_test.go | 3 +-- 2 files changed, 26 insertions(+), 8 deletions(-) (limited to 'ethdb') diff --git a/ethdb/database.go b/ethdb/database.go index c75136a1b..38e454c00 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -17,6 +17,7 @@ package ethdb import ( + "path/filepath" "strconv" "strings" "sync" @@ -36,6 +37,14 @@ import ( var OpenFileLimit = 64 +// cacheRatio specifies how the total alloted cache is distributed between the +// various system databases. +var cacheRatio = map[string]float64{ + "blockchain": 1.0 / 13.0, + "extra": 2.0 / 13.0, + "state": 10.0 / 13.0, +} + type LDBDatabase struct { fn string // filename for reporting db *leveldb.DB // LevelDB instance @@ -57,14 +66,24 @@ type LDBDatabase struct { // NewLDBDatabase returns a LevelDB wrapped object. LDBDatabase does not persist data by // it self but requires a background poller which syncs every X. `Flush` should be called // when data needs to be stored and written to disk. -func NewLDBDatabase(file string) (*LDBDatabase, error) { - // Open the db - db, err := leveldb.OpenFile(file, &opt.Options{OpenFilesCacheCapacity: OpenFileLimit}) - // check for corruption and attempt to recover - if _, iscorrupted := err.(*errors.ErrCorrupted); iscorrupted { +func NewLDBDatabase(file string, cache int) (*LDBDatabase, error) { + // Calculate the cache allowance for this particular database + cache = int(float64(cache) * cacheRatio[filepath.Base(file)]) + if cache < 16 { + cache = 16 + } + glog.V(logger.Info).Infof("Alloted %dMB cache to %s", cache, file) + + // Open the db and recover any potential corruptions + db, err := leveldb.OpenFile(file, &opt.Options{ + OpenFilesCacheCapacity: OpenFileLimit, + BlockCacheCapacity: cache / 2 * opt.MiB, + WriteBuffer: cache / 4 * opt.MiB, // Two of these are used internally + }) + if _, corrupted := err.(*errors.ErrCorrupted); corrupted { db, err = leveldb.RecoverFile(file, nil) } - // (re) check for errors and abort if opening of the db failed + // (Re)check for errors and abort if opening of the db failed if err != nil { return nil, err } diff --git a/ethdb/database_test.go b/ethdb/database_test.go index 29292d016..41947a698 100644 --- a/ethdb/database_test.go +++ b/ethdb/database_test.go @@ -28,8 +28,7 @@ func newDb() *LDBDatabase { if common.FileExist(file) { os.RemoveAll(file) } - - db, _ := NewLDBDatabase(file) + db, _ := NewLDBDatabase(file, 0) return db } -- cgit v1.2.3 From 3f047be5aa93b6222506445414ca909dd59c7eeb Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 22 Jul 2015 18:48:40 +0200 Subject: all: update license headers to distiguish GPL/LGPL All code outside of cmd/ is licensed as LGPL. The headers now reflect this by calling the whole work "the go-ethereum library". --- ethdb/database.go | 8 ++++---- ethdb/database_test.go | 8 ++++---- ethdb/memory_database.go | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) (limited to 'ethdb') diff --git a/ethdb/database.go b/ethdb/database.go index c75136a1b..febf8d9f9 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -1,18 +1,18 @@ // Copyright 2014 The go-ethereum Authors -// This file is part of go-ethereum. +// This file is part of the go-ethereum library. // // go-ethereum is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// go-ethereum is distributed in the hope that it will be useful, +// The go-ethereum library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with go-ethereum. If not, see . +// along with the go-ethereum library. If not, see . package ethdb diff --git a/ethdb/database_test.go b/ethdb/database_test.go index 29292d016..ce2ca48d0 100644 --- a/ethdb/database_test.go +++ b/ethdb/database_test.go @@ -1,18 +1,18 @@ // Copyright 2014 The go-ethereum Authors -// This file is part of go-ethereum. +// This file is part of the go-ethereum library. // // go-ethereum is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// go-ethereum is distributed in the hope that it will be useful, +// The go-ethereum library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with go-ethereum. If not, see . +// along with the go-ethereum library. If not, see . package ethdb diff --git a/ethdb/memory_database.go b/ethdb/memory_database.go index 3fba9f406..3dcb9fa98 100644 --- a/ethdb/memory_database.go +++ b/ethdb/memory_database.go @@ -1,18 +1,18 @@ // Copyright 2014 The go-ethereum Authors -// This file is part of go-ethereum. +// This file is part of the go-ethereum library. // // go-ethereum is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// go-ethereum is distributed in the hope that it will be useful, +// The go-ethereum library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with go-ethereum. If not, see . +// along with the go-ethereum library. If not, see . package ethdb -- cgit v1.2.3 From 8f56eea77d9d4cd0218c9b0ca7b8c0501780b62b Mon Sep 17 00:00:00 2001 From: Jeffrey Wilcke Date: Mon, 20 Jul 2015 11:44:41 +0200 Subject: ethdb, trie: removed RLE compression --- ethdb/database.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'ethdb') diff --git a/ethdb/database.go b/ethdb/database.go index 9c9dfe9a1..bf27df592 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -23,7 +23,6 @@ import ( "sync" "time" - "github.com/ethereum/go-ethereum/compression/rle" "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/metrics" @@ -100,12 +99,12 @@ func (self *LDBDatabase) Put(key []byte, value []byte) error { defer self.putTimer.UpdateSince(time.Now()) } // Generate the data to write to disk, update the meter and write - dat := rle.Compress(value) + //value = rle.Compress(value) if self.writeMeter != nil { - self.writeMeter.Mark(int64(len(dat))) + self.writeMeter.Mark(int64(len(value))) } - return self.db.Put(key, dat, nil) + return self.db.Put(key, value, nil) } // Get returns the given key if it's present. @@ -126,7 +125,8 @@ func (self *LDBDatabase) Get(key []byte) ([]byte, error) { if self.readMeter != nil { self.readMeter.Mark(int64(len(dat))) } - return rle.Decompress(dat) + return dat, nil + //return rle.Decompress(dat) } // Delete deletes the key from the queue and database -- cgit v1.2.3 From bfbcfbe4a9dd9125391b56d6a13158cc5ec71c89 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 23 Jul 2015 18:35:11 +0200 Subject: all: fix license headers one more time I forgot to update one instance of "go-ethereum" in commit 3f047be5a. --- ethdb/database.go | 2 +- ethdb/database_test.go | 2 +- ethdb/memory_database.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'ethdb') diff --git a/ethdb/database.go b/ethdb/database.go index febf8d9f9..c93344409 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -1,7 +1,7 @@ // Copyright 2014 The go-ethereum Authors // This file is part of the go-ethereum library. // -// go-ethereum is free software: you can redistribute it and/or modify +// The go-ethereum library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. diff --git a/ethdb/database_test.go b/ethdb/database_test.go index ce2ca48d0..2465036aa 100644 --- a/ethdb/database_test.go +++ b/ethdb/database_test.go @@ -1,7 +1,7 @@ // Copyright 2014 The go-ethereum Authors // This file is part of the go-ethereum library. // -// go-ethereum is free software: you can redistribute it and/or modify +// The go-ethereum library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. diff --git a/ethdb/memory_database.go b/ethdb/memory_database.go index 3dcb9fa98..70b03dfad 100644 --- a/ethdb/memory_database.go +++ b/ethdb/memory_database.go @@ -1,7 +1,7 @@ // Copyright 2014 The go-ethereum Authors // This file is part of the go-ethereum library. // -// go-ethereum is free software: you can redistribute it and/or modify +// The go-ethereum library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. -- cgit v1.2.3