From f6d1bfe45bf3709d7bad40bf563b5c09228622e3 Mon Sep 17 00:00:00 2001 From: obscuren Date: Fri, 14 Feb 2014 23:56:09 +0100 Subject: The great merge --- ethdb/.gitignore | 12 +++++++++ ethdb/README.md | 11 +++++++++ ethdb/database.go | 64 ++++++++++++++++++++++++++++++++++++++++++++++++ ethdb/database_test.go | 6 +++++ ethdb/memory_database.go | 49 ++++++++++++++++++++++++++++++++++++ 5 files changed, 142 insertions(+) create mode 100644 ethdb/.gitignore create mode 100644 ethdb/README.md create mode 100644 ethdb/database.go create mode 100644 ethdb/database_test.go create mode 100644 ethdb/memory_database.go (limited to 'ethdb') diff --git a/ethdb/.gitignore b/ethdb/.gitignore new file mode 100644 index 000000000..f725d58d1 --- /dev/null +++ b/ethdb/.gitignore @@ -0,0 +1,12 @@ +# See http://help.github.com/ignore-files/ for more about ignoring files. +# +# If you find yourself ignoring temporary files generated by your text editor +# or operating system, you probably want to add a global ignore instead: +# git config --global core.excludesfile ~/.gitignore_global + +/tmp +*/**/*un~ +*un~ +.DS_Store +*/**/.DS_Store + diff --git a/ethdb/README.md b/ethdb/README.md new file mode 100644 index 000000000..5bed8eedc --- /dev/null +++ b/ethdb/README.md @@ -0,0 +1,11 @@ +# ethdb + +The ethdb package contains the ethereum database interfaces + +# Installation + +`go get github.com/ethereum/ethdb-go` + +# Usage + +Todo :-) diff --git a/ethdb/database.go b/ethdb/database.go new file mode 100644 index 000000000..76e4b4e4d --- /dev/null +++ b/ethdb/database.go @@ -0,0 +1,64 @@ +package ethdb + +import ( + "fmt" + "github.com/ethereum/eth-go/ethutil" + "github.com/syndtr/goleveldb/leveldb" + "path" +) + +type LDBDatabase struct { + db *leveldb.DB +} + +func NewLDBDatabase() (*LDBDatabase, error) { + dbPath := path.Join(ethutil.Config.ExecPath, "database") + + // Open the db + db, err := leveldb.OpenFile(dbPath, nil) + if err != nil { + return nil, err + } + + database := &LDBDatabase{db: db} + + return database, nil +} + +func (db *LDBDatabase) Put(key []byte, value []byte) { + err := db.db.Put(key, value, nil) + if err != nil { + fmt.Println("Error put", err) + } +} + +func (db *LDBDatabase) Get(key []byte) ([]byte, error) { + return db.db.Get(key, nil) +} + +func (db *LDBDatabase) LastKnownTD() []byte { + data, _ := db.db.Get([]byte("LastKnownTotalDifficulty"), nil) + + if len(data) == 0 { + data = []byte{0x0} + } + + return data +} + +func (db *LDBDatabase) Close() { + // Close the leveldb database + db.db.Close() +} + +func (db *LDBDatabase) Print() { + iter := db.db.NewIterator(nil) + for iter.Next() { + key := iter.Key() + value := iter.Value() + + fmt.Printf("%x(%d): ", key, len(key)) + node := ethutil.NewValueFromBytes(value) + fmt.Printf("%v\n", node) + } +} diff --git a/ethdb/database_test.go b/ethdb/database_test.go new file mode 100644 index 000000000..bb1b4de2a --- /dev/null +++ b/ethdb/database_test.go @@ -0,0 +1,6 @@ +package ethdb + +import ( + _ "fmt" + _ "testing" +) diff --git a/ethdb/memory_database.go b/ethdb/memory_database.go new file mode 100644 index 000000000..656de9f0e --- /dev/null +++ b/ethdb/memory_database.go @@ -0,0 +1,49 @@ +package ethdb + +import ( + "fmt" + "github.com/ethereum/eth-go/ethutil" +) + +/* + * This is a test memory database. Do not use for any production it does not get persisted + */ +type MemDatabase struct { + db map[string][]byte +} + +func NewMemDatabase() (*MemDatabase, error) { + db := &MemDatabase{db: make(map[string][]byte)} + + return db, nil +} + +func (db *MemDatabase) Put(key []byte, value []byte) { + db.db[string(key)] = value +} + +func (db *MemDatabase) Get(key []byte) ([]byte, error) { + return db.db[string(key)], nil +} + +func (db *MemDatabase) Print() { + for key, val := range db.db { + fmt.Printf("%x(%d): ", key, len(key)) + dec, _ := ethutil.Decode(val, 0) + node := ethutil.Conv(dec) + fmt.Printf("%q\n", node.AsRaw()) + } +} + +func (db *MemDatabase) Close() { +} + +func (db *MemDatabase) LastKnownTD() []byte { + data, _ := db.Get([]byte("LastKnownTotalDifficulty")) + + if len(data) == 0 || data == nil { + data = []byte{0x0} + } + + return data +} -- cgit v1.2.3 From 73fd358d940418b15dec850f50407bd2e504d88c Mon Sep 17 00:00:00 2001 From: obscuren Date: Sat, 15 Feb 2014 01:34:18 +0100 Subject: Removed RlpValue in favour of Value --- ethdb/memory_database.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'ethdb') diff --git a/ethdb/memory_database.go b/ethdb/memory_database.go index 656de9f0e..cd9f24000 100644 --- a/ethdb/memory_database.go +++ b/ethdb/memory_database.go @@ -29,9 +29,8 @@ func (db *MemDatabase) Get(key []byte) ([]byte, error) { func (db *MemDatabase) Print() { for key, val := range db.db { fmt.Printf("%x(%d): ", key, len(key)) - dec, _ := ethutil.Decode(val, 0) - node := ethutil.Conv(dec) - fmt.Printf("%q\n", node.AsRaw()) + node := ethutil.NewValueFromBytes(val) + fmt.Printf("%q\n", node.Interface()) } } -- cgit v1.2.3 From 55c1c220d07c37610c9aa61c583eccb3c8e3829e Mon Sep 17 00:00:00 2001 From: obscuren Date: Mon, 24 Feb 2014 12:12:01 +0100 Subject: Added delete to database interfaces --- ethdb/database.go | 4 ++++ ethdb/memory_database.go | 6 ++++++ 2 files changed, 10 insertions(+) (limited to 'ethdb') diff --git a/ethdb/database.go b/ethdb/database.go index 76e4b4e4d..1e987920d 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -36,6 +36,10 @@ func (db *LDBDatabase) Get(key []byte) ([]byte, error) { return db.db.Get(key, nil) } +func (db *LDBDatabase) Delete(key []byte) error { + return db.db.Delete(key, nil) +} + func (db *LDBDatabase) LastKnownTD() []byte { data, _ := db.db.Get([]byte("LastKnownTotalDifficulty"), nil) diff --git a/ethdb/memory_database.go b/ethdb/memory_database.go index cd9f24000..9e91eb7d6 100644 --- a/ethdb/memory_database.go +++ b/ethdb/memory_database.go @@ -26,6 +26,12 @@ func (db *MemDatabase) Get(key []byte) ([]byte, error) { return db.db[string(key)], nil } +func (db *MemDatabase) Delete(key []byte) error { + delete(db.db, string(key)) + + return nil +} + func (db *MemDatabase) Print() { for key, val := range db.db { fmt.Printf("%x(%d): ", key, len(key)) -- cgit v1.2.3 From 4f4175a3e295272025d312c800f65fc1a143a9ca Mon Sep 17 00:00:00 2001 From: obscuren Date: Tue, 25 Feb 2014 11:21:03 +0100 Subject: Addad db name for new ldb --- ethdb/database.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'ethdb') diff --git a/ethdb/database.go b/ethdb/database.go index 1e987920d..d1967f023 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -11,8 +11,8 @@ type LDBDatabase struct { db *leveldb.DB } -func NewLDBDatabase() (*LDBDatabase, error) { - dbPath := path.Join(ethutil.Config.ExecPath, "database") +func NewLDBDatabase(name string) (*LDBDatabase, error) { + dbPath := path.Join(ethutil.Config.ExecPath, name) // Open the db db, err := leveldb.OpenFile(dbPath, nil) @@ -40,6 +40,10 @@ func (db *LDBDatabase) Delete(key []byte) error { return db.db.Delete(key, nil) } +func (db *LDBDatabase) Db() *leveldb.DB { + return db.db +} + func (db *LDBDatabase) LastKnownTD() []byte { data, _ := db.db.Get([]byte("LastKnownTotalDifficulty"), nil) -- cgit v1.2.3 From c9f3d1c00ba70016be4bb871f9ecd50d456c6985 Mon Sep 17 00:00:00 2001 From: obscuren Date: Fri, 28 Feb 2014 10:36:06 +0100 Subject: leveldb API changed for NewIterator. Fixes #20 --- ethdb/database.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'ethdb') diff --git a/ethdb/database.go b/ethdb/database.go index d1967f023..3dbff36de 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -54,13 +54,19 @@ func (db *LDBDatabase) LastKnownTD() []byte { return data } +func (db *LDBDatabase) GetKeys() []*ethutil.Key { + data, _ := db.Get([]byte("KeyRing")) + + return []*ethutil.Key{ethutil.NewKeyFromBytes(data)} +} + func (db *LDBDatabase) Close() { // Close the leveldb database db.db.Close() } func (db *LDBDatabase) Print() { - iter := db.db.NewIterator(nil) + iter := db.db.NewIterator(nil, nil) for iter.Next() { key := iter.Key() value := iter.Value() -- cgit v1.2.3 From c0fcefa3a023c643040b3d9e300980bd32c33dff Mon Sep 17 00:00:00 2001 From: obscuren Date: Fri, 28 Feb 2014 12:18:41 +0100 Subject: Added a GetKeys method to support multiple accounts --- ethdb/memory_database.go | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'ethdb') diff --git a/ethdb/memory_database.go b/ethdb/memory_database.go index 9e91eb7d6..b0fa64ed7 100644 --- a/ethdb/memory_database.go +++ b/ethdb/memory_database.go @@ -26,6 +26,12 @@ func (db *MemDatabase) Get(key []byte) ([]byte, error) { return db.db[string(key)], nil } +func (db *MemDatabase) GetKeys() []*ethutil.Key { + data, _ := db.Get([]byte("KeyRing")) + + return []*ethutil.Key{ethutil.NewKeyFromBytes(data)} +} + func (db *MemDatabase) Delete(key []byte) error { delete(db.db, string(key)) -- cgit v1.2.3 From f4fa0d48cb10f925908062357be965c54370cba9 Mon Sep 17 00:00:00 2001 From: obscuren Date: Wed, 14 May 2014 13:54:40 +0200 Subject: Moved keyring to ethutil & removed old methods. Implements #20 --- ethdb/database.go | 2 ++ ethdb/memory_database.go | 2 ++ 2 files changed, 4 insertions(+) (limited to 'ethdb') diff --git a/ethdb/database.go b/ethdb/database.go index 3dbff36de..09e9d8c7d 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -54,11 +54,13 @@ func (db *LDBDatabase) LastKnownTD() []byte { return data } +/* func (db *LDBDatabase) GetKeys() []*ethutil.Key { data, _ := db.Get([]byte("KeyRing")) return []*ethutil.Key{ethutil.NewKeyFromBytes(data)} } +*/ func (db *LDBDatabase) Close() { // Close the leveldb database diff --git a/ethdb/memory_database.go b/ethdb/memory_database.go index b0fa64ed7..1e9d2899a 100644 --- a/ethdb/memory_database.go +++ b/ethdb/memory_database.go @@ -26,11 +26,13 @@ func (db *MemDatabase) Get(key []byte) ([]byte, error) { return db.db[string(key)], nil } +/* func (db *MemDatabase) GetKeys() []*ethutil.Key { data, _ := db.Get([]byte("KeyRing")) return []*ethutil.Key{ethutil.NewKeyFromBytes(data)} } +*/ func (db *MemDatabase) Delete(key []byte) error { delete(db.db, string(key)) -- cgit v1.2.3 From 962255b373a0c8d7c2459cb25aaaa0dc4e492ecf Mon Sep 17 00:00:00 2001 From: obscuren Date: Sat, 23 Aug 2014 11:00:33 +0200 Subject: Removed old code --- ethdb/database.go | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) (limited to 'ethdb') diff --git a/ethdb/database.go b/ethdb/database.go index 09e9d8c7d..e4b069930 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -2,9 +2,10 @@ package ethdb import ( "fmt" + "path" + "github.com/ethereum/eth-go/ethutil" "github.com/syndtr/goleveldb/leveldb" - "path" ) type LDBDatabase struct { @@ -45,7 +46,7 @@ func (db *LDBDatabase) Db() *leveldb.DB { } func (db *LDBDatabase) LastKnownTD() []byte { - data, _ := db.db.Get([]byte("LastKnownTotalDifficulty"), nil) + data, _ := db.db.Get([]byte("LTD"), nil) if len(data) == 0 { data = []byte{0x0} @@ -54,14 +55,6 @@ func (db *LDBDatabase) LastKnownTD() []byte { return data } -/* -func (db *LDBDatabase) GetKeys() []*ethutil.Key { - data, _ := db.Get([]byte("KeyRing")) - - return []*ethutil.Key{ethutil.NewKeyFromBytes(data)} -} -*/ - func (db *LDBDatabase) Close() { // Close the leveldb database db.db.Close() -- cgit v1.2.3 From 76c9c8d653ae49e347598f1bbd34c821354c9567 Mon Sep 17 00:00:00 2001 From: obscuren Date: Mon, 3 Nov 2014 00:31:15 +0100 Subject: Compress data on db level. Closes #174 --- ethdb/database.go | 51 +++++++++++++++++++++++++++++++++----------------- ethdb/database_test.go | 26 +++++++++++++++++++++++-- 2 files changed, 58 insertions(+), 19 deletions(-) (limited to 'ethdb') diff --git a/ethdb/database.go b/ethdb/database.go index de3afd74b..a59782047 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -4,12 +4,15 @@ import ( "fmt" "path" + "github.com/ethereum/go-ethereum/compression/rle" "github.com/ethereum/go-ethereum/ethutil" "github.com/syndtr/goleveldb/leveldb" + "github.com/syndtr/goleveldb/leveldb/iterator" ) type LDBDatabase struct { - db *leveldb.DB + db *leveldb.DB + comp bool } func NewLDBDatabase(name string) (*LDBDatabase, error) { @@ -21,32 +24,42 @@ func NewLDBDatabase(name string) (*LDBDatabase, error) { return nil, err } - database := &LDBDatabase{db: db} + database := &LDBDatabase{db: db, comp: true} return database, nil } -func (db *LDBDatabase) Put(key []byte, value []byte) { - err := db.db.Put(key, value, nil) +func (self *LDBDatabase) Put(key []byte, value []byte) { + if self.comp { + value = rle.Compress(value) + } + + err := self.db.Put(key, value, nil) if err != nil { fmt.Println("Error put", err) } } -func (db *LDBDatabase) Get(key []byte) ([]byte, error) { - return db.db.Get(key, nil) -} +func (self *LDBDatabase) Get(key []byte) ([]byte, error) { + dat, err := self.db.Get(key, nil) + if err != nil { + return nil, err + } + + if self.comp { + //fmt.Println("get", dat) + return rle.Decompress(dat) + } -func (db *LDBDatabase) Delete(key []byte) error { - return db.db.Delete(key, nil) + return dat, nil } -func (db *LDBDatabase) Db() *leveldb.DB { - return db.db +func (self *LDBDatabase) Delete(key []byte) error { + return self.db.Delete(key, nil) } -func (db *LDBDatabase) LastKnownTD() []byte { - data, _ := db.db.Get([]byte("LTD"), nil) +func (self *LDBDatabase) LastKnownTD() []byte { + data, _ := self.Get([]byte("LTD")) if len(data) == 0 { data = []byte{0x0} @@ -55,13 +68,17 @@ func (db *LDBDatabase) LastKnownTD() []byte { return data } -func (db *LDBDatabase) Close() { +func (self *LDBDatabase) NewIterator() iterator.Iterator { + return self.db.NewIterator(nil, nil) +} + +func (self *LDBDatabase) Close() { // Close the leveldb database - db.db.Close() + self.db.Close() } -func (db *LDBDatabase) Print() { - iter := db.db.NewIterator(nil, nil) +func (self *LDBDatabase) Print() { + iter := self.db.NewIterator(nil, nil) for iter.Next() { key := iter.Key() value := iter.Value() diff --git a/ethdb/database_test.go b/ethdb/database_test.go index bb1b4de2a..2cbaf58e0 100644 --- a/ethdb/database_test.go +++ b/ethdb/database_test.go @@ -1,6 +1,28 @@ package ethdb +/* import ( - _ "fmt" - _ "testing" + "bytes" + "testing" ) + +func TestCompression(t *testing.T) { + ethutil.ReadConfig("", "/tmp", "") + + db, err := NewLDBDatabase("testdb") + if err != nil { + t.Fatal(err) + } + + in := make([]byte, 10) + db.Put([]byte("test1"), in) + out, err := db.Get([]byte("test1")) + if err != nil { + t.Fatal(err) + } + + if bytes.Compare(out, in) != 0 { + t.Error("put get", in, out) + } +} +*/ -- cgit v1.2.3 From a82b89e2d524a9b7f758dc2d981e8af835d8cd2a Mon Sep 17 00:00:00 2001 From: obscuren Date: Mon, 3 Nov 2014 23:45:44 +0100 Subject: Added storage root to dump --- ethdb/database.go | 1 - 1 file changed, 1 deletion(-) (limited to 'ethdb') diff --git a/ethdb/database.go b/ethdb/database.go index a59782047..19aa83466 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -47,7 +47,6 @@ func (self *LDBDatabase) Get(key []byte) ([]byte, error) { } if self.comp { - //fmt.Println("get", dat) return rle.Decompress(dat) } -- cgit v1.2.3 From 0f460ad26e864ae8b4c4cf99147c5b57a10f3be9 Mon Sep 17 00:00:00 2001 From: obscuren Date: Wed, 19 Nov 2014 16:21:28 +0100 Subject: Added caching and database interface to trie * Reimplemented caching for trie * Reimplemented resetting and persisting trie --- ethdb/memory_database.go | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'ethdb') diff --git a/ethdb/memory_database.go b/ethdb/memory_database.go index 459373eea..48aa830e7 100644 --- a/ethdb/memory_database.go +++ b/ethdb/memory_database.go @@ -23,6 +23,10 @@ func (db *MemDatabase) Put(key []byte, value []byte) { db.db[string(key)] = value } +func (db *MemDatabase) Set(key []byte, value []byte) { + db.Put(key, value) +} + func (db *MemDatabase) Get(key []byte) ([]byte, error) { return db.db[string(key)], nil } -- cgit v1.2.3