aboutsummaryrefslogtreecommitdiffstats
path: root/consensus/ethash/ethash.go
diff options
context:
space:
mode:
authorPéter Szilágyi <peterke@gmail.com>2017-04-05 06:16:29 +0800
committerFelix Lange <fjl@users.noreply.github.com>2017-04-05 06:16:29 +0800
commit09777952ee476ff80d4b6e63b5041ff5ca0e441b (patch)
treee85320f88f548201e3476b3e7095e96fd071617b /consensus/ethash/ethash.go
parente50a5b77712d891ff409aa942a5cbc24e721b332 (diff)
downloadgo-tangerine-09777952ee476ff80d4b6e63b5041ff5ca0e441b.tar
go-tangerine-09777952ee476ff80d4b6e63b5041ff5ca0e441b.tar.gz
go-tangerine-09777952ee476ff80d4b6e63b5041ff5ca0e441b.tar.bz2
go-tangerine-09777952ee476ff80d4b6e63b5041ff5ca0e441b.tar.lz
go-tangerine-09777952ee476ff80d4b6e63b5041ff5ca0e441b.tar.xz
go-tangerine-09777952ee476ff80d4b6e63b5041ff5ca0e441b.tar.zst
go-tangerine-09777952ee476ff80d4b6e63b5041ff5ca0e441b.zip
core, consensus: pluggable consensus engines (#3817)
This commit adds pluggable consensus engines to go-ethereum. In short, it introduces a generic consensus interface, and refactors the entire codebase to use this interface.
Diffstat (limited to 'consensus/ethash/ethash.go')
-rw-r--r--consensus/ethash/ethash.go587
1 files changed, 587 insertions, 0 deletions
diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go
new file mode 100644
index 000000000..aa5b2d8a0
--- /dev/null
+++ b/consensus/ethash/ethash.go
@@ -0,0 +1,587 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package ethash implements the ethash proof-of-work consensus engine.
+package ethash
+
+import (
+ "errors"
+ "fmt"
+ "math"
+ "math/big"
+ "math/rand"
+ "os"
+ "path/filepath"
+ "reflect"
+ "strconv"
+ "sync"
+ "time"
+ "unsafe"
+
+ mmap "github.com/edsrzf/mmap-go"
+ "github.com/ethereum/go-ethereum/consensus"
+ "github.com/ethereum/go-ethereum/log"
+ "github.com/ethereum/go-ethereum/rpc"
+ metrics "github.com/rcrowley/go-metrics"
+)
+
+var ErrInvalidDumpMagic = errors.New("invalid dump magic")
+
+var (
+ // maxUint256 is a big integer representing 2^256-1
+ maxUint256 = new(big.Int).Exp(big.NewInt(2), big.NewInt(256), big.NewInt(0))
+
+ // sharedEthash is a full instance that can be shared between multiple users.
+ sharedEthash = New("", 3, 0, "", 1, 0)
+
+ // algorithmRevision is the data structure version used for file naming.
+ algorithmRevision = 23
+
+ // dumpMagic is a dataset dump header to sanity check a data dump.
+ dumpMagic = []uint32{0xbaddcafe, 0xfee1dead}
+)
+
+// isLittleEndian returns whether the local system is running in little or big
+// endian byte order.
+func isLittleEndian() bool {
+ n := uint32(0x01020304)
+ return *(*byte)(unsafe.Pointer(&n)) == 0x04
+}
+
+// memoryMap tries to memory map a file of uint32s for read only access.
+func memoryMap(path string) (*os.File, mmap.MMap, []uint32, error) {
+ file, err := os.OpenFile(path, os.O_RDONLY, 0644)
+ if err != nil {
+ return nil, nil, nil, err
+ }
+ mem, buffer, err := memoryMapFile(file, false)
+ if err != nil {
+ file.Close()
+ return nil, nil, nil, err
+ }
+ for i, magic := range dumpMagic {
+ if buffer[i] != magic {
+ mem.Unmap()
+ file.Close()
+ return nil, nil, nil, ErrInvalidDumpMagic
+ }
+ }
+ return file, mem, buffer[len(dumpMagic):], err
+}
+
+// memoryMapFile tries to memory map an already opened file descriptor.
+func memoryMapFile(file *os.File, write bool) (mmap.MMap, []uint32, error) {
+ // Try to memory map the file
+ flag := mmap.RDONLY
+ if write {
+ flag = mmap.RDWR
+ }
+ mem, err := mmap.Map(file, flag, 0)
+ if err != nil {
+ return nil, nil, err
+ }
+ // Yay, we managed to memory map the file, here be dragons
+ header := *(*reflect.SliceHeader)(unsafe.Pointer(&mem))
+ header.Len /= 4
+ header.Cap /= 4
+
+ return mem, *(*[]uint32)(unsafe.Pointer(&header)), nil
+}
+
+// memoryMapAndGenerate tries to memory map a temporary file of uint32s for write
+// access, fill it with the data from a generator and then move it into the final
+// path requested.
+func memoryMapAndGenerate(path string, size uint64, generator func(buffer []uint32)) (*os.File, mmap.MMap, []uint32, error) {
+ // Ensure the data folder exists
+ if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+ return nil, nil, nil, err
+ }
+ // Create a huge temporary empty file to fill with data
+ temp := path + "." + strconv.Itoa(rand.Int())
+
+ dump, err := os.Create(temp)
+ if err != nil {
+ return nil, nil, nil, err
+ }
+ if err = dump.Truncate(int64(len(dumpMagic))*4 + int64(size)); err != nil {
+ return nil, nil, nil, err
+ }
+ // Memory map the file for writing and fill it with the generator
+ mem, buffer, err := memoryMapFile(dump, true)
+ if err != nil {
+ dump.Close()
+ return nil, nil, nil, err
+ }
+ copy(buffer, dumpMagic)
+
+ data := buffer[len(dumpMagic):]
+ generator(data)
+
+ if err := mem.Flush(); err != nil {
+ mem.Unmap()
+ dump.Close()
+ return nil, nil, nil, err
+ }
+ os.Rename(temp, path)
+ return dump, mem, data, nil
+}
+
+// cache wraps an ethash cache with some metadata to allow easier concurrent use.
+type cache struct {
+ epoch uint64 // Epoch for which this cache is relevant
+
+ dump *os.File // File descriptor of the memory mapped cache
+ mmap mmap.MMap // Memory map itself to unmap before releasing
+
+ cache []uint32 // The actual cache data content (may be memory mapped)
+ used time.Time // Timestamp of the last use for smarter eviction
+ once sync.Once // Ensures the cache is generated only once
+ lock sync.Mutex // Ensures thread safety for updating the usage time
+}
+
+// generate ensures that the cache content is generated before use.
+func (c *cache) generate(dir string, limit int, test bool) {
+ c.once.Do(func() {
+ // If we have a testing cache, generate and return
+ if test {
+ c.cache = make([]uint32, 1024/4)
+ generateCache(c.cache, c.epoch, seedHash(c.epoch*epochLength+1))
+ return
+ }
+ // If we don't store anything on disk, generate and return
+ size := cacheSize(c.epoch*epochLength + 1)
+ seed := seedHash(c.epoch*epochLength + 1)
+
+ if dir == "" {
+ c.cache = make([]uint32, size/4)
+ generateCache(c.cache, c.epoch, seed)
+ return
+ }
+ // Disk storage is needed, this will get fancy
+ var endian string
+ if !isLittleEndian() {
+ endian = ".be"
+ }
+ path := filepath.Join(dir, fmt.Sprintf("cache-R%d-%x%s", algorithmRevision, seed[:8], endian))
+ logger := log.New("epoch", c.epoch)
+
+ // Try to load the file from disk and memory map it
+ var err error
+ c.dump, c.mmap, c.cache, err = memoryMap(path)
+ if err == nil {
+ logger.Debug("Loaded old ethash cache from disk")
+ return
+ }
+ logger.Debug("Failed to load old ethash cache", "err", err)
+
+ // No previous cache available, create a new cache file to fill
+ c.dump, c.mmap, c.cache, err = memoryMapAndGenerate(path, size, func(buffer []uint32) { generateCache(buffer, c.epoch, seed) })
+ if err != nil {
+ logger.Error("Failed to generate mapped ethash cache", "err", err)
+
+ c.cache = make([]uint32, size/4)
+ generateCache(c.cache, c.epoch, seed)
+ }
+ // Iterate over all previous instances and delete old ones
+ for ep := int(c.epoch) - limit; ep >= 0; ep-- {
+ seed := seedHash(uint64(ep)*epochLength + 1)
+ path := filepath.Join(dir, fmt.Sprintf("cache-R%d-%x%s", algorithmRevision, seed[:8], endian))
+ os.Remove(path)
+ }
+ })
+}
+
+// release closes any file handlers and memory maps open.
+func (c *cache) release() {
+ if c.mmap != nil {
+ c.mmap.Unmap()
+ c.mmap = nil
+ }
+ if c.dump != nil {
+ c.dump.Close()
+ c.dump = nil
+ }
+}
+
+// dataset wraps an ethash dataset with some metadata to allow easier concurrent use.
+type dataset struct {
+ epoch uint64 // Epoch for which this cache is relevant
+
+ dump *os.File // File descriptor of the memory mapped cache
+ mmap mmap.MMap // Memory map itself to unmap before releasing
+
+ dataset []uint32 // The actual cache data content
+ used time.Time // Timestamp of the last use for smarter eviction
+ once sync.Once // Ensures the cache is generated only once
+ lock sync.Mutex // Ensures thread safety for updating the usage time
+}
+
+// generate ensures that the dataset content is generated before use.
+func (d *dataset) generate(dir string, limit int, test bool) {
+ d.once.Do(func() {
+ // If we have a testing dataset, generate and return
+ if test {
+ cache := make([]uint32, 1024/4)
+ generateCache(cache, d.epoch, seedHash(d.epoch*epochLength+1))
+
+ d.dataset = make([]uint32, 32*1024/4)
+ generateDataset(d.dataset, d.epoch, cache)
+
+ return
+ }
+ // If we don't store anything on disk, generate and return
+ csize := cacheSize(d.epoch*epochLength + 1)
+ dsize := datasetSize(d.epoch*epochLength + 1)
+ seed := seedHash(d.epoch*epochLength + 1)
+
+ if dir == "" {
+ cache := make([]uint32, csize/4)
+ generateCache(cache, d.epoch, seed)
+
+ d.dataset = make([]uint32, dsize/4)
+ generateDataset(d.dataset, d.epoch, cache)
+ }
+ // Disk storage is needed, this will get fancy
+ var endian string
+ if !isLittleEndian() {
+ endian = ".be"
+ }
+ path := filepath.Join(dir, fmt.Sprintf("full-R%d-%x%s", algorithmRevision, seed[:8], endian))
+ logger := log.New("epoch", d.epoch)
+
+ // Try to load the file from disk and memory map it
+ var err error
+ d.dump, d.mmap, d.dataset, err = memoryMap(path)
+ if err == nil {
+ logger.Debug("Loaded old ethash dataset from disk")
+ return
+ }
+ logger.Debug("Failed to load old ethash dataset", "err", err)
+
+ // No previous dataset available, create a new dataset file to fill
+ cache := make([]uint32, csize/4)
+ generateCache(cache, d.epoch, seed)
+
+ d.dump, d.mmap, d.dataset, err = memoryMapAndGenerate(path, dsize, func(buffer []uint32) { generateDataset(buffer, d.epoch, cache) })
+ if err != nil {
+ logger.Error("Failed to generate mapped ethash dataset", "err", err)
+
+ d.dataset = make([]uint32, dsize/2)
+ generateDataset(d.dataset, d.epoch, cache)
+ }
+ // Iterate over all previous instances and delete old ones
+ for ep := int(d.epoch) - limit; ep >= 0; ep-- {
+ seed := seedHash(uint64(ep)*epochLength + 1)
+ path := filepath.Join(dir, fmt.Sprintf("full-R%d-%x%s", algorithmRevision, seed[:8], endian))
+ os.Remove(path)
+ }
+ })
+}
+
+// release closes any file handlers and memory maps open.
+func (d *dataset) release() {
+ if d.mmap != nil {
+ d.mmap.Unmap()
+ d.mmap = nil
+ }
+ if d.dump != nil {
+ d.dump.Close()
+ d.dump = nil
+ }
+}
+
+// MakeCache generates a new ethash cache and optionally stores it to disk.
+func MakeCache(block uint64, dir string) {
+ c := cache{epoch: block/epochLength + 1}
+ c.generate(dir, math.MaxInt32, false)
+ c.release()
+}
+
+// MakeDataset generates a new ethash dataset and optionally stores it to disk.
+func MakeDataset(block uint64, dir string) {
+ d := dataset{epoch: block/epochLength + 1}
+ d.generate(dir, math.MaxInt32, false)
+ d.release()
+}
+
+// Ethash is a consensus engine based on proot-of-work implementing the ethash
+// algorithm.
+type Ethash struct {
+ cachedir string // Data directory to store the verification caches
+ cachesinmem int // Number of caches to keep in memory
+ cachesondisk int // Number of caches to keep on disk
+ dagdir string // Data directory to store full mining datasets
+ dagsinmem int // Number of mining datasets to keep in memory
+ dagsondisk int // Number of mining datasets to keep on disk
+
+ caches map[uint64]*cache // In memory caches to avoid regenerating too often
+ fcache *cache // Pre-generated cache for the estimated future epoch
+ datasets map[uint64]*dataset // In memory datasets to avoid regenerating too often
+ fdataset *dataset // Pre-generated dataset for the estimated future epoch
+
+ // Mining related fields
+ rand *rand.Rand // Properly seeded random source for nonces
+ threads int // Number of threads to mine on if mining
+ update chan struct{} // Notification channel to update mining parameters
+ hashrate metrics.Meter // Meter tracking the average hashrate
+
+ // The fields below are hooks for testing
+ tester bool // Flag whether to use a smaller test dataset
+ shared *Ethash // Shared PoW verifier to avoid cache regeneration
+ fakeMode bool // Flag whether to disable PoW checking
+ fakeFull bool // Flag whether to disable all consensus rules
+ fakeFail uint64 // Block number which fails PoW check even in fake mode
+ fakeDelay time.Duration // Time delay to sleep for before returning from verify
+
+ lock sync.Mutex // Ensures thread safety for the in-memory caches and mining fields
+}
+
+// New creates a full sized ethash PoW scheme.
+func New(cachedir string, cachesinmem, cachesondisk int, dagdir string, dagsinmem, dagsondisk int) *Ethash {
+ if cachesinmem <= 0 {
+ log.Warn("One ethash cache must alwast be in memory", "requested", cachesinmem)
+ cachesinmem = 1
+ }
+ if cachedir != "" && cachesondisk > 0 {
+ log.Info("Disk storage enabled for ethash caches", "dir", cachedir, "count", cachesondisk)
+ }
+ if dagdir != "" && dagsondisk > 0 {
+ log.Info("Disk storage enabled for ethash DAGs", "dir", dagdir, "count", dagsondisk)
+ }
+ return &Ethash{
+ cachedir: cachedir,
+ cachesinmem: cachesinmem,
+ cachesondisk: cachesondisk,
+ dagdir: dagdir,
+ dagsinmem: dagsinmem,
+ dagsondisk: dagsondisk,
+ caches: make(map[uint64]*cache),
+ datasets: make(map[uint64]*dataset),
+ update: make(chan struct{}),
+ hashrate: metrics.NewMeter(),
+ }
+}
+
+// NewTester creates a small sized ethash PoW scheme useful only for testing
+// purposes.
+func NewTester() *Ethash {
+ return &Ethash{
+ cachesinmem: 1,
+ caches: make(map[uint64]*cache),
+ datasets: make(map[uint64]*dataset),
+ tester: true,
+ update: make(chan struct{}),
+ hashrate: metrics.NewMeter(),
+ }
+}
+
+// NewFaker creates a ethash consensus engine with a fake PoW scheme that accepts
+// all blocks' seal as valid, though they still have to conform to the Ethereum
+// consensus rules.
+func NewFaker() *Ethash {
+ return &Ethash{fakeMode: true}
+}
+
+// NewFakeFailer creates a ethash consensus engine with a fake PoW scheme that
+// accepts all blocks as valid apart from the single one specified, though they
+// still have to conform to the Ethereum consensus rules.
+func NewFakeFailer(fail uint64) *Ethash {
+ return &Ethash{fakeMode: true, fakeFail: fail}
+}
+
+// NewFakeDelayer creates a ethash consensus engine with a fake PoW scheme that
+// accepts all blocks as valid, but delays verifications by some time, though
+// they still have to conform to the Ethereum consensus rules.
+func NewFakeDelayer(delay time.Duration) *Ethash {
+ return &Ethash{fakeMode: true, fakeDelay: delay}
+}
+
+// NewFullFaker creates a ethash consensus engine with a full fake scheme that
+// accepts all blocks as valid, without checking any consensus rules whatsoever.
+func NewFullFaker() *Ethash {
+ return &Ethash{fakeMode: true, fakeFull: true}
+}
+
+// NewShared creates a full sized ethash PoW shared between all requesters running
+// in the same process.
+func NewShared() *Ethash {
+ return &Ethash{shared: sharedEthash}
+}
+
+// cache tries to retrieve a verification cache for the specified block number
+// by first checking against a list of in-memory caches, then against caches
+// stored on disk, and finally generating one if none can be found.
+func (ethash *Ethash) cache(block uint64) []uint32 {
+ epoch := block / epochLength
+
+ // If we have a PoW for that epoch, use that
+ ethash.lock.Lock()
+
+ current, future := ethash.caches[epoch], (*cache)(nil)
+ if current == nil {
+ // No in-memory cache, evict the oldest if the cache limit was reached
+ for len(ethash.caches) > 0 && len(ethash.caches) >= ethash.cachesinmem {
+ var evict *cache
+ for _, cache := range ethash.caches {
+ if evict == nil || evict.used.After(cache.used) {
+ evict = cache
+ }
+ }
+ delete(ethash.caches, evict.epoch)
+ evict.release()
+
+ log.Trace("Evicted ethash cache", "epoch", evict.epoch, "used", evict.used)
+ }
+ // If we have the new cache pre-generated, use that, otherwise create a new one
+ if ethash.fcache != nil && ethash.fcache.epoch == epoch {
+ log.Trace("Using pre-generated cache", "epoch", epoch)
+ current, ethash.fcache = ethash.fcache, nil
+ } else {
+ log.Trace("Requiring new ethash cache", "epoch", epoch)
+ current = &cache{epoch: epoch}
+ }
+ ethash.caches[epoch] = current
+
+ // If we just used up the future cache, or need a refresh, regenerate
+ if ethash.fcache == nil || ethash.fcache.epoch <= epoch {
+ if ethash.fcache != nil {
+ ethash.fcache.release()
+ }
+ log.Trace("Requiring new future ethash cache", "epoch", epoch+1)
+ future = &cache{epoch: epoch + 1}
+ ethash.fcache = future
+ }
+ }
+ current.used = time.Now()
+ ethash.lock.Unlock()
+
+ // Wait for generation finish, bump the timestamp and finalize the cache
+ current.generate(ethash.cachedir, ethash.cachesondisk, ethash.tester)
+
+ current.lock.Lock()
+ current.used = time.Now()
+ current.lock.Unlock()
+
+ // If we exhausted the future cache, now's a good time to regenerate it
+ if future != nil {
+ go future.generate(ethash.cachedir, ethash.cachesondisk, ethash.tester)
+ }
+ return current.cache
+}
+
+// dataset tries to retrieve a mining dataset for the specified block number
+// by first checking against a list of in-memory datasets, then against DAGs
+// stored on disk, and finally generating one if none can be found.
+func (ethash *Ethash) dataset(block uint64) []uint32 {
+ epoch := block / epochLength
+
+ // If we have a PoW for that epoch, use that
+ ethash.lock.Lock()
+
+ current, future := ethash.datasets[epoch], (*dataset)(nil)
+ if current == nil {
+ // No in-memory dataset, evict the oldest if the dataset limit was reached
+ for len(ethash.datasets) > 0 && len(ethash.datasets) >= ethash.dagsinmem {
+ var evict *dataset
+ for _, dataset := range ethash.datasets {
+ if evict == nil || evict.used.After(dataset.used) {
+ evict = dataset
+ }
+ }
+ delete(ethash.datasets, evict.epoch)
+ evict.release()
+
+ log.Trace("Evicted ethash dataset", "epoch", evict.epoch, "used", evict.used)
+ }
+ // If we have the new cache pre-generated, use that, otherwise create a new one
+ if ethash.fdataset != nil && ethash.fdataset.epoch == epoch {
+ log.Trace("Using pre-generated dataset", "epoch", epoch)
+ current = &dataset{epoch: ethash.fdataset.epoch} // Reload from disk
+ ethash.fdataset = nil
+ } else {
+ log.Trace("Requiring new ethash dataset", "epoch", epoch)
+ current = &dataset{epoch: epoch}
+ }
+ ethash.datasets[epoch] = current
+
+ // If we just used up the future dataset, or need a refresh, regenerate
+ if ethash.fdataset == nil || ethash.fdataset.epoch <= epoch {
+ if ethash.fdataset != nil {
+ ethash.fdataset.release()
+ }
+ log.Trace("Requiring new future ethash dataset", "epoch", epoch+1)
+ future = &dataset{epoch: epoch + 1}
+ ethash.fdataset = future
+ }
+ }
+ current.used = time.Now()
+ ethash.lock.Unlock()
+
+ // Wait for generation finish, bump the timestamp and finalize the cache
+ current.generate(ethash.dagdir, ethash.dagsondisk, ethash.tester)
+
+ current.lock.Lock()
+ current.used = time.Now()
+ current.lock.Unlock()
+
+ // If we exhausted the future dataset, now's a good time to regenerate it
+ if future != nil {
+ go future.generate(ethash.dagdir, ethash.dagsondisk, ethash.tester)
+ }
+ return current.dataset
+}
+
+// Threads returns the number of mining threads currently enabled. This doesn't
+// necessarily mean that mining is running!
+func (ethash *Ethash) Threads() int {
+ ethash.lock.Lock()
+ defer ethash.lock.Unlock()
+
+ return ethash.threads
+}
+
+// SetThreads updates the number of mining threads currently enabled. Calling
+// this method does not start mining, only sets the thread count. If zero is
+// specified, the miner will use all cores of the machine.
+func (ethash *Ethash) SetThreads(threads int) {
+ ethash.lock.Lock()
+ defer ethash.lock.Unlock()
+
+ // Update the threads and ping any running seal to pull in any changes
+ ethash.threads = threads
+ select {
+ case ethash.update <- struct{}{}:
+ default:
+ }
+}
+
+// Hashrate implements PoW, returning the measured rate of the search invocations
+// per second over the last minute.
+func (ethash *Ethash) Hashrate() float64 {
+ return ethash.hashrate.Rate1()
+}
+
+// APIs implements consensus.Engine, returning the user facing RPC APIs. Currently
+// that is empty.
+func (ethash *Ethash) APIs(chain consensus.ChainReader) []rpc.API {
+ return nil
+}
+
+// SeedHash is the seed to use for generating a verification cache and the mining
+// dataset.
+func SeedHash(block uint64) []byte {
+ return seedHash(block)
+}