cmd/geth, core/rawdb: seamless freezer consistency, friendly removedb

4 files changed, 67 insertions, 32 deletions

diff --git a/core/rawdb/accessors_metadata.go b/core/rawdb/accessors_metadata.go
index e6235f010..f8d09fbdd 100644
--- a/core/rawdb/accessors_metadata.go
+++ b/core/rawdb/accessors_metadata.go
@@ -80,20 +80,6 @@ func WriteChainConfig(db ethdb.KeyValueWriter, hash common.Hash, cfg *params.Cha
 	}
 }
 
-// ReadAncientPath retrieves ancient database path which is recorded during the
-// first node setup or forcibly changed by user.
-func ReadAncientPath(db ethdb.KeyValueReader) string {
-	data, _ := db.Get(ancientKey)
-	return string(data)
-}
-
-// WriteAncientPath writes ancient database path into the key-value database.
-func WriteAncientPath(db ethdb.KeyValueWriter, path string) {
-	if err := db.Put(ancientKey, []byte(path)); err != nil {
-		log.Crit("Failed to store ancient path", "err", err)
-	}
-}
-
 // ReadPreimage retrieves a single preimage of the provided hash.
 func ReadPreimage(db ethdb.KeyValueReader, hash common.Hash) []byte {
 	data, _ := db.Get(preimageKey(hash))
diff --git a/core/rawdb/database.go b/core/rawdb/database.go
index 37379147c..353b7dce6 100644
--- a/core/rawdb/database.go
+++ b/core/rawdb/database.go
@@ -18,6 +18,7 @@ package rawdb
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"os"
 	"time"
@@ -104,10 +105,74 @@ func NewDatabase(db ethdb.KeyValueStore) ethdb.Database {
 // value data store with a freezer moving immutable chain segments into cold
 // storage.
 func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace string) (ethdb.Database, error) {
+	// Create the idle freezer instance
 	frdb, err := newFreezer(freezer, namespace)
 	if err != nil {
 		return nil, err
 	}
+	// Since the freezer can be stored separately from the user's key-value database,
+	// there's a fairly high probability that the user requests invalid combinations
+	// of the freezer and database. Ensure that we don't shoot ourselves in the foot
+	// by serving up conflicting data, leading to both datastores getting corrupted.
+	//
+	//   - If both the freezer and key-value store is empty (no genesis), we just
+	//     initialized a new empty freezer, so everything's fine.
+	//   - If the key-value store is empty, but the freezer is not, we need to make
+	//     sure the user's genesis matches the freezer. That will be checked in the
+	//     blockchain, since we don't have the genesis block here (nor should we at
+	//     this point care, the key-value/freezer combo is valid).
+	//   - If neither the key-value store nor the freezer is empty, cross validate
+	//     the genesis hashes to make sure they are compatible. If they are, also
+	//     ensure that there's no gap between the freezer and sunsequently leveldb.
+	//   - If the key-value store is not empty, but the freezer is we might just be
+	//     upgrading to the freezer release, or we might have had a small chain and
+	//     not frozen anything yet. Ensure that no blocks are missing yet from the
+	//     key-value store, since that would mean we already had an old freezer.
+
+	// If the genesis hash is empty, we have a new key-value store, so nothing to
+	// validate in this method. If, however, the genesis hash is not nil, compare
+	// it to the freezer content.
+	if kvgenesis, _ := db.Get(headerHashKey(0)); len(kvgenesis) > 0 {
+		if frozen, _ := frdb.Ancients(); frozen > 0 {
+			// If the freezer already contains something, ensure that the genesis blocks
+			// match, otherwise we might mix up freezers across chains and destroy both
+			// the freezer and the key-value store.
+			if frgenesis, _ := frdb.Ancient(freezerHashTable, 0); !bytes.Equal(kvgenesis, frgenesis) {
+				return nil, fmt.Errorf("genesis mismatch: %#x (leveldb) != %#x (ancients)", kvgenesis, frgenesis)
+			}
+			// Key-value store and freezer belong to the same network. Ensure that they
+			// are contiguous, otherwise we might end up with a non-functional freezer.
+			if kvhash, _ := db.Get(headerHashKey(frozen)); len(kvhash) == 0 {
+				// Subsequent header after the freezer limit is missing from the database.
+				// Reject startup is the database has a more recent head.
+				if *ReadHeaderNumber(db, ReadHeadHeaderHash(db)) > frozen-1 {
+					return nil, fmt.Errorf("gap (#%d) in the chain between ancients and leveldb", frozen)
+				}
+				// Database contains only older data than the freezer, this happens if the
+				// state was wiped and reinited from an existing freezer.
+			} else {
+				// Key-value store continues where the freezer left off, all is fine. We might
+				// have duplicate blocks (crash after freezer write but before kay-value store
+				// deletion, but that's fine).
+			}
+		} else {
+			// If the freezer is empty, ensure nothing was moved yet from the key-value
+			// store, otherwise we'll end up missing data. We check block #1 to decide
+			// if we froze anything previously or not, but do take care of databases with
+			// only the genesis block.
+			if ReadHeadHeaderHash(db) != common.BytesToHash(kvgenesis) {
+				// Key-value store contains more data than the genesis block, make sure we
+				// didn't freeze anything yet.
+				if kvblob, _ := db.Get(headerHashKey(1)); len(kvblob) == 0 {
+					return nil, errors.New("ancient chain segments already extracted, please set --datadir.ancient to the correct path")
+				}
+				// Block #1 is still in the database, we're allowed to init a new feezer
+			} else {
+				// The head header is still the genesis, we're allowed to init a new feezer
+			}
+		}
+	}
+	// Freezer is consistent with the key-value database, permit combining the two
 	go frdb.freeze(db)
 
 	return &freezerdb{
@@ -151,19 +216,6 @@ func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer
 		kvdb.Close()
 		return nil, err
 	}
-	// Make sure we always use the same ancient store.
-	//
-	//                 | stored == nil    | stored != nil
-	// ----------------+------------------+----------------------
-	// freezer == nil  | non-freezer mode | ancient store missing
-	// freezer != nil  | initialize       | ensure consistency
-	stored := ReadAncientPath(kvdb)
-	if stored == "" && freezer != "" {
-		WriteAncientPath(kvdb, freezer)
-	} else if stored != freezer {
-		log.Warn("Ancient path mismatch", "stored", stored, "given", freezer)
-		log.Crit("Please use a consistent ancient path or migrate it via the command line tool `geth migrate-ancient`")
-	}
 	return frdb, nil
 }
 
@@ -243,7 +295,7 @@ func InspectDatabase(db ethdb.Database) error {
 			trieSize += size
 		default:
 			var accounted bool
-			for _, meta := range [][]byte{databaseVerisionKey, headHeaderKey, headBlockKey, headFastBlockKey, fastTrieProgressKey, ancientKey} {
+			for _, meta := range [][]byte{databaseVerisionKey, headHeaderKey, headBlockKey, headFastBlockKey, fastTrieProgressKey} {
 				if bytes.Equal(key, meta) {
 					metadata += size
 					accounted = true
diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go
index ebccf7816..673a181e4 100644
--- a/core/rawdb/freezer_table.go
+++ b/core/rawdb/freezer_table.go
@@ -259,7 +259,7 @@ func (t *freezerTable) preopen() (err error) {
 	// The repair might have already opened (some) files
 	t.releaseFilesAfter(0, false)
 	// Open all except head in RDONLY
-	for i := uint32(t.tailId); i < t.headId; i++ {
+	for i := t.tailId; i < t.headId; i++ {
 		if _, err = t.openFile(i, os.O_RDONLY); err != nil {
 			return err
 		}
diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go
index 0d54a3c8b..a44a2c99f 100644
--- a/core/rawdb/schema.go
+++ b/core/rawdb/schema.go
@@ -41,9 +41,6 @@ var (
 	// fastTrieProgressKey tracks the number of trie entries imported during fast sync.
 	fastTrieProgressKey = []byte("TrieSync")
 
-	// ancientKey tracks the absolute path of ancient database.
-	ancientKey = []byte("AncientPath")
-
 	// Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes).
 	headerPrefix       = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header
 	headerTDSuffix     = []byte("t") // headerPrefix + num (uint64 big endian) + hash + headerTDSuffix -> td