aboutsummaryrefslogtreecommitdiffstats
path: root/core/rawdb
diff options
context:
space:
mode:
Diffstat (limited to 'core/rawdb')
-rw-r--r--core/rawdb/freezer.go7
-rw-r--r--core/rawdb/freezer_table.go44
-rw-r--r--core/rawdb/freezer_table_test.go68
3 files changed, 76 insertions, 43 deletions
diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go
index 741ff9adb..3f377447c 100644
--- a/core/rawdb/freezer.go
+++ b/core/rawdb/freezer.go
@@ -80,8 +80,9 @@ type freezer struct {
func newFreezer(datadir string, namespace string) (*freezer, error) {
// Create the initial freezer object
var (
- readMeter = metrics.NewRegisteredMeter(namespace+"ancient/read", nil)
- writeMeter = metrics.NewRegisteredMeter(namespace+"ancient/write", nil)
+ readMeter = metrics.NewRegisteredMeter(namespace+"ancient/read", nil)
+ writeMeter = metrics.NewRegisteredMeter(namespace+"ancient/write", nil)
+ sizeCounter = metrics.NewRegisteredCounter(namespace+"ancient/size", nil)
)
// Ensure the datadir is not a symbolic link if it exists.
if info, err := os.Lstat(datadir); !os.IsNotExist(err) {
@@ -102,7 +103,7 @@ func newFreezer(datadir string, namespace string) (*freezer, error) {
instanceLock: lock,
}
for name, disableSnappy := range freezerNoSnappy {
- table, err := newTable(datadir, name, readMeter, writeMeter, disableSnappy)
+ table, err := newTable(datadir, name, readMeter, writeMeter, sizeCounter, disableSnappy)
if err != nil {
for _, table := range freezer.tables {
table.Close()
diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go
index 1e5c7cd0b..2fe354a06 100644
--- a/core/rawdb/freezer_table.go
+++ b/core/rawdb/freezer_table.go
@@ -94,17 +94,18 @@ type freezerTable struct {
// to count how many historic items have gone missing.
itemOffset uint32 // Offset (number of discarded items)
- headBytes uint32 // Number of bytes written to the head file
- readMeter metrics.Meter // Meter for measuring the effective amount of data read
- writeMeter metrics.Meter // Meter for measuring the effective amount of data written
+ headBytes uint32 // Number of bytes written to the head file
+ readMeter metrics.Meter // Meter for measuring the effective amount of data read
+ writeMeter metrics.Meter // Meter for measuring the effective amount of data written
+ sizeCounter metrics.Counter // Counter for tracking the combined size of all freezer tables
logger log.Logger // Logger with database path and table name ambedded
lock sync.RWMutex // Mutex protecting the data file descriptors
}
// newTable opens a freezer table with default settings - 2G files
-func newTable(path string, name string, readMeter metrics.Meter, writeMeter metrics.Meter, disableSnappy bool) (*freezerTable, error) {
- return newCustomTable(path, name, readMeter, writeMeter, 2*1000*1000*1000, disableSnappy)
+func newTable(path string, name string, readMeter metrics.Meter, writeMeter metrics.Meter, sizeCounter metrics.Counter, disableSnappy bool) (*freezerTable, error) {
+ return newCustomTable(path, name, readMeter, writeMeter, sizeCounter, 2*1000*1000*1000, disableSnappy)
}
// openFreezerFileForAppend opens a freezer table file and seeks to the end
@@ -148,7 +149,7 @@ func truncateFreezerFile(file *os.File, size int64) error {
// newCustomTable opens a freezer table, creating the data and index files if they are
// non existent. Both files are truncated to the shortest common length to ensure
// they don't go out of sync.
-func newCustomTable(path string, name string, readMeter metrics.Meter, writeMeter metrics.Meter, maxFilesize uint32, noCompression bool) (*freezerTable, error) {
+func newCustomTable(path string, name string, readMeter metrics.Meter, writeMeter metrics.Meter, sizeCounter metrics.Counter, maxFilesize uint32, noCompression bool) (*freezerTable, error) {
// Ensure the containing directory exists and open the indexEntry file
if err := os.MkdirAll(path, 0755); err != nil {
return nil, err
@@ -171,6 +172,7 @@ func newCustomTable(path string, name string, readMeter metrics.Meter, writeMete
files: make(map[uint32]*os.File),
readMeter: readMeter,
writeMeter: writeMeter,
+ sizeCounter: sizeCounter,
name: name,
path: path,
logger: log.New("database", path, "table", name),
@@ -181,6 +183,14 @@ func newCustomTable(path string, name string, readMeter metrics.Meter, writeMete
tab.Close()
return nil, err
}
+ // Initialize the starting size counter
+ size, err := tab.sizeNolock()
+ if err != nil {
+ tab.Close()
+ return nil, err
+ }
+ tab.sizeCounter.Inc(int64(size))
+
return tab, nil
}
@@ -321,6 +331,11 @@ func (t *freezerTable) truncate(items uint64) error {
if atomic.LoadUint64(&t.items) <= items {
return nil
}
+ // We need to truncate, save the old size for metrics tracking
+ oldSize, err := t.sizeNolock()
+ if err != nil {
+ return err
+ }
// Something's out of sync, truncate the table's offset index
t.logger.Warn("Truncating freezer table", "items", t.items, "limit", items)
if err := truncateFreezerFile(t.index, int64(items+1)*indexEntrySize); err != nil {
@@ -355,6 +370,14 @@ func (t *freezerTable) truncate(items uint64) error {
// All data files truncated, set internal counters and return
atomic.StoreUint64(&t.items, items)
atomic.StoreUint32(&t.headBytes, expected.offset)
+
+ // Retrieve the new size and update the total size counter
+ newSize, err := t.sizeNolock()
+ if err != nil {
+ return err
+ }
+ t.sizeCounter.Dec(int64(oldSize - newSize))
+
return nil
}
@@ -483,7 +506,10 @@ func (t *freezerTable) Append(item uint64, blob []byte) error {
}
// Write indexEntry
t.index.Write(idx.marshallBinary())
+
t.writeMeter.Mark(int64(bLen + indexEntrySize))
+ t.sizeCounter.Inc(int64(bLen + indexEntrySize))
+
atomic.AddUint64(&t.items, 1)
return nil
}
@@ -562,6 +588,12 @@ func (t *freezerTable) size() (uint64, error) {
t.lock.RLock()
defer t.lock.RUnlock()
+ return t.sizeNolock()
+}
+
+// sizeNolock returns the total data size in the freezer table without obtaining
+// the mutex first.
+func (t *freezerTable) sizeNolock() (uint64, error) {
stat, err := t.index.Stat()
if err != nil {
return 0, err
diff --git a/core/rawdb/freezer_table_test.go b/core/rawdb/freezer_table_test.go
index e63fb63a3..116e26a7f 100644
--- a/core/rawdb/freezer_table_test.go
+++ b/core/rawdb/freezer_table_test.go
@@ -56,7 +56,7 @@ func TestFreezerBasics(t *testing.T) {
// set cutoff at 50 bytes
f, err := newCustomTable(os.TempDir(),
fmt.Sprintf("unittest-%d", rand.Uint64()),
- metrics.NewMeter(), metrics.NewMeter(), 50, true)
+ metrics.NewMeter(), metrics.NewMeter(), metrics.NewCounter(), 50, true)
if err != nil {
t.Fatal(err)
}
@@ -98,12 +98,12 @@ func TestFreezerBasicsClosing(t *testing.T) {
t.Parallel()
// set cutoff at 50 bytes
var (
- fname = fmt.Sprintf("basics-close-%d", rand.Uint64())
- m1, m2 = metrics.NewMeter(), metrics.NewMeter()
- f *freezerTable
- err error
+ fname = fmt.Sprintf("basics-close-%d", rand.Uint64())
+ rm, wm, sc = metrics.NewMeter(), metrics.NewMeter(), metrics.NewCounter()
+ f *freezerTable
+ err error
)
- f, err = newCustomTable(os.TempDir(), fname, m1, m2, 50, true)
+ f, err = newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
if err != nil {
t.Fatal(err)
}
@@ -112,7 +112,7 @@ func TestFreezerBasicsClosing(t *testing.T) {
data := getChunk(15, x)
f.Append(uint64(x), data)
f.Close()
- f, err = newCustomTable(os.TempDir(), fname, m1, m2, 50, true)
+ f, err = newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
}
defer f.Close()
@@ -126,7 +126,7 @@ func TestFreezerBasicsClosing(t *testing.T) {
t.Fatalf("test %d, got \n%x != \n%x", y, got, exp)
}
f.Close()
- f, err = newCustomTable(os.TempDir(), fname, m1, m2, 50, true)
+ f, err = newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
if err != nil {
t.Fatal(err)
}
@@ -136,11 +136,11 @@ func TestFreezerBasicsClosing(t *testing.T) {
// TestFreezerRepairDanglingHead tests that we can recover if index entries are removed
func TestFreezerRepairDanglingHead(t *testing.T) {
t.Parallel()
- wm, rm := metrics.NewMeter(), metrics.NewMeter()
+ rm, wm, sc := metrics.NewMeter(), metrics.NewMeter(), metrics.NewCounter()
fname := fmt.Sprintf("dangling_headtest-%d", rand.Uint64())
{ // Fill table
- f, err := newCustomTable(os.TempDir(), fname, rm, wm, 50, true)
+ f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
if err != nil {
t.Fatal(err)
}
@@ -169,7 +169,7 @@ func TestFreezerRepairDanglingHead(t *testing.T) {
idxFile.Close()
// Now open it again
{
- f, err := newCustomTable(os.TempDir(), fname, rm, wm, 50, true)
+ f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
// The last item should be missing
if _, err = f.Retrieve(0xff); err == nil {
t.Errorf("Expected error for missing index entry")
@@ -184,11 +184,11 @@ func TestFreezerRepairDanglingHead(t *testing.T) {
// TestFreezerRepairDanglingHeadLarge tests that we can recover if very many index entries are removed
func TestFreezerRepairDanglingHeadLarge(t *testing.T) {
t.Parallel()
- wm, rm := metrics.NewMeter(), metrics.NewMeter()
+ rm, wm, sc := metrics.NewMeter(), metrics.NewMeter(), metrics.NewCounter()
fname := fmt.Sprintf("dangling_headtest-%d", rand.Uint64())
{ // Fill a table and close it
- f, err := newCustomTable(os.TempDir(), fname, wm, rm, 50, true)
+ f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
if err != nil {
t.Fatal(err)
}
@@ -216,7 +216,7 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) {
idxFile.Close()
// Now open it again
{
- f, err := newCustomTable(os.TempDir(), fname, rm, wm, 50, true)
+ f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
// The first item should be there
if _, err = f.Retrieve(0); err != nil {
t.Fatal(err)
@@ -234,7 +234,7 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) {
}
// And if we open it, we should now be able to read all of them (new values)
{
- f, _ := newCustomTable(os.TempDir(), fname, rm, wm, 50, true)
+ f, _ := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
for y := 1; y < 255; y++ {
exp := getChunk(15, ^y)
got, err := f.Retrieve(uint64(y))
@@ -251,11 +251,11 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) {
// TestSnappyDetection tests that we fail to open a snappy database and vice versa
func TestSnappyDetection(t *testing.T) {
t.Parallel()
- wm, rm := metrics.NewMeter(), metrics.NewMeter()
+ rm, wm, sc := metrics.NewMeter(), metrics.NewMeter(), metrics.NewCounter()
fname := fmt.Sprintf("snappytest-%d", rand.Uint64())
// Open with snappy
{
- f, err := newCustomTable(os.TempDir(), fname, wm, rm, 50, true)
+ f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
if err != nil {
t.Fatal(err)
}
@@ -268,7 +268,7 @@ func TestSnappyDetection(t *testing.T) {
}
// Open without snappy
{
- f, err := newCustomTable(os.TempDir(), fname, wm, rm, 50, false)
+ f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, false)
if _, err = f.Retrieve(0); err == nil {
f.Close()
t.Fatalf("expected empty table")
@@ -277,7 +277,7 @@ func TestSnappyDetection(t *testing.T) {
// Open with snappy
{
- f, err := newCustomTable(os.TempDir(), fname, wm, rm, 50, true)
+ f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
// There should be 255 items
if _, err = f.Retrieve(0xfe); err != nil {
f.Close()
@@ -302,11 +302,11 @@ func assertFileSize(f string, size int64) error {
// the index is repaired
func TestFreezerRepairDanglingIndex(t *testing.T) {
t.Parallel()
- wm, rm := metrics.NewMeter(), metrics.NewMeter()
+ rm, wm, sc := metrics.NewMeter(), metrics.NewMeter(), metrics.NewCounter()
fname := fmt.Sprintf("dangling_indextest-%d", rand.Uint64())
{ // Fill a table and close it
- f, err := newCustomTable(os.TempDir(), fname, wm, rm, 50, true)
+ f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
if err != nil {
t.Fatal(err)
}
@@ -342,7 +342,7 @@ func TestFreezerRepairDanglingIndex(t *testing.T) {
// 45, 45, 15
// with 3+3+1 items
{
- f, err := newCustomTable(os.TempDir(), fname, wm, rm, 50, true)
+ f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
if err != nil {
t.Fatal(err)
}
@@ -359,11 +359,11 @@ func TestFreezerRepairDanglingIndex(t *testing.T) {
func TestFreezerTruncate(t *testing.T) {
t.Parallel()
- wm, rm := metrics.NewMeter(), metrics.NewMeter()
+ rm, wm, sc := metrics.NewMeter(), metrics.NewMeter(), metrics.NewCounter()
fname := fmt.Sprintf("truncation-%d", rand.Uint64())
{ // Fill table
- f, err := newCustomTable(os.TempDir(), fname, rm, wm, 50, true)
+ f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
if err != nil {
t.Fatal(err)
}
@@ -380,7 +380,7 @@ func TestFreezerTruncate(t *testing.T) {
}
// Reopen, truncate
{
- f, err := newCustomTable(os.TempDir(), fname, rm, wm, 50, true)
+ f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
if err != nil {
t.Fatal(err)
}
@@ -402,10 +402,10 @@ func TestFreezerTruncate(t *testing.T) {
// That will rewind the index, and _should_ truncate the head file
func TestFreezerRepairFirstFile(t *testing.T) {
t.Parallel()
- wm, rm := metrics.NewMeter(), metrics.NewMeter()
+ rm, wm, sc := metrics.NewMeter(), metrics.NewMeter(), metrics.NewCounter()
fname := fmt.Sprintf("truncationfirst-%d", rand.Uint64())
{ // Fill table
- f, err := newCustomTable(os.TempDir(), fname, rm, wm, 50, true)
+ f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
if err != nil {
t.Fatal(err)
}
@@ -433,7 +433,7 @@ func TestFreezerRepairFirstFile(t *testing.T) {
}
// Reopen
{
- f, err := newCustomTable(os.TempDir(), fname, wm, rm, 50, true)
+ f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
if err != nil {
t.Fatal(err)
}
@@ -458,10 +458,10 @@ func TestFreezerRepairFirstFile(t *testing.T) {
// - check that we did not keep the rdonly file descriptors
func TestFreezerReadAndTruncate(t *testing.T) {
t.Parallel()
- wm, rm := metrics.NewMeter(), metrics.NewMeter()
+ rm, wm, sc := metrics.NewMeter(), metrics.NewMeter(), metrics.NewCounter()
fname := fmt.Sprintf("read_truncate-%d", rand.Uint64())
{ // Fill table
- f, err := newCustomTable(os.TempDir(), fname, rm, wm, 50, true)
+ f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
if err != nil {
t.Fatal(err)
}
@@ -478,7 +478,7 @@ func TestFreezerReadAndTruncate(t *testing.T) {
}
// Reopen and read all files
{
- f, err := newCustomTable(os.TempDir(), fname, wm, rm, 50, true)
+ f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 50, true)
if err != nil {
t.Fatal(err)
}
@@ -504,10 +504,10 @@ func TestFreezerReadAndTruncate(t *testing.T) {
func TestOffset(t *testing.T) {
t.Parallel()
- wm, rm := metrics.NewMeter(), metrics.NewMeter()
+ rm, wm, sc := metrics.NewMeter(), metrics.NewMeter(), metrics.NewCounter()
fname := fmt.Sprintf("offset-%d", rand.Uint64())
{ // Fill table
- f, err := newCustomTable(os.TempDir(), fname, rm, wm, 40, true)
+ f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 40, true)
if err != nil {
t.Fatal(err)
}
@@ -563,7 +563,7 @@ func TestOffset(t *testing.T) {
}
// Now open again
{
- f, err := newCustomTable(os.TempDir(), fname, rm, wm, 40, true)
+ f, err := newCustomTable(os.TempDir(), fname, rm, wm, sc, 40, true)
if err != nil {
t.Fatal(err)
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-03-26 18:17:04 +0800