From 3ff2f756368f2d8ec0d1d9d25f6ba9cdabd7383e Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Thu, 13 Sep 2018 11:42:19 +0200 Subject: swarm: Chunk refactor (#17659) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Janos Guljas Co-authored-by: Balint Gabor Co-authored-by: Anton Evangelatov Co-authored-by: Viktor Trón --- swarm/storage/memstore_test.go | 133 +++++++++-------------------------------- 1 file changed, 27 insertions(+), 106 deletions(-) (limited to 'swarm/storage/memstore_test.go') diff --git a/swarm/storage/memstore_test.go b/swarm/storage/memstore_test.go index 2c1b0e89e..6b370d2b4 100644 --- a/swarm/storage/memstore_test.go +++ b/swarm/storage/memstore_test.go @@ -18,11 +18,6 @@ package storage import ( "context" - "crypto/rand" - "encoding/binary" - "io/ioutil" - "os" - "sync" "testing" "github.com/ethereum/go-ethereum/swarm/log" @@ -33,40 +28,32 @@ func newTestMemStore() *MemStore { return NewMemStore(storeparams, nil) } -func testMemStoreRandom(n int, processors int, chunksize int64, t *testing.T) { +func testMemStoreRandom(n int, chunksize int64, t *testing.T) { m := newTestMemStore() defer m.Close() - testStoreRandom(m, processors, n, chunksize, t) + testStoreRandom(m, n, chunksize, t) } -func testMemStoreCorrect(n int, processors int, chunksize int64, t *testing.T) { +func testMemStoreCorrect(n int, chunksize int64, t *testing.T) { m := newTestMemStore() defer m.Close() - testStoreCorrect(m, processors, n, chunksize, t) + testStoreCorrect(m, n, chunksize, t) } func TestMemStoreRandom_1(t *testing.T) { - testMemStoreRandom(1, 1, 0, t) + testMemStoreRandom(1, 0, t) } func TestMemStoreCorrect_1(t *testing.T) { - testMemStoreCorrect(1, 1, 4104, t) + testMemStoreCorrect(1, 4104, t) } -func TestMemStoreRandom_1_1k(t *testing.T) { - testMemStoreRandom(1, 1000, 0, t) +func TestMemStoreRandom_1k(t *testing.T) { + testMemStoreRandom(1000, 0, t) } -func TestMemStoreCorrect_1_1k(t *testing.T) { - testMemStoreCorrect(1, 100, 4096, t) -} - -func TestMemStoreRandom_8_1k(t *testing.T) { - testMemStoreRandom(8, 1000, 0, t) -} - -func TestMemStoreCorrect_8_1k(t *testing.T) { - testMemStoreCorrect(8, 1000, 4096, t) +func TestMemStoreCorrect_1k(t *testing.T) { + testMemStoreCorrect(100, 4096, t) } func TestMemStoreNotFound(t *testing.T) { @@ -82,13 +69,13 @@ func TestMemStoreNotFound(t *testing.T) { func benchmarkMemStorePut(n int, processors int, chunksize int64, b *testing.B) { m := newTestMemStore() defer m.Close() - benchmarkStorePut(m, processors, n, chunksize, b) + benchmarkStorePut(m, n, chunksize, b) } func benchmarkMemStoreGet(n int, processors int, chunksize int64, b *testing.B) { m := newTestMemStore() defer m.Close() - benchmarkStoreGet(m, processors, n, chunksize, b) + benchmarkStoreGet(m, n, chunksize, b) } func BenchmarkMemStorePut_1_500(b *testing.B) { @@ -107,104 +94,70 @@ func BenchmarkMemStoreGet_8_500(b *testing.B) { benchmarkMemStoreGet(500, 8, 4096, b) } -func newLDBStore(t *testing.T) (*LDBStore, func()) { - dir, err := ioutil.TempDir("", "bzz-storage-test") - if err != nil { - t.Fatal(err) - } - log.Trace("memstore.tempdir", "dir", dir) - - ldbparams := NewLDBStoreParams(NewDefaultStoreParams(), dir) - db, err := NewLDBStore(ldbparams) - if err != nil { - t.Fatal(err) - } - - cleanup := func() { - db.Close() - err := os.RemoveAll(dir) - if err != nil { - t.Fatal(err) - } - } - - return db, cleanup -} - func TestMemStoreAndLDBStore(t *testing.T) { ldb, cleanup := newLDBStore(t) ldb.setCapacity(4000) defer cleanup() cacheCap := 200 - requestsCap := 200 - memStore := NewMemStore(NewStoreParams(4000, 200, 200, nil, nil), nil) + memStore := NewMemStore(NewStoreParams(4000, 200, nil, nil), nil) tests := []struct { - n int // number of chunks to push to memStore - chunkSize uint64 // size of chunk (by default in Swarm - 4096) - request bool // whether or not to set the ReqC channel on the random chunks + n int // number of chunks to push to memStore + chunkSize int64 // size of chunk (by default in Swarm - 4096) }{ { n: 1, chunkSize: 4096, - request: false, }, { n: 201, chunkSize: 4096, - request: false, }, { n: 501, chunkSize: 4096, - request: false, }, { n: 3100, chunkSize: 4096, - request: false, }, { n: 100, chunkSize: 4096, - request: true, }, } for i, tt := range tests { log.Info("running test", "idx", i, "tt", tt) - var chunks []*Chunk + var chunks []Chunk for i := 0; i < tt.n; i++ { - var c *Chunk - if tt.request { - c = NewRandomRequestChunk(tt.chunkSize) - } else { - c = NewRandomChunk(tt.chunkSize) - } - + c := GenerateRandomChunk(tt.chunkSize) chunks = append(chunks, c) } for i := 0; i < tt.n; i++ { - go ldb.Put(context.TODO(), chunks[i]) - memStore.Put(context.TODO(), chunks[i]) + err := ldb.Put(context.TODO(), chunks[i]) + if err != nil { + t.Fatal(err) + } + err = memStore.Put(context.TODO(), chunks[i]) + if err != nil { + t.Fatal(err) + } if got := memStore.cache.Len(); got > cacheCap { t.Fatalf("expected to get cache capacity less than %v, but got %v", cacheCap, got) } - if got := memStore.requests.Len(); got > requestsCap { - t.Fatalf("expected to get requests capacity less than %v, but got %v", requestsCap, got) - } } for i := 0; i < tt.n; i++ { - _, err := memStore.Get(context.TODO(), chunks[i].Addr) + _, err := memStore.Get(context.TODO(), chunks[i].Address()) if err != nil { if err == ErrChunkNotFound { - _, err := ldb.Get(context.TODO(), chunks[i].Addr) + _, err := ldb.Get(context.TODO(), chunks[i].Address()) if err != nil { t.Fatalf("couldn't get chunk %v from ldb, got error: %v", i, err) } @@ -213,37 +166,5 @@ func TestMemStoreAndLDBStore(t *testing.T) { } } } - - // wait for all chunks to be stored before ending the test are cleaning up - for i := 0; i < tt.n; i++ { - <-chunks[i].dbStoredC - } - } -} - -func NewRandomChunk(chunkSize uint64) *Chunk { - c := &Chunk{ - Addr: make([]byte, 32), - ReqC: nil, - SData: make([]byte, chunkSize+8), // SData should be chunkSize + 8 bytes reserved for length - dbStoredC: make(chan bool), - dbStoredMu: &sync.Mutex{}, } - - rand.Read(c.SData) - - binary.LittleEndian.PutUint64(c.SData[:8], chunkSize) - - hasher := MakeHashFunc(SHA3Hash)() - hasher.Write(c.SData) - copy(c.Addr, hasher.Sum(nil)) - - return c -} - -func NewRandomRequestChunk(chunkSize uint64) *Chunk { - c := NewRandomChunk(chunkSize) - c.ReqC = make(chan bool) - - return c } -- cgit v1.2.3