From e187711c6545487d4cac3701f0f506bb536234e2 Mon Sep 17 00:00:00 2001 From: ethersphere Date: Wed, 20 Jun 2018 14:06:27 +0200 Subject: swarm: network rewrite merge --- swarm/storage/memstore_test.go | 228 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 213 insertions(+), 15 deletions(-) (limited to 'swarm/storage/memstore_test.go') diff --git a/swarm/storage/memstore_test.go b/swarm/storage/memstore_test.go index 2e0ab535a..5c68a4b4b 100644 --- a/swarm/storage/memstore_test.go +++ b/swarm/storage/memstore_test.go @@ -17,34 +17,232 @@ package storage import ( + "crypto/rand" + "encoding/binary" + "io/ioutil" + "os" + "sync" "testing" + + "github.com/ethereum/go-ethereum/swarm/log" ) -func testMemStore(l int64, branches int64, t *testing.T) { - m := NewMemStore(nil, defaultCacheCapacity) - testStore(m, l, branches, t) +func newTestMemStore() *MemStore { + storeparams := NewDefaultStoreParams() + return NewMemStore(storeparams, nil) +} + +func testMemStoreRandom(n int, processors int, chunksize int64, t *testing.T) { + m := newTestMemStore() + defer m.Close() + testStoreRandom(m, processors, n, chunksize, t) +} + +func testMemStoreCorrect(n int, processors int, chunksize int64, t *testing.T) { + m := newTestMemStore() + defer m.Close() + testStoreCorrect(m, processors, n, chunksize, t) +} + +func TestMemStoreRandom_1(t *testing.T) { + testMemStoreRandom(1, 1, 0, t) } -func TestMemStore128_10000(t *testing.T) { - testMemStore(10000, 128, t) +func TestMemStoreCorrect_1(t *testing.T) { + testMemStoreCorrect(1, 1, 4104, t) } -func TestMemStore128_1000(t *testing.T) { - testMemStore(1000, 128, t) +func TestMemStoreRandom_1_1k(t *testing.T) { + testMemStoreRandom(1, 1000, 0, t) } -func TestMemStore128_100(t *testing.T) { - testMemStore(100, 128, t) +func TestMemStoreCorrect_1_1k(t *testing.T) { + testMemStoreCorrect(1, 100, 4096, t) } -func TestMemStore2_100(t *testing.T) { - testMemStore(100, 2, t) +func TestMemStoreRandom_8_1k(t *testing.T) { + testMemStoreRandom(8, 1000, 0, t) +} + +func TestMemStoreCorrect_8_1k(t *testing.T) { + testMemStoreCorrect(8, 1000, 4096, t) } func TestMemStoreNotFound(t *testing.T) { - m := NewMemStore(nil, defaultCacheCapacity) - _, err := m.Get(ZeroKey) - if err != notFound { - t.Errorf("Expected notFound, got %v", err) + m := newTestMemStore() + defer m.Close() + + _, err := m.Get(ZeroAddr) + if err != ErrChunkNotFound { + t.Errorf("Expected ErrChunkNotFound, got %v", err) + } +} + +func benchmarkMemStorePut(n int, processors int, chunksize int64, b *testing.B) { + m := newTestMemStore() + defer m.Close() + benchmarkStorePut(m, processors, n, chunksize, b) +} + +func benchmarkMemStoreGet(n int, processors int, chunksize int64, b *testing.B) { + m := newTestMemStore() + defer m.Close() + benchmarkStoreGet(m, processors, n, chunksize, b) +} + +func BenchmarkMemStorePut_1_500(b *testing.B) { + benchmarkMemStorePut(500, 1, 4096, b) +} + +func BenchmarkMemStorePut_8_500(b *testing.B) { + benchmarkMemStorePut(500, 8, 4096, b) +} + +func BenchmarkMemStoreGet_1_500(b *testing.B) { + benchmarkMemStoreGet(500, 1, 4096, b) +} + +func BenchmarkMemStoreGet_8_500(b *testing.B) { + benchmarkMemStoreGet(500, 8, 4096, b) +} + +func newLDBStore(t *testing.T) (*LDBStore, func()) { + dir, err := ioutil.TempDir("", "bzz-storage-test") + if err != nil { + t.Fatal(err) + } + log.Trace("memstore.tempdir", "dir", dir) + + ldbparams := NewLDBStoreParams(NewDefaultStoreParams(), dir) + db, err := NewLDBStore(ldbparams) + if err != nil { + t.Fatal(err) } + + cleanup := func() { + db.Close() + err := os.RemoveAll(dir) + if err != nil { + t.Fatal(err) + } + } + + return db, cleanup +} + +func TestMemStoreAndLDBStore(t *testing.T) { + ldb, cleanup := newLDBStore(t) + ldb.setCapacity(4000) + defer cleanup() + + cacheCap := 200 + requestsCap := 200 + memStore := NewMemStore(NewStoreParams(4000, 200, 200, nil, nil), nil) + + tests := []struct { + n int // number of chunks to push to memStore + chunkSize uint64 // size of chunk (by default in Swarm - 4096) + request bool // whether or not to set the ReqC channel on the random chunks + }{ + { + n: 1, + chunkSize: 4096, + request: false, + }, + { + n: 201, + chunkSize: 4096, + request: false, + }, + { + n: 501, + chunkSize: 4096, + request: false, + }, + { + n: 3100, + chunkSize: 4096, + request: false, + }, + { + n: 100, + chunkSize: 4096, + request: true, + }, + } + + for i, tt := range tests { + log.Info("running test", "idx", i, "tt", tt) + var chunks []*Chunk + + for i := 0; i < tt.n; i++ { + var c *Chunk + if tt.request { + c = NewRandomRequestChunk(tt.chunkSize) + } else { + c = NewRandomChunk(tt.chunkSize) + } + + chunks = append(chunks, c) + } + + for i := 0; i < tt.n; i++ { + go ldb.Put(chunks[i]) + memStore.Put(chunks[i]) + + if got := memStore.cache.Len(); got > cacheCap { + t.Fatalf("expected to get cache capacity less than %v, but got %v", cacheCap, got) + } + + if got := memStore.requests.Len(); got > requestsCap { + t.Fatalf("expected to get requests capacity less than %v, but got %v", requestsCap, got) + } + } + + for i := 0; i < tt.n; i++ { + _, err := memStore.Get(chunks[i].Addr) + if err != nil { + if err == ErrChunkNotFound { + _, err := ldb.Get(chunks[i].Addr) + if err != nil { + t.Fatalf("couldn't get chunk %v from ldb, got error: %v", i, err) + } + } else { + t.Fatalf("got error from memstore: %v", err) + } + } + } + + // wait for all chunks to be stored before ending the test are cleaning up + for i := 0; i < tt.n; i++ { + <-chunks[i].dbStoredC + } + } +} + +func NewRandomChunk(chunkSize uint64) *Chunk { + c := &Chunk{ + Addr: make([]byte, 32), + ReqC: nil, + SData: make([]byte, chunkSize+8), // SData should be chunkSize + 8 bytes reserved for length + dbStoredC: make(chan bool), + dbStoredMu: &sync.Mutex{}, + } + + rand.Read(c.SData) + + binary.LittleEndian.PutUint64(c.SData[:8], chunkSize) + + hasher := MakeHashFunc(SHA3Hash)() + hasher.Write(c.SData) + copy(c.Addr, hasher.Sum(nil)) + + return c +} + +func NewRandomRequestChunk(chunkSize uint64) *Chunk { + c := NewRandomChunk(chunkSize) + c.ReqC = make(chan bool) + + return c } -- cgit v1.2.3