whisper: big refactoring (#13852)

* whisper: GetMessages fixed; size restriction updated
* whisper: made PoW and MaxMsgSize customizable
* whisper: test added
* whisper: sym key management changed
* whisper: identity management refactored
* whisper: API refactoring (Post and Filter)
* whisper: big refactoring complete
* whisper: spelling fix
* whisper: variable topic size allowed for a filter
* whisper: final update
* whisper: formatting
* whisper: file exchange introduced in wnode
* whisper: bugfix
* whisper: API updated + new tests
* whisper: statistics updated
* whisper: wnode server updated
* whisper: allowed filtering for variable topic size
* whisper: tests added
* whisper: resolving merge conflicts
* whisper: refactoring (documenting mostly)
* whsiper: tests fixed
* whisper: down cased error messages
* whisper: documenting the API functions
* whisper: logging fixed
* whisper: fixed wnode parameters
* whisper: logs fixed (typos)

1 files changed, 289 insertions, 66 deletions

diff --git a/whisper/whisperv5/whisper_test.go b/whisper/whisperv5/whisper_test.go
index 8d63d443c..d5668259e 100644
--- a/whisper/whisperv5/whisper_test.go
+++ b/whisper/whisperv5/whisper_test.go
@@ -21,9 +21,6 @@ import (
 	mrand "math/rand"
 	"testing"
 	"time"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/crypto"
 )
 
 func TestWhisperBasic(t *testing.T) {
@@ -55,16 +52,19 @@ func TestWhisperBasic(t *testing.T) {
 	if peer != nil {
 		t.Fatal("found peer for random key.")
 	}
-	if err := w.MarkPeerTrusted(peerID); err == nil {
+	if err := w.AllowP2PMessagesFromPeer(peerID); err == nil {
 		t.Fatalf("failed MarkPeerTrusted.")
 	}
 	exist := w.HasSymKey("non-existing")
 	if exist {
 		t.Fatalf("failed HasSymKey.")
 	}
-	key := w.GetSymKey("non-existing")
+	key, err := w.GetSymKey("non-existing")
+	if err == nil {
+		t.Fatalf("failed GetSymKey(non-existing): false positive.")
+	}
 	if key != nil {
-		t.Fatalf("failed GetSymKey.")
+		t.Fatalf("failed GetSymKey: false positive.")
 	}
 	mail := w.Envelopes()
 	if len(mail) != 0 {
@@ -80,7 +80,7 @@ func TestWhisperBasic(t *testing.T) {
 	if _, err := deriveKeyMaterial(peerID, ver); err != unknownVersionError(ver) {
 		t.Fatalf("failed deriveKeyMaterial with param = %v: %s.", peerID, err)
 	}
-	derived, err := deriveKeyMaterial(peerID, 0)
+	derived, err = deriveKeyMaterial(peerID, 0)
 	if err != nil {
 		t.Fatalf("failed second deriveKeyMaterial with param = %v: %s.", peerID, err)
 	}
@@ -92,8 +92,8 @@ func TestWhisperBasic(t *testing.T) {
 	}
 
 	buf := []byte{0xFF, 0xE5, 0x80, 0x2, 0}
-	le := bytesToIntLittleEndian(buf)
-	be := BytesToIntBigEndian(buf)
+	le := bytesToUintLittleEndian(buf)
+	be := BytesToUintBigEndian(buf)
 	if le != uint64(0x280e5ff) {
 		t.Fatalf("failed bytesToIntLittleEndian: %d.", le)
 	}
@@ -101,7 +101,14 @@ func TestWhisperBasic(t *testing.T) {
 		t.Fatalf("failed BytesToIntBigEndian: %d.", be)
 	}
 
-	pk := w.NewIdentity()
+	id, err := w.NewKeyPair()
+	if err != nil {
+		t.Fatalf("failed to generate new key pair: %s.", err)
+	}
+	pk, err := w.GetPrivateKey(id)
+	if err != nil {
+		t.Fatalf("failed to retrieve new key pair: %s.", err)
+	}
 	if !validatePrivateKey(pk) {
 		t.Fatalf("failed validatePrivateKey: %v.", pk)
 	}
@@ -112,67 +119,112 @@ func TestWhisperBasic(t *testing.T) {
 
 func TestWhisperIdentityManagement(t *testing.T) {
 	w := New()
-	id1 := w.NewIdentity()
-	id2 := w.NewIdentity()
-	pub1 := common.ToHex(crypto.FromECDSAPub(&id1.PublicKey))
-	pub2 := common.ToHex(crypto.FromECDSAPub(&id2.PublicKey))
-	pk1 := w.GetIdentity(pub1)
-	pk2 := w.GetIdentity(pub2)
-	if !w.HasIdentity(pub1) {
-		t.Fatalf("failed HasIdentity(pub1).")
+	id1, err := w.NewKeyPair()
+	if err != nil {
+		t.Fatalf("failed to generate new key pair: %s.", err)
 	}
-	if !w.HasIdentity(pub2) {
-		t.Fatalf("failed HasIdentity(pub2).")
+	id2, err := w.NewKeyPair()
+	if err != nil {
+		t.Fatalf("failed to generate new key pair: %s.", err)
 	}
-	if pk1 != id1 {
-		t.Fatalf("failed GetIdentity(pub1).")
+	pk1, err := w.GetPrivateKey(id1)
+	if err != nil {
+		t.Fatalf("failed to retrieve the key pair: %s.", err)
+	}
+	pk2, err := w.GetPrivateKey(id2)
+	if err != nil {
+		t.Fatalf("failed to retrieve the key pair: %s.", err)
+	}
+
+	if !w.HasKeyPair(id1) {
+		t.Fatalf("failed HasIdentity(pk1).")
+	}
+	if !w.HasKeyPair(id2) {
+		t.Fatalf("failed HasIdentity(pk2).")
+	}
+	if pk1 == nil {
+		t.Fatalf("failed GetIdentity(pk1).")
+	}
+	if pk2 == nil {
+		t.Fatalf("failed GetIdentity(pk2).")
+	}
+
+	if !validatePrivateKey(pk1) {
+		t.Fatalf("pk1 is invalid.")
 	}
-	if pk2 != id2 {
-		t.Fatalf("failed GetIdentity(pub2).")
+	if !validatePrivateKey(pk2) {
+		t.Fatalf("pk2 is invalid.")
 	}
 
 	// Delete one identity
-	w.DeleteIdentity(pub1)
-	pk1 = w.GetIdentity(pub1)
-	pk2 = w.GetIdentity(pub2)
-	if w.HasIdentity(pub1) {
+	done := w.DeleteKeyPair(id1)
+	if !done {
+		t.Fatalf("failed to delete id1.")
+	}
+	pk1, err = w.GetPrivateKey(id1)
+	if err == nil {
+		t.Fatalf("retrieve the key pair: false positive.")
+	}
+	pk2, err = w.GetPrivateKey(id2)
+	if err != nil {
+		t.Fatalf("failed to retrieve the key pair: %s.", err)
+	}
+	if w.HasKeyPair(id1) {
 		t.Fatalf("failed DeleteIdentity(pub1): still exist.")
 	}
-	if !w.HasIdentity(pub2) {
+	if !w.HasKeyPair(id2) {
 		t.Fatalf("failed DeleteIdentity(pub1): pub2 does not exist.")
 	}
 	if pk1 != nil {
 		t.Fatalf("failed DeleteIdentity(pub1): first key still exist.")
 	}
-	if pk2 != id2 {
+	if pk2 == nil {
 		t.Fatalf("failed DeleteIdentity(pub1): second key does not exist.")
 	}
 
 	// Delete again non-existing identity
-	w.DeleteIdentity(pub1)
-	pk1 = w.GetIdentity(pub1)
-	pk2 = w.GetIdentity(pub2)
-	if w.HasIdentity(pub1) {
+	done = w.DeleteKeyPair(id1)
+	if done {
+		t.Fatalf("delete id1: false positive.")
+	}
+	pk1, err = w.GetPrivateKey(id1)
+	if err == nil {
+		t.Fatalf("retrieve the key pair: false positive.")
+	}
+	pk2, err = w.GetPrivateKey(id2)
+	if err != nil {
+		t.Fatalf("failed to retrieve the key pair: %s.", err)
+	}
+	if w.HasKeyPair(id1) {
 		t.Fatalf("failed delete non-existing identity: exist.")
 	}
-	if !w.HasIdentity(pub2) {
+	if !w.HasKeyPair(id2) {
 		t.Fatalf("failed delete non-existing identity: pub2 does not exist.")
 	}
 	if pk1 != nil {
 		t.Fatalf("failed delete non-existing identity: first key exist.")
 	}
-	if pk2 != id2 {
+	if pk2 == nil {
 		t.Fatalf("failed delete non-existing identity: second key does not exist.")
 	}
 
 	// Delete second identity
-	w.DeleteIdentity(pub2)
-	pk1 = w.GetIdentity(pub1)
-	pk2 = w.GetIdentity(pub2)
-	if w.HasIdentity(pub1) {
+	done = w.DeleteKeyPair(id2)
+	if !done {
+		t.Fatalf("failed to delete id2.")
+	}
+	pk1, err = w.GetPrivateKey(id1)
+	if err == nil {
+		t.Fatalf("retrieve the key pair: false positive.")
+	}
+	pk2, err = w.GetPrivateKey(id2)
+	if err == nil {
+		t.Fatalf("retrieve the key pair: false positive.")
+	}
+	if w.HasKeyPair(id1) {
 		t.Fatalf("failed delete second identity: first identity exist.")
 	}
-	if w.HasIdentity(pub2) {
+	if w.HasKeyPair(id2) {
 		t.Fatalf("failed delete second identity: still exist.")
 	}
 	if pk1 != nil {
@@ -186,23 +238,30 @@ func TestWhisperIdentityManagement(t *testing.T) {
 func TestWhisperSymKeyManagement(t *testing.T) {
 	InitSingleTest()
 
+	var err error
 	var k1, k2 []byte
 	w := New()
 	id1 := string("arbitrary-string-1")
 	id2 := string("arbitrary-string-2")
 
-	err := w.GenerateSymKey(id1)
+	id1, err = w.GenerateSymKey()
 	if err != nil {
 		t.Fatalf("failed GenerateSymKey with seed %d: %s.", seed, err)
 	}
 
-	k1 = w.GetSymKey(id1)
-	k2 = w.GetSymKey(id2)
+	k1, err = w.GetSymKey(id1)
+	if err != nil {
+		t.Fatalf("failed GetSymKey(id1).")
+	}
+	k2, err = w.GetSymKey(id2)
+	if err == nil {
+		t.Fatalf("failed GetSymKey(id2): false positive.")
+	}
 	if !w.HasSymKey(id1) {
 		t.Fatalf("failed HasSymKey(id1).")
 	}
 	if w.HasSymKey(id2) {
-		t.Fatalf("failed HasSymKey(id2).")
+		t.Fatalf("failed HasSymKey(id2): false positive.")
 	}
 	if k1 == nil {
 		t.Fatalf("first key does not exist.")
@@ -212,37 +271,49 @@ func TestWhisperSymKeyManagement(t *testing.T) {
 	}
 
 	// add existing id, nothing should change
-	randomKey := make([]byte, 16)
+	randomKey := make([]byte, aesKeyLength)
 	mrand.Read(randomKey)
-	err = w.AddSymKey(id1, randomKey)
-	if err == nil {
-		t.Fatalf("failed AddSymKey with seed %d.", seed)
+	id1, err = w.AddSymKeyDirect(randomKey)
+	if err != nil {
+		t.Fatalf("failed AddSymKey with seed %d: %s.", seed, err)
 	}
 
-	k1 = w.GetSymKey(id1)
-	k2 = w.GetSymKey(id2)
+	k1, err = w.GetSymKey(id1)
+	if err != nil {
+		t.Fatalf("failed w.GetSymKey(id1).")
+	}
+	k2, err = w.GetSymKey(id2)
+	if err == nil {
+		t.Fatalf("failed w.GetSymKey(id2): false positive.")
+	}
 	if !w.HasSymKey(id1) {
 		t.Fatalf("failed w.HasSymKey(id1).")
 	}
 	if w.HasSymKey(id2) {
-		t.Fatalf("failed w.HasSymKey(id2).")
+		t.Fatalf("failed w.HasSymKey(id2): false positive.")
 	}
 	if k1 == nil {
 		t.Fatalf("first key does not exist.")
 	}
-	if bytes.Equal(k1, randomKey) {
-		t.Fatalf("k1 == randomKey.")
+	if !bytes.Equal(k1, randomKey) {
+		t.Fatalf("k1 != randomKey.")
 	}
 	if k2 != nil {
 		t.Fatalf("second key already exist.")
 	}
 
-	err = w.AddSymKey(id2, randomKey) // add non-existing (yet)
+	id2, err = w.AddSymKeyDirect(randomKey)
 	if err != nil {
 		t.Fatalf("failed AddSymKey(id2) with seed %d: %s.", seed, err)
 	}
-	k1 = w.GetSymKey(id1)
-	k2 = w.GetSymKey(id2)
+	k1, err = w.GetSymKey(id1)
+	if err != nil {
+		t.Fatalf("failed w.GetSymKey(id1).")
+	}
+	k2, err = w.GetSymKey(id2)
+	if err != nil {
+		t.Fatalf("failed w.GetSymKey(id2).")
+	}
 	if !w.HasSymKey(id1) {
 		t.Fatalf("HasSymKey(id1) failed.")
 	}
@@ -255,11 +326,11 @@ func TestWhisperSymKeyManagement(t *testing.T) {
 	if k2 == nil {
 		t.Fatalf("k2 does not exist.")
 	}
-	if bytes.Equal(k1, k2) {
-		t.Fatalf("k1 == k2.")
+	if !bytes.Equal(k1, k2) {
+		t.Fatalf("k1 != k2.")
 	}
-	if bytes.Equal(k1, randomKey) {
-		t.Fatalf("k1 == randomKey.")
+	if !bytes.Equal(k1, randomKey) {
+		t.Fatalf("k1 != randomKey.")
 	}
 	if len(k1) != aesKeyLength {
 		t.Fatalf("wrong length of k1.")
@@ -269,8 +340,17 @@ func TestWhisperSymKeyManagement(t *testing.T) {
 	}
 
 	w.DeleteSymKey(id1)
-	k1 = w.GetSymKey(id1)
-	k2 = w.GetSymKey(id2)
+	k1, err = w.GetSymKey(id1)
+	if err == nil {
+		t.Fatalf("failed w.GetSymKey(id1): false positive.")
+	}
+	if k1 != nil {
+		t.Fatalf("failed GetSymKey(id1): false positive.")
+	}
+	k2, err = w.GetSymKey(id2)
+	if err != nil {
+		t.Fatalf("failed w.GetSymKey(id2).")
+	}
 	if w.HasSymKey(id1) {
 		t.Fatalf("failed to delete first key: still exist.")
 	}
@@ -286,8 +366,17 @@ func TestWhisperSymKeyManagement(t *testing.T) {
 
 	w.DeleteSymKey(id1)
 	w.DeleteSymKey(id2)
-	k1 = w.GetSymKey(id1)
-	k2 = w.GetSymKey(id2)
+	k1, err = w.GetSymKey(id1)
+	if err == nil {
+		t.Fatalf("failed w.GetSymKey(id1): false positive.")
+	}
+	k2, err = w.GetSymKey(id2)
+	if err == nil {
+		t.Fatalf("failed w.GetSymKey(id2): false positive.")
+	}
+	if k1 != nil || k2 != nil {
+		t.Fatalf("k1 or k2 is not nil")
+	}
 	if w.HasSymKey(id1) {
 		t.Fatalf("failed to delete second key: first key exist.")
 	}
@@ -300,13 +389,63 @@ func TestWhisperSymKeyManagement(t *testing.T) {
 	if k2 != nil {
 		t.Fatalf("failed to delete second key: second key is not nil.")
 	}
+
+	randomKey = make([]byte, aesKeyLength+1)
+	mrand.Read(randomKey)
+	id1, err = w.AddSymKeyDirect(randomKey)
+	if err == nil {
+		t.Fatalf("added the key with wrong size, seed %d.", seed)
+	}
+
+	const password = "arbitrary data here"
+	id1, err = w.AddSymKeyFromPassword(password)
+	if err != nil {
+		t.Fatalf("failed AddSymKeyFromPassword(id1) with seed %d: %s.", seed, err)
+	}
+	id2, err = w.AddSymKeyFromPassword(password)
+	if err != nil {
+		t.Fatalf("failed AddSymKeyFromPassword(id2) with seed %d: %s.", seed, err)
+	}
+	k1, err = w.GetSymKey(id1)
+	if err != nil {
+		t.Fatalf("failed w.GetSymKey(id1).")
+	}
+	k2, err = w.GetSymKey(id2)
+	if err != nil {
+		t.Fatalf("failed w.GetSymKey(id2).")
+	}
+	if !w.HasSymKey(id1) {
+		t.Fatalf("HasSymKey(id1) failed.")
+	}
+	if !w.HasSymKey(id2) {
+		t.Fatalf("HasSymKey(id2) failed.")
+	}
+	if k1 == nil {
+		t.Fatalf("k1 does not exist.")
+	}
+	if k2 == nil {
+		t.Fatalf("k2 does not exist.")
+	}
+	if !bytes.Equal(k1, k2) {
+		t.Fatalf("k1 != k2.")
+	}
+	if len(k1) != aesKeyLength {
+		t.Fatalf("wrong length of k1.")
+	}
+	if len(k2) != aesKeyLength {
+		t.Fatalf("wrong length of k2.")
+	}
+	if !validateSymmetricKey(k2) {
+		t.Fatalf("key validation failed.")
+	}
 }
 
 func TestExpiry(t *testing.T) {
 	InitSingleTest()
 
 	w := New()
-	w.test = true
+	w.SetMinimumPoW(0.0000001)
+	defer w.SetMinimumPoW(DefaultMinimumPoW)
 	w.Start(nil)
 	defer w.Stop()
 
@@ -354,3 +493,87 @@ func TestExpiry(t *testing.T) {
 		t.Fatalf("expire failed, seed: %d.", seed)
 	}
 }
+
+func TestCustomization(t *testing.T) {
+	InitSingleTest()
+
+	w := New()
+	defer w.SetMinimumPoW(DefaultMinimumPoW)
+	defer w.SetMaxMessageLength(DefaultMaxMessageLength)
+	w.Start(nil)
+	defer w.Stop()
+
+	const smallPoW = 0.00001
+
+	f, err := generateFilter(t, true)
+	params, err := generateMessageParams()
+	if err != nil {
+		t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err)
+	}
+
+	params.KeySym = f.KeySym
+	params.Topic = BytesToTopic(f.Topics[2])
+	params.PoW = smallPoW
+	params.TTL = 3600 * 24 // one day
+	msg := NewSentMessage(params)
+	env, err := msg.Wrap(params)
+	if err != nil {
+		t.Fatalf("failed Wrap with seed %d: %s.", seed, err)
+	}
+
+	err = w.Send(env)
+	if err == nil {
+		t.Fatalf("successfully sent envelope with PoW %.06f, false positive (seed %d).", env.PoW(), seed)
+	}
+
+	w.SetMinimumPoW(smallPoW / 2)
+	err = w.Send(env)
+	if err != nil {
+		t.Fatalf("failed to send envelope with seed %d: %s.", seed, err)
+	}
+
+	params.TTL++
+	msg = NewSentMessage(params)
+	env, err = msg.Wrap(params)
+	if err != nil {
+		t.Fatalf("failed Wrap with seed %d: %s.", seed, err)
+	}
+	w.SetMaxMessageLength(env.size() - 1)
+	err = w.Send(env)
+	if err == nil {
+		t.Fatalf("successfully sent oversized envelope (seed %d): false positive.", seed)
+	}
+
+	w.SetMaxMessageLength(DefaultMaxMessageLength)
+	err = w.Send(env)
+	if err != nil {
+		t.Fatalf("failed to send second envelope with seed %d: %s.", seed, err)
+	}
+
+	// wait till received or timeout
+	var received bool
+	for j := 0; j < 20; j++ {
+		time.Sleep(100 * time.Millisecond)
+		if len(w.Envelopes()) > 1 {
+			received = true
+			break
+		}
+	}
+
+	if !received {
+		t.Fatalf("did not receive the sent envelope, seed: %d.", seed)
+	}
+
+	// check w.messages()
+	id, err := w.Subscribe(f)
+	time.Sleep(5 * time.Millisecond)
+	mail := f.Retrieve()
+	if len(mail) > 0 {
+		t.Fatalf("received premature mail")
+	}
+
+	mail = w.Messages(id)
+	if len(mail) != 2 {
+		t.Fatalf("failed to get whisper messages")
+	}
+}