cmd/clef, signer: initial poc of the standalone signer (#16154)

* signer: introduce external signer command

* cmd/signer, rpc: Implement new signer. Add info about remote user to Context

* signer: refactored request/response, made use of urfave.cli

* cmd/signer: Use common flags

* cmd/signer: methods to validate calldata against abi

* cmd/signer: work on abi parser

* signer: add mutex around UI

* cmd/signer: add json 4byte directory, remove passwords from api

* cmd/signer: minor changes

* cmd/signer: Use ErrRequestDenied, enable lightkdf

* cmd/signer: implement tests

* cmd/signer: made possible for UI to modify tx parameters

* cmd/signer: refactors, removed channels in ui comms, added UI-api via stdin/out

* cmd/signer: Made lowercase json-definitions, added UI-signer test functionality

* cmd/signer: update documentation

* cmd/signer: fix bugs, improve abi detection, abi argument display

* cmd/signer: minor change in json format

* cmd/signer: rework json communication

* cmd/signer: implement mixcase addresses in API, fix json id bug

* cmd/signer: rename fromaccount, update pythonpoc with new json encoding format

* cmd/signer: make use of new abi interface

* signer: documentation

* signer/main: remove redundant  option

* signer: implement audit logging

* signer: create package 'signer', minor changes

* common: add 0x-prefix to mixcaseaddress in json marshalling + validation

* signer, rules, storage: implement rules + ephemeral storage for signer rules

* signer: implement OnApprovedTx, change signing response (API BREAKAGE)

* signer: refactoring + documentation

* signer/rules: implement dispatching to next handler

* signer: docs

* signer/rules: hide json-conversion from users, ensure context is cleaned

* signer: docs

* signer: implement validation rules, change signature of call_info

* signer: fix log flaw with string pointer

* signer: implement custom 4byte databsae that saves submitted signatures

* signer/storage: implement aes-gcm-backed credential storage

* accounts: implement json unmarshalling of url

* signer: fix listresponse, fix gas->uint64

* node: make http/ipc start methods public

* signer: add ipc capability+review concerns

* accounts: correct docstring

* signer: address review concerns

* rpc: go fmt -s

* signer: review concerns+ baptize Clef

* signer,node: move Start-functions to separate file

* signer: formatting

3 files changed, 341 insertions, 0 deletions

diff --git a/signer/storage/aes_gcm_storage.go b/signer/storage/aes_gcm_storage.go
new file mode 100644
index 000000000..1ac347558
--- /dev/null
+++ b/signer/storage/aes_gcm_storage.go
@@ -0,0 +1,164 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+//
+package storage
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"os"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+type storedCredential struct {
+	// The iv
+	Iv []byte `json:"iv"`
+	// The ciphertext
+	CipherText []byte `json:"c"`
+}
+
+// AESEncryptedStorage is a storage type which is backed by a json-faile. The json-file contains
+// key-value mappings, where the keys are _not_ encrypted, only the values are.
+type AESEncryptedStorage struct {
+	// File to read/write credentials
+	filename string
+	// Key stored in base64
+	key []byte
+}
+
+// NewAESEncryptedStorage creates a new encrypted storage backed by the given file/key
+func NewAESEncryptedStorage(filename string, key []byte) *AESEncryptedStorage {
+	return &AESEncryptedStorage{
+		filename: filename,
+		key:      key,
+	}
+}
+
+// Put stores a value by key. 0-length keys results in no-op
+func (s *AESEncryptedStorage) Put(key, value string) {
+	if len(key) == 0 {
+		return
+	}
+	data, err := s.readEncryptedStorage()
+	if err != nil {
+		log.Warn("Failed to read encrypted storage", "err", err, "file", s.filename)
+		return
+	}
+	ciphertext, iv, err := encrypt(s.key, []byte(value))
+	if err != nil {
+		log.Warn("Failed to encrypt entry", "err", err)
+		return
+	}
+	encrypted := storedCredential{Iv: iv, CipherText: ciphertext}
+	data[key] = encrypted
+	if err = s.writeEncryptedStorage(data); err != nil {
+		log.Warn("Failed to write entry", "err", err)
+	}
+}
+
+// Get returns the previously stored value, or the empty string if it does not exist or key is of 0-length
+func (s *AESEncryptedStorage) Get(key string) string {
+	if len(key) == 0 {
+		return ""
+	}
+	data, err := s.readEncryptedStorage()
+	if err != nil {
+		log.Warn("Failed to read encrypted storage", "err", err, "file", s.filename)
+		return ""
+	}
+	encrypted, exist := data[key]
+	if !exist {
+		log.Warn("Key does not exist", "key", key)
+		return ""
+	}
+	entry, err := decrypt(s.key, encrypted.Iv, encrypted.CipherText)
+	if err != nil {
+		log.Warn("Failed to decrypt key", "key", key)
+		return ""
+	}
+	return string(entry)
+}
+
+// readEncryptedStorage reads the file with encrypted creds
+func (s *AESEncryptedStorage) readEncryptedStorage() (map[string]storedCredential, error) {
+	creds := make(map[string]storedCredential)
+	raw, err := ioutil.ReadFile(s.filename)
+
+	if err != nil {
+		if os.IsNotExist(err) {
+			// Doesn't exist yet
+			return creds, nil
+
+		} else {
+			log.Warn("Failed to read encrypted storage", "err", err, "file", s.filename)
+		}
+	}
+	if err = json.Unmarshal(raw, &creds); err != nil {
+		log.Warn("Failed to unmarshal encrypted storage", "err", err, "file", s.filename)
+		return nil, err
+	}
+	return creds, nil
+}
+
+// writeEncryptedStorage write the file with encrypted creds
+func (s *AESEncryptedStorage) writeEncryptedStorage(creds map[string]storedCredential) error {
+	raw, err := json.Marshal(creds)
+	if err != nil {
+		return err
+	}
+	if err = ioutil.WriteFile(s.filename, raw, 0600); err != nil {
+		return err
+	}
+	return nil
+}
+
+func encrypt(key []byte, plaintext []byte) ([]byte, []byte, error) {
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, nil, err
+	}
+	aesgcm, err := cipher.NewGCM(block)
+	nonce := make([]byte, aesgcm.NonceSize())
+	if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
+		return nil, nil, err
+	}
+	if err != nil {
+		return nil, nil, err
+	}
+	ciphertext := aesgcm.Seal(nil, nonce, plaintext, nil)
+	return ciphertext, nonce, nil
+}
+
+func decrypt(key []byte, nonce []byte, ciphertext []byte) ([]byte, error) {
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+	aesgcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return nil, err
+	}
+	plaintext, err := aesgcm.Open(nil, nonce, ciphertext, nil)
+	if err != nil {
+		return nil, err
+	}
+	return plaintext, nil
+}
diff --git a/signer/storage/aes_gcm_storage_test.go b/signer/storage/aes_gcm_storage_test.go
new file mode 100644
index 000000000..77804905a
--- /dev/null
+++ b/signer/storage/aes_gcm_storage_test.go
@@ -0,0 +1,115 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+//
+package storage
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/mattn/go-colorable"
+)
+
+func TestEncryption(t *testing.T) {
+	//	key := []byte("AES256Key-32Characters1234567890")
+	//	plaintext := []byte(value)
+	key := []byte("AES256Key-32Characters1234567890")
+	plaintext := []byte("exampleplaintext")
+
+	c, iv, err := encrypt(key, plaintext)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fmt.Printf("Ciphertext %x, nonce %x\n", c, iv)
+
+	p, err := decrypt(key, iv, c)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fmt.Printf("Plaintext %v\n", string(p))
+	if !bytes.Equal(plaintext, p) {
+		t.Errorf("Failed: expected plaintext recovery, got %v expected %v", string(plaintext), string(p))
+	}
+}
+
+func TestFileStorage(t *testing.T) {
+
+	a := map[string]storedCredential{
+		"secret": {
+			Iv:         common.Hex2Bytes("cdb30036279601aeee60f16b"),
+			CipherText: common.Hex2Bytes("f311ac49859d7260c2c464c28ffac122daf6be801d3cfd3edcbde7e00c9ff74f"),
+		},
+		"secret2": {
+			Iv:         common.Hex2Bytes("afb8a7579bf971db9f8ceeed"),
+			CipherText: common.Hex2Bytes("2df87baf86b5073ef1f03e3cc738de75b511400f5465bb0ddeacf47ae4dc267d"),
+		},
+	}
+	d, err := ioutil.TempDir("", "eth-encrypted-storage-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	stored := &AESEncryptedStorage{
+		filename: fmt.Sprintf("%v/vault.json", d),
+		key:      []byte("AES256Key-32Characters1234567890"),
+	}
+	stored.writeEncryptedStorage(a)
+	read := &AESEncryptedStorage{
+		filename: fmt.Sprintf("%v/vault.json", d),
+		key:      []byte("AES256Key-32Characters1234567890"),
+	}
+	creds, err := read.readEncryptedStorage()
+	if err != nil {
+		t.Fatal(err)
+	}
+	for k, v := range a {
+		if v2, exist := creds[k]; !exist {
+			t.Errorf("Missing entry %v", k)
+		} else {
+			if !bytes.Equal(v.CipherText, v2.CipherText) {
+				t.Errorf("Wrong ciphertext, expected %x got %x", v.CipherText, v2.CipherText)
+			}
+			if !bytes.Equal(v.Iv, v2.Iv) {
+				t.Errorf("Wrong iv")
+			}
+		}
+	}
+}
+func TestEnd2End(t *testing.T) {
+	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(3), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
+
+	d, err := ioutil.TempDir("", "eth-encrypted-storage-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s1 := &AESEncryptedStorage{
+		filename: fmt.Sprintf("%v/vault.json", d),
+		key:      []byte("AES256Key-32Characters1234567890"),
+	}
+	s2 := &AESEncryptedStorage{
+		filename: fmt.Sprintf("%v/vault.json", d),
+		key:      []byte("AES256Key-32Characters1234567890"),
+	}
+
+	s1.Put("bazonk", "foobar")
+	if v := s2.Get("bazonk"); v != "foobar" {
+		t.Errorf("Expected bazonk->foobar, got '%v'", v)
+	}
+}
diff --git a/signer/storage/storage.go b/signer/storage/storage.go
new file mode 100644
index 000000000..60f4e3892
--- /dev/null
+++ b/signer/storage/storage.go
@@ -0,0 +1,62 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+//
+
+package storage
+
+import (
+	"fmt"
+)
+
+type Storage interface {
+	// Put stores a value by key. 0-length keys results in no-op
+	Put(key, value string)
+	// Get returns the previously stored value, or the empty string if it does not exist or key is of 0-length
+	Get(key string) string
+}
+
+// EphemeralStorage is an in-memory storage that does
+// not persist values to disk. Mainly used for testing
+type EphemeralStorage struct {
+	data      map[string]string
+	namespace string
+}
+
+func (s *EphemeralStorage) Put(key, value string) {
+	if len(key) == 0 {
+		return
+	}
+	fmt.Printf("storage: put %v -> %v\n", key, value)
+	s.data[key] = value
+}
+
+func (s *EphemeralStorage) Get(key string) string {
+	if len(key) == 0 {
+		return ""
+	}
+	fmt.Printf("storage: get %v\n", key)
+	if v, exist := s.data[key]; exist {
+		return v
+	}
+	return ""
+}
+
+func NewEphemeralStorage() Storage {
+	s := &EphemeralStorage{
+		data: make(map[string]string),
+	}
+	return s
+}