accounts, eth, clique, signer: support for external signer API (#18079)

* accounts, eth, clique: implement external backend + move sighash calc to backend

* signer: implement account_Version on external API

* accounts/external: enable ipc, add copyright

* accounts, internal, signer: formatting

* node: go fmt

* flags: disallow --dev in combo with --externalsigner

* accounts: remove clique-specific signing method, replace with more generic

* accounts, consensus: formatting + fix error in tests

* signer/core: remove (test-) import cycle

* clique: remove unused import

* accounts: remove CliqueHash and avoid dependency on package crypto

* consensus/clique: unduplicate header encoding

5 files changed, 311 insertions, 12 deletions

diff --git a/accounts/accounts.go b/accounts/accounts.go
index cb1eae281..11232b19a 100644
--- a/accounts/accounts.go
+++ b/accounts/accounts.go
@@ -18,12 +18,14 @@
 package accounts
 
 import (
+	"fmt"
 	"math/big"
 
 	ethereum "github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/event"
+	"golang.org/x/crypto/sha3"
 )
 
 // Account represents an Ethereum account located at a specific location defined
@@ -87,8 +89,20 @@ type Wallet interface {
 	// chain state reader.
 	SelfDerive(base DerivationPath, chain ethereum.ChainStateReader)
 
-	// SignHash requests the wallet to sign the given hash.
+	// SignData requests the wallet to sign the hash of the given data
+	// It looks up the account specified either solely via its address contained within,
+	// or optionally with the aid of any location metadata from the embedded URL field.
 	//
+	// If the wallet requires additional authentication to sign the request (e.g.
+	// a password to decrypt the account, or a PIN code o verify the transaction),
+	// an AuthNeededError instance will be returned, containing infos for the user
+	// about which fields or actions are needed. The user may retry by providing
+	// the needed details via SignHashWithPassphrase, or by other means (e.g. unlock
+	// the account in a keystore).
+	SignData(account Account, mimeType string, data []byte) ([]byte, error)
+
+	// Signtext requests the wallet to sign the hash of a given piece of data, prefixed
+	// by the Ethereum prefix scheme
 	// It looks up the account specified either solely via its address contained within,
 	// or optionally with the aid of any location metadata from the embedded URL field.
 	//
@@ -98,7 +112,7 @@ type Wallet interface {
 	// about which fields or actions are needed. The user may retry by providing
 	// the needed details via SignHashWithPassphrase, or by other means (e.g. unlock
 	// the account in a keystore).
-	SignHash(account Account, hash []byte) ([]byte, error)
+	SignText(account Account, text []byte) ([]byte, error)
 
 	// SignTx requests the wallet to sign the given transaction.
 	//
@@ -113,12 +127,12 @@ type Wallet interface {
 	// the account in a keystore).
 	SignTx(account Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error)
 
-	// SignHashWithPassphrase requests the wallet to sign the given hash with the
+	// SignTextWithPassphrase requests the wallet to sign the given text with the
 	// given passphrase as extra authentication information.
 	//
 	// It looks up the account specified either solely via its address contained within,
 	// or optionally with the aid of any location metadata from the embedded URL field.
-	SignHashWithPassphrase(account Account, passphrase string, hash []byte) ([]byte, error)
+	SignTextWithPassphrase(account Account, passphrase string, hash []byte) ([]byte, error)
 
 	// SignTxWithPassphrase requests the wallet to sign the given transaction, with the
 	// given passphrase as extra authentication information.
@@ -148,6 +162,19 @@ type Backend interface {
 	Subscribe(sink chan<- WalletEvent) event.Subscription
 }
 
+// TextHash is a helper function that calculates a hash for the given message that can be
+// safely used to calculate a signature from.
+//
+// The hash is calulcated as
+//   keccak256("\x19Ethereum Signed Message:\n"${message length}${message}).
+//
+// This gives context to the signed message and prevents signing of transactions.
+func TextHash(data []byte) []byte {
+	hash := sha3.NewLegacyKeccak256()
+	fmt.Fprintf(hash, "\x19Ethereum Signed Message:\n%d%s", len(data), data)
+	return hash.Sum(nil)
+}
+
 // WalletEventType represents the different event types that can be fired by
 // the wallet subscription subsystem.
 type WalletEventType int
diff --git a/accounts/accounts_test.go b/accounts/accounts_test.go
new file mode 100644
index 000000000..a49e3954e
--- /dev/null
+++ b/accounts/accounts_test.go
@@ -0,0 +1,32 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package accounts
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+)
+
+func TestTextHash(t *testing.T) {
+	hash := TextHash([]byte("Hello Joe"))
+	want := hexutil.MustDecode("0xa080337ae51c4e064c189e113edd0ba391df9206e2f49db658bb32cf2911730b")
+	if !bytes.Equal(hash, want) {
+		t.Fatalf("wrong hash: %x", hash)
+	}
+}
diff --git a/accounts/external/backend.go b/accounts/external/backend.go
new file mode 100644
index 000000000..35b9c276d
--- /dev/null
+++ b/accounts/external/backend.go
@@ -0,0 +1,220 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package external
+
+import (
+	"fmt"
+	"math/big"
+	"sync"
+
+	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/event"
+	"github.com/ethereum/go-ethereum/internal/ethapi"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/signer/core"
+)
+
+type ExternalBackend struct {
+	signers []accounts.Wallet
+}
+
+func (eb *ExternalBackend) Wallets() []accounts.Wallet {
+	return eb.signers
+}
+
+func NewExternalBackend(endpoint string) (*ExternalBackend, error) {
+	signer, err := NewExternalSigner(endpoint)
+	if err != nil {
+		return nil, err
+	}
+	return &ExternalBackend{
+		signers: []accounts.Wallet{signer},
+	}, nil
+}
+
+func (eb *ExternalBackend) Subscribe(sink chan<- accounts.WalletEvent) event.Subscription {
+	return event.NewSubscription(func(quit <-chan struct{}) error {
+		<-quit
+		return nil
+	})
+}
+
+// ExternalSigner provides an API to interact with an external signer (clef)
+// It proxies request to the external signer while forwarding relevant
+// request headers
+type ExternalSigner struct {
+	client   *rpc.Client
+	endpoint string
+	status   string
+	cacheMu  sync.RWMutex
+	cache    []accounts.Account
+}
+
+func NewExternalSigner(endpoint string) (*ExternalSigner, error) {
+	client, err := rpc.Dial(endpoint)
+	if err != nil {
+		return nil, err
+	}
+	extsigner := &ExternalSigner{
+		client:   client,
+		endpoint: endpoint,
+	}
+	// Check if reachable
+	version, err := extsigner.pingVersion()
+	if err != nil {
+		return nil, err
+	}
+	extsigner.status = fmt.Sprintf("ok [version=%v]", version)
+	return extsigner, nil
+}
+
+func (api *ExternalSigner) URL() accounts.URL {
+	return accounts.URL{
+		Scheme: "extapi",
+		Path:   api.endpoint,
+	}
+}
+
+func (api *ExternalSigner) Status() (string, error) {
+	return api.status, nil
+}
+
+func (api *ExternalSigner) Open(passphrase string) error {
+	return fmt.Errorf("operation not supported on external signers")
+}
+
+func (api *ExternalSigner) Close() error {
+	return fmt.Errorf("operation not supported on external signers")
+}
+
+func (api *ExternalSigner) Accounts() []accounts.Account {
+	var accnts []accounts.Account
+	res, err := api.listAccounts()
+	if err != nil {
+		log.Error("account listing failed", "error", err)
+		return accnts
+	}
+	for _, addr := range res {
+		accnts = append(accnts, accounts.Account{
+			URL: accounts.URL{
+				Scheme: "extapi",
+				Path:   api.endpoint,
+			},
+			Address: addr,
+		})
+	}
+	api.cacheMu.Lock()
+	api.cache = accnts
+	api.cacheMu.Unlock()
+	return accnts
+}
+
+func (api *ExternalSigner) Contains(account accounts.Account) bool {
+	api.cacheMu.RLock()
+	defer api.cacheMu.RUnlock()
+	for _, a := range api.cache {
+		if a.Address == account.Address && (account.URL == (accounts.URL{}) || account.URL == api.URL()) {
+			return true
+		}
+	}
+	return false
+}
+
+func (api *ExternalSigner) Derive(path accounts.DerivationPath, pin bool) (accounts.Account, error) {
+	return accounts.Account{}, fmt.Errorf("operation not supported on external signers")
+}
+
+func (api *ExternalSigner) SelfDerive(base accounts.DerivationPath, chain ethereum.ChainStateReader) {
+	log.Error("operation SelfDerive not supported on external signers")
+}
+
+func (api *ExternalSigner) signHash(account accounts.Account, hash []byte) ([]byte, error) {
+	return []byte{}, fmt.Errorf("operation not supported on external signers")
+}
+
+// SignData signs keccak256(data). The mimetype parameter describes the type of data being signed
+func (api *ExternalSigner) SignData(account accounts.Account, mimeType string, data []byte) ([]byte, error) {
+	// TODO! Replace this with a call to clef SignData with correct mime-type for Clique, once we
+	// have that in place
+	return api.signHash(account, crypto.Keccak256(data))
+}
+
+func (api *ExternalSigner) SignText(account accounts.Account, text []byte) ([]byte, error) {
+	return api.signHash(account, accounts.TextHash(text))
+}
+
+func (api *ExternalSigner) SignTx(account accounts.Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
+	res := ethapi.SignTransactionResult{}
+	to := common.NewMixedcaseAddress(*tx.To())
+	data := hexutil.Bytes(tx.Data())
+	args := &core.SendTxArgs{
+		Data:     &data,
+		Nonce:    hexutil.Uint64(tx.Nonce()),
+		Value:    hexutil.Big(*tx.Value()),
+		Gas:      hexutil.Uint64(tx.Gas()),
+		GasPrice: hexutil.Big(*tx.GasPrice()),
+		To:       &to,
+		From:     common.NewMixedcaseAddress(account.Address),
+	}
+
+	if err := api.client.Call(&res, "account_signTransaction", args); err != nil {
+		return nil, err
+	}
+	return res.Tx, nil
+}
+
+func (api *ExternalSigner) SignTextWithPassphrase(account accounts.Account, passphrase string, text []byte) ([]byte, error) {
+	return []byte{}, fmt.Errorf("operation not supported on external signers")
+}
+
+func (api *ExternalSigner) SignTxWithPassphrase(account accounts.Account, passphrase string, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
+	return nil, fmt.Errorf("operation not supported on external signers")
+}
+
+func (api *ExternalSigner) listAccounts() ([]common.Address, error) {
+	var res []common.Address
+	if err := api.client.Call(&res, "account_list"); err != nil {
+		return nil, err
+	}
+	return res, nil
+}
+
+func (api *ExternalSigner) signCliqueBlock(a common.Address, rlpBlock hexutil.Bytes) (hexutil.Bytes, error) {
+	var sig hexutil.Bytes
+	if err := api.client.Call(&sig, "account_signData", "application/clique", a, rlpBlock); err != nil {
+		return nil, err
+	}
+	if sig[64] != 27 && sig[64] != 28 {
+		return nil, fmt.Errorf("invalid Ethereum signature (V is not 27 or 28)")
+	}
+	sig[64] -= 27 // Transform V from 27/28 to 0/1 for Clique use
+	return sig, nil
+}
+
+func (api *ExternalSigner) pingVersion() (string, error) {
+	var v string
+	if err := api.client.Call(&v, "account_version"); err != nil {
+		return "", err
+	}
+	return v, nil
+}
diff --git a/accounts/keystore/wallet.go b/accounts/keystore/wallet.go
index 2f774cc94..0490f39ff 100644
--- a/accounts/keystore/wallet.go
+++ b/accounts/keystore/wallet.go
@@ -22,6 +22,7 @@ import (
 	ethereum "github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
 )
 
 // keystoreWallet implements the accounts.Wallet interface for the original
@@ -78,11 +79,11 @@ func (w *keystoreWallet) Derive(path accounts.DerivationPath, pin bool) (account
 // there is no notion of hierarchical account derivation for plain keystore accounts.
 func (w *keystoreWallet) SelfDerive(base accounts.DerivationPath, chain ethereum.ChainStateReader) {}
 
-// SignHash implements accounts.Wallet, attempting to sign the given hash with
+// signHash attempts to sign the given hash with
 // the given account. If the wallet does not wrap this particular account, an
 // error is returned to avoid account leakage (even though in theory we may be
 // able to sign via our shared keystore backend).
-func (w *keystoreWallet) SignHash(account accounts.Account, hash []byte) ([]byte, error) {
+func (w *keystoreWallet) signHash(account accounts.Account, hash []byte) ([]byte, error) {
 	// Make sure the requested account is contained within
 	if !w.Contains(account) {
 		return nil, accounts.ErrUnknownAccount
@@ -91,6 +92,15 @@ func (w *keystoreWallet) SignHash(account accounts.Account, hash []byte) ([]byte
 	return w.keystore.SignHash(account, hash)
 }
 
+// SignData signs keccak256(data). The mimetype parameter describes the type of data being signed
+func (w *keystoreWallet) SignData(account accounts.Account, mimeType string, data []byte) ([]byte, error) {
+	return w.signHash(account, crypto.Keccak256(data))
+}
+
+func (w *keystoreWallet) SignText(account accounts.Account, text []byte) ([]byte, error) {
+	return w.signHash(account, accounts.TextHash(text))
+}
+
 // SignTx implements accounts.Wallet, attempting to sign the given transaction
 // with the given account. If the wallet does not wrap this particular account,
 // an error is returned to avoid account leakage (even though in theory we may
@@ -106,13 +116,13 @@ func (w *keystoreWallet) SignTx(account accounts.Account, tx *types.Transaction,
 
 // SignHashWithPassphrase implements accounts.Wallet, attempting to sign the
 // given hash with the given account using passphrase as extra authentication.
-func (w *keystoreWallet) SignHashWithPassphrase(account accounts.Account, passphrase string, hash []byte) ([]byte, error) {
+func (w *keystoreWallet) SignTextWithPassphrase(account accounts.Account, passphrase string, text []byte) ([]byte, error) {
 	// Make sure the requested account is contained within
 	if !w.Contains(account) {
 		return nil, accounts.ErrUnknownAccount
 	}
 	// Account seems valid, request the keystore to sign
-	return w.keystore.SignHashWithPassphrase(account, passphrase, hash)
+	return w.keystore.SignHashWithPassphrase(account, passphrase, accounts.TextHash(text))
 }
 
 // SignTxWithPassphrase implements accounts.Wallet, attempting to sign the given
diff --git a/accounts/usbwallet/wallet.go b/accounts/usbwallet/wallet.go
index 6cef6e0fb..a99dcd0f5 100644
--- a/accounts/usbwallet/wallet.go
+++ b/accounts/usbwallet/wallet.go
@@ -29,6 +29,7 @@ import (
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/karalabe/hid"
 )
@@ -495,12 +496,21 @@ func (w *wallet) SelfDerive(base accounts.DerivationPath, chain ethereum.ChainSt
 	w.deriveChain = chain
 }
 
-// SignHash implements accounts.Wallet, however signing arbitrary data is not
+// signHash implements accounts.Wallet, however signing arbitrary data is not
 // supported for hardware wallets, so this method will always return an error.
-func (w *wallet) SignHash(account accounts.Account, hash []byte) ([]byte, error) {
+func (w *wallet) signHash(account accounts.Account, hash []byte) ([]byte, error) {
 	return nil, accounts.ErrNotSupported
 }
 
+// SignData signs keccak256(data). The mimetype parameter describes the type of data being signed
+func (w *wallet) SignData(account accounts.Account, mimeType string, data []byte) ([]byte, error) {
+	return w.signHash(account, crypto.Keccak256(data))
+}
+
+func (w *wallet) SignText(account accounts.Account, text []byte) ([]byte, error) {
+	return w.signHash(account, accounts.TextHash(text))
+}
+
 // SignTx implements accounts.Wallet. It sends the transaction over to the Ledger
 // wallet to request a confirmation from the user. It returns either the signed
 // transaction or a failure if the user denied the transaction.
@@ -550,8 +560,8 @@ func (w *wallet) SignTx(account accounts.Account, tx *types.Transaction, chainID
 // SignHashWithPassphrase implements accounts.Wallet, however signing arbitrary
 // data is not supported for Ledger wallets, so this method will always return
 // an error.
-func (w *wallet) SignHashWithPassphrase(account accounts.Account, passphrase string, hash []byte) ([]byte, error) {
-	return w.SignHash(account, hash)
+func (w *wallet) SignTextWithPassphrase(account accounts.Account, passphrase string, text []byte) ([]byte, error) {
+	return w.SignText(account, accounts.TextHash(text))
 }
 
 // SignTxWithPassphrase implements accounts.Wallet, attempting to sign the given