swarm/storage/mru: Client-side MRU signatures (#784)

* swarm/storage/mru: Add embedded publickey and remove ENS dep

This commit breaks swarm, swarm/api...
but tests in swarm/storage/mru pass

* swarm: Refactor swarm, swarm/api to mru changes, make tests pass

* swarm/storage/mru: Remove self from recv, remove test ens vldtr

* swarm/storage/mru: Remove redundant test, expose ResourceHash mthd

* swarm/storage/mru: Make HeaderGetter mandatory + godoc fixes

* swarm/storage: Remove validator prefix for metadata chunk

* swarm/storage/mru: Use Address instead of PublicKey

* swarm/storage/mru: Change index from name to metadata chunk addr

* swarm/storage/mru: Refactor swarm/api/... to MRU index changes

* swarm/storage/mru: Refactor cleanup

* swarm/storage/mru: Rebase cleanup

* swarm: Use constructor for GenericSigner MRU in swarm.go

* swarm/storage: Change to BMTHash for MRU hashing

* swarm/storage: Reduce loglevel on chunk validator logs

* swarm/storage/mru: Delint

* swarm: MRU Rebase cleanup

* swarm/storage/mru: client-side mru signatures

Rebase to PR #668 and fix all conflicts

* swarm/storage/mru:  refactor and documentation

* swarm/resource/mru: error-checking  tests for parseUpdate/newUpdateChunk

* swarm/storage/mru: Added resourcemetadata tests

* swarm/storage/mru: Added tests  for UpdateRequest

* swarm/storage/mru: more test coverage for UpdateRequest and comments

* swarm/storage/mru: Avoid fake chunks in parseUpdate()

* swarm/storage/mru: Documented resource.go extensively

moved some functions where they make most sense

* swarm/storage/mru: increase test coverage for UpdateRequest and

variable name changes throughout to increase consistency

* swarm/storage/mru: moved default timestamp to NewCreateRequest-

* swarm/storage/mru: lookup refactor

* swarm/storage/mru: added comments and renamed raw flag to rawmru

* swarm/storage/mru: fix receiver typo

* swarm/storage/mru: refactored update chunk new/create

* swarm/storage/mru:  refactored signature digest to avoid malleability

* swarm/storage/mru: optimize update data serialization

* swarm/storage/mru: refactor and cleanup

* swarm/storage/mru: add timestamp struct and serialization

* swarm/storage/mru: fix lint error and mark some old code for deletion

* swarm/storage/mru: remove unnecessary variable

* swarm/storage/mru: Added more comments throughout

* swarm/storage/mru: Refactored metadata chunk layout + extensive error...

* swarm/storage/mru: refactor cli parser
Changed resource info output to JSON

* swarm/storage/mru: refactor serialization for extensibility

refactored error messages to NewErrorf

* swarm/storage/mru: Moved Signature to resource_sign.
Check Sign errors in server tests

* swarm/storage/mru: Remove isSafeName() checks

* swarm/storage/mru: scrubbed off all references to "block" for time

* swarm/storage/mru: removed superfluous isSynced() call.

* swarm/storage/mru: remove isMultihash() and ToSafeName functions

* swarm/storage/mru: various fixes and comments

* swarm/storage/mru: decoupled cli for independent create/update
* Made resource name optional
* Removed unused LookupPrevious

* swarm/storage/mru: Decoupled resource create / update & refactor

* swarm/storage/mru: Fixed some comments as per issues raised in PR #743

* swarm/storage/mru: Cosmetic changes as per #743 comments

* swarm/storage/mru: refct request encoder/decoder > marshal/unmarshal

* swarm/storage/mru: Cosmetic changes as per review in #748

* swarm/storage/mru: removed timestamp proof placeholder

* swarm/storage/mru: cosmetic/doc/fixes changes as per comments in #704

* swarm/storage/mru: removed unnecessary check in Handler.update

* swarm/storage/mru: Implemented Marshaler/Unmarshaler iface in Request

* swarm/storage/mru: Fixed linter error

* swarm/storage/mru: removed redundant address in signature digest

* swarm/storage/mru: fixed bug: LookupLatestVersionInPeriod not working

* swarm/storage/mru: Unfold Request creation API for create or update+create
set common time source for mru package

* swarm/api/http: fix HandleGetResource error variable shadowed
when requesting a resource that does not exist

* swarm/storage/mru: Add simple check to detect duplicate updates

* swarm/storage/mru: moved Multihash() to the right place.

* cmd/swarm: remove unneeded clientaccountmanager.go

* swarm/storage/mru: Changed some comments as per reviews in #784

* swarm/storage/mru: Made SignedResourceUpdate.GetDigest() public

* swarm/storage/mru: cosmetic changes as per comments in #784

* cmd/swarm: Inverted --multihash flag default

* swarm/storage/mru: removed Verify from SignedResourceUpdate.fromChunk

* swarm/storage/mru: Moved validation code out of serializer
Cosmetic / comment changes

* swarm/storage/mru: Added unit tests for UpdateLookup

* swarm/storage/mru: Increased coverage of metadata serialization

* swarm/storage/mru: Increased test coverage of updateHeader serializers

* swarm/storage/mru: Add resourceUpdate serializer test

1 files changed, 97 insertions, 13 deletions

diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go
index 8e1c26a33..e8bc1bdae 100644
--- a/swarm/api/http/server_test.go
+++ b/swarm/api/http/server_test.go
@@ -34,12 +34,13 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/swarm/api"
 	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
 	"github.com/ethereum/go-ethereum/swarm/multihash"
 	"github.com/ethereum/go-ethereum/swarm/storage"
+	"github.com/ethereum/go-ethereum/swarm/storage/mru"
 	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
@@ -94,6 +95,14 @@ func serverFunc(api *api.API) testutil.TestServer {
 	return NewServer(api, "")
 }
 
+func newTestSigner() (*mru.GenericSigner, error) {
+	privKey, err := crypto.HexToECDSA("deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef")
+	if err != nil {
+		return nil, err
+	}
+	return mru.NewGenericSigner(privKey), nil
+}
+
 // test the transparent resolving of multihash resource types with bzz:// scheme
 //
 // first upload data, and store the multihash to the resulting manifest in a resource update
@@ -101,6 +110,8 @@ func serverFunc(api *api.API) testutil.TestServer {
 // and raw retrieve of that hash should return the data
 func TestBzzResourceMultihash(t *testing.T) {
 
+	signer, _ := newTestSigner()
+
 	srv := testutil.NewTestSwarmServer(t, serverFunc)
 	defer srv.Close()
 
@@ -123,15 +134,35 @@ func TestBzzResourceMultihash(t *testing.T) {
 	s := common.FromHex(string(b))
 	mh := multihash.ToMultihash(s)
 
-	mhHex := hexutil.Encode(mh)
 	log.Info("added data", "manifest", string(b), "data", common.ToHex(mh))
 
 	// our mutable resource "name"
 	keybytes := "foo.eth"
 
+	updateRequest, err := mru.NewCreateUpdateRequest(&mru.ResourceMetadata{
+		Name:      keybytes,
+		Frequency: 13,
+		StartTime: srv.GetCurrentTime(),
+		Owner:     signer.Address(),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	updateRequest.SetData(mh, true)
+
+	if err := updateRequest.Sign(signer); err != nil {
+		t.Fatal(err)
+	}
+	log.Info("added data", "manifest", string(b), "data", common.ToHex(mh))
+
+	body, err := updateRequest.MarshalJSON()
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	// create the multihash update
-	url = fmt.Sprintf("%s/bzz-resource:/%s/13", srv.URL, keybytes)
-	resp, err = http.Post(url, "application/octet-stream", bytes.NewReader([]byte(mhHex)))
+	url = fmt.Sprintf("%s/bzz-resource:/", srv.URL)
+	resp, err = http.Post(url, "application/json", bytes.NewReader(body))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -149,9 +180,9 @@ func TestBzzResourceMultihash(t *testing.T) {
 		t.Fatalf("data %s could not be unmarshaled: %v", b, err)
 	}
 
-	correctManifestAddrHex := "d689648fb9e00ddc7ebcf474112d5881c5bf7dbc6e394681b1d224b11b59b5e0"
+	correctManifestAddrHex := "6d3bc4664c97d8b821cb74bcae43f592494fb46d2d9cd31e69f3c7c802bbbd8e"
 	if rsrcResp.Hex() != correctManifestAddrHex {
-		t.Fatalf("Response resource key mismatch, expected '%s', got '%s'", correctManifestAddrHex, rsrcResp)
+		t.Fatalf("Response resource key mismatch, expected '%s', got '%s'", correctManifestAddrHex, rsrcResp.Hex())
 	}
 
 	// get bzz manifest transparent resource resolve
@@ -176,6 +207,8 @@ func TestBzzResourceMultihash(t *testing.T) {
 // Test resource updates using the raw update methods
 func TestBzzResource(t *testing.T) {
 	srv := testutil.NewTestSwarmServer(t, serverFunc)
+	signer, _ := newTestSigner()
+
 	defer srv.Close()
 
 	// our mutable resource "name"
@@ -188,9 +221,29 @@ func TestBzzResource(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	updateRequest, err := mru.NewCreateUpdateRequest(&mru.ResourceMetadata{
+		Name:      keybytes,
+		Frequency: 13,
+		StartTime: srv.GetCurrentTime(),
+		Owner:     signer.Address(),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	updateRequest.SetData(databytes, false)
+
+	if err := updateRequest.Sign(signer); err != nil {
+		t.Fatal(err)
+	}
+
+	body, err := updateRequest.MarshalJSON()
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	// creates resource and sets update 1
-	url := fmt.Sprintf("%s/bzz-resource:/%s/raw/13", srv.URL, []byte(keybytes))
-	resp, err := http.Post(url, "application/octet-stream", bytes.NewReader(databytes))
+	url := fmt.Sprintf("%s/bzz-resource:/", srv.URL)
+	resp, err := http.Post(url, "application/json", bytes.NewReader(body))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -208,7 +261,7 @@ func TestBzzResource(t *testing.T) {
 		t.Fatalf("data %s could not be unmarshaled: %v", b, err)
 	}
 
-	correctManifestAddrHex := "d689648fb9e00ddc7ebcf474112d5881c5bf7dbc6e394681b1d224b11b59b5e0"
+	correctManifestAddrHex := "6d3bc4664c97d8b821cb74bcae43f592494fb46d2d9cd31e69f3c7c802bbbd8e"
 	if rsrcResp.Hex() != correctManifestAddrHex {
 		t.Fatalf("Response resource key mismatch, expected '%s', got '%s'", correctManifestAddrHex, rsrcResp.Hex())
 	}
@@ -235,8 +288,7 @@ func TestBzzResource(t *testing.T) {
 	if len(manifest.Entries) != 1 {
 		t.Fatalf("Manifest has %d entries", len(manifest.Entries))
 	}
-
-	correctRootKeyHex := "f667277e004e8486c7a3631fd226802430e84e9a81b6085d31f512a591ae0065"
+	correctRootKeyHex := "68f7ba07ac8867a4c841a4d4320e3cdc549df23702dc7285fcb6acf65df48562"
 	if manifest.Entries[0].Hash != correctRootKeyHex {
 		t.Fatalf("Expected manifest path '%s', got '%s'", correctRootKeyHex, manifest.Entries[0].Hash)
 	}
@@ -262,6 +314,11 @@ func TestBzzResource(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+
+	if resp.StatusCode != http.StatusNotFound {
+		t.Fatalf("Expected get non-existent resource to fail with StatusNotFound (404), got %d", resp.StatusCode)
+	}
+
 	resp.Body.Close()
 
 	// get latest update (1.1) through resource directly
@@ -285,9 +342,36 @@ func TestBzzResource(t *testing.T) {
 
 	// update 2
 	log.Info("update 2")
-	url = fmt.Sprintf("%s/bzz-resource:/%s/raw", srv.URL, correctManifestAddrHex)
+
+	// 1.- get metadata about this resource
+	url = fmt.Sprintf("%s/bzz-resource:/%s/", srv.URL, correctManifestAddrHex)
+	resp, err = http.Get(url + "meta")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("Get resource metadata returned %s", resp.Status)
+	}
+	b, err = ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	updateRequest = &mru.Request{}
+	if err = updateRequest.UnmarshalJSON(b); err != nil {
+		t.Fatalf("Error decoding resource metadata: %s", err)
+	}
 	data := []byte("foo")
-	resp, err = http.Post(url, "application/octet-stream", bytes.NewReader(data))
+	updateRequest.SetData(data, false)
+	if err = updateRequest.Sign(signer); err != nil {
+		t.Fatal(err)
+	}
+	body, err = updateRequest.MarshalJSON()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	resp, err = http.Post(url, "application/json", bytes.NewReader(body))
 	if err != nil {
 		t.Fatal(err)
 	}