Swarm MRUs: Adaptive frequency / Predictable lookups / API simplification (#17559)

* swarm/storage/mru: Adaptive Frequency

swarm/storage/mru/lookup: fixed getBaseTime
Added NewEpoch constructor

swarm/api/client: better error handling in GetResource()


swarm/storage/mru: Renamed structures.
Renamed ResourceMetadata to ResourceID. 
Renamed ResourceID.Name to ResourceID.Topic

swarm/storage/mru: Added binarySerializer interface and test tools

swarm/storage/mru/lookup: Changed base time to time and + marshallers

swarm/storage/mru:  Added ResourceID (former resourceMetadata)

swarm/storage/mru: Added ResourceViewId and serialization tests

swarm/storage/mru/lookup: fixed epoch unmarshaller. Added Epoch Equals

swarm/storage/mru: Fixes as per review comments

cmd/swarm: reworded resource create/update help text regarding topic

swarm/storage/mru: Added UpdateLookup and serializer tests

swarm/storage/mru: Added UpdateHeader, serializers and tests

swarm/storage/mru: changed UpdateAddr / epoch to Base()

swarm/storage/mru: Added resourceUpdate serializer and tests

swarm/storage/mru: Added SignedResourceUpdate tests and serializers

swarm/storage/mru/lookup: fixed GetFirstEpoch bug

swarm/storage/mru: refactor, comments, cleanup

Also added tests for Topic
swarm/storage/mru: handler tests pass

swarm/storage/mru: all resource package tests pass

swarm/storage/mru: resource test pass after adding
timestamp checking support

swarm/storage/mru: Added JSON serializers to ResourceIDView structures

swarm/storage/mru: Sever, client, API test pass

swarm/storage/mru: server test pass

swarm/storage/mru: Added topic length check

swarm/storage/mru: removed some literals,
improved "previous lookup" test case

swarm/storage/mru: some fixes and comments as per review

swarm/storage/mru: first working version without metadata chunk

swarm/storage/mru: Various fixes as per review

swarm/storage/mru: client test pass

swarm/storage/mru: resource query strings and manifest-less queries


swarm/storage/mru: simplify naming

swarm/storage/mru: first autofreq working version



swarm/storage/mru: renamed ToValues to AppendValues

swarm/resource/mru: Added ToValues / FromValues for URL query strings

swarm/storage/mru: Changed POST resource to work with query strings.
No more JSON.

swarm/storage/mru: removed resourceid

swarm/storage/mru: Opened up structures

swarm/storage/mru: Merged Request and SignedResourceUpdate

swarm/storage/mru: removed initial data from CLI resource create

swarm/storage/mru: Refactor Topic as a direct fixed-length array

swarm/storage/mru/lookup: Comprehensive GetNextLevel tests

swarm/storage/mru: Added comments

Added length checks in Topic
swarm/storage/mru: fixes in tests and some code comments

swarm/storage/mru/lookup: new optimized lookup algorithm

swarm/api: moved getResourceView to api out of server

swarm/storage/mru: Lookup algorithm working

swarm/storage/mru: comments and renamed NewLookupParams

Deleted commented code


swarm/storage/mru/lookup: renamed Epoch.LaterThan to After

swarm/storage/mru/lookup: Comments and tidying naming



swarm/storage/mru: fix lookup algorithm

swarm/storage/mru: exposed lookup hint
removed updateheader

swarm/storage/mru/lookup: changed GetNextEpoch for initial values

swarm/storage/mru: resource tests pass

swarm/storage/mru: valueSerializer interface and tests



swarm/storage/mru/lookup: Comments, improvements, fixes, more tests

swarm/storage/mru: renamed UpdateLookup to ID, LookupParams to Query

swarm/storage/mru: renamed query receiver var



swarm/cmd: MRU CLI tests

* cmd/swarm: remove rogue fmt

* swarm/storage/mru: Add version / header for future use

* swarm/storage/mru: Fixes/comments as per review

cmd/swarm: remove rogue fmt

swarm/storage/mru: Add version / header for future use-

* swarm/storage/mru: fix linter errors

* cmd/swarm: Speeded up TestCLIResourceUpdate

4 files changed, 301 insertions, 82 deletions

diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go
index 71d707c2b..f1bec770e 100644
--- a/cmd/swarm/main.go
+++ b/cmd/swarm/main.go
@@ -203,21 +203,29 @@ var (
 		Usage:  "Number of recent chunks cached in memory (default 5000)",
 		EnvVar: SWARM_ENV_STORE_CACHE_CAPACITY,
 	}
-	SwarmResourceMultihashFlag = cli.BoolFlag{
-		Name:  "multihash",
-		Usage: "Determines how to interpret data for a resource update. If not present, data will be interpreted as raw, literal data that will be included in the resource",
+	SwarmCompressedFlag = cli.BoolFlag{
+		Name:  "compressed",
+		Usage: "Prints encryption keys in compressed form",
 	}
 	SwarmResourceNameFlag = cli.StringFlag{
 		Name:  "name",
-		Usage: "User-defined name for the new resource",
+		Usage: "User-defined name for the new resource, limited to 32 characters. If combined with topic, the resource will be a subtopic with this name",
+	}
+	SwarmResourceTopicFlag = cli.StringFlag{
+		Name:  "topic",
+		Usage: "User-defined topic this resource is tracking, hex encoded. Limited to 64 hexadecimal characters",
 	}
 	SwarmResourceDataOnCreateFlag = cli.StringFlag{
 		Name:  "data",
 		Usage: "Initializes the resource with the given hex-encoded data. Data must be prefixed by 0x",
 	}
-	SwarmCompressedFlag = cli.BoolFlag{
-		Name:  "compressed",
-		Usage: "Prints encryption keys in compressed form",
+	SwarmResourceManifestFlag = cli.StringFlag{
+		Name:  "manifest",
+		Usage: "Refers to the resource through a manifest",
+	}
+	SwarmResourceUserFlag = cli.StringFlag{
+		Name:  "user",
+		Usage: "Indicates the user who updates the resource",
 	}
 )
 
@@ -347,27 +355,53 @@ func init() {
 					Action:             resourceCreate,
 					CustomHelpTemplate: helpTemplate,
 					Name:               "create",
-					Usage:              "creates a new Mutable Resource",
-					ArgsUsage:          "<frequency>",
-					Description:        "creates a new Mutable Resource",
-					Flags:              []cli.Flag{SwarmResourceNameFlag, SwarmResourceDataOnCreateFlag, SwarmResourceMultihashFlag},
+					Usage:              "creates and publishes a new Mutable Resource manifest",
+					Description: `creates and publishes a new Mutable Resource manifest pointing to a specified user's updates about a particular topic.
+					The resource topic can be built in the following ways:
+					* use --topic to set the topic to an arbitrary binary hex string.
+					* use --name to set the topic to a human-readable name.
+					    For example --name could be set to "profile-picture", meaning this Mutable Resource allows to get this user's current profile picture.
+					* use both --topic and --name to create named subtopics. 
+						For example, --topic could be set to an Ethereum contract address and --name could be set to "comments", meaning
+						the Mutable Resource tracks a discussion about that contract.
+					The --user flag allows to have this manifest refer to a user other than yourself. If not specified,
+					it will then default to your local account (--bzzaccount)`,
+					Flags: []cli.Flag{SwarmResourceNameFlag, SwarmResourceTopicFlag, SwarmResourceUserFlag},
 				},
 				{
 					Action:             resourceUpdate,
 					CustomHelpTemplate: helpTemplate,
 					Name:               "update",
 					Usage:              "updates the content of an existing Mutable Resource",
-					ArgsUsage:          "<Manifest Address or ENS domain> <0x Hex data>",
-					Description:        "updates the content of an existing Mutable Resource",
-					Flags:              []cli.Flag{SwarmResourceMultihashFlag},
+					ArgsUsage:          "<0x Hex data>",
+					Description: `publishes a new update on the specified topic
+					The resource topic can be built in the following ways:
+					* use --topic to set the topic to an arbitrary binary hex string.
+					* use --name to set the topic to a human-readable name.
+					    For example --name could be set to "profile-picture", meaning this Mutable Resource allows to get this user's current profile picture.
+					* use both --topic and --name to create named subtopics. 
+						For example, --topic could be set to an Ethereum contract address and --name could be set to "comments", meaning
+						the Mutable Resource tracks a discussion about that contract.
+					
+					If you have a manifest, you can specify it with --manifest to refer to the resource,
+					instead of using --topic / --name
+					`,
+					Flags: []cli.Flag{SwarmResourceManifestFlag, SwarmResourceNameFlag, SwarmResourceTopicFlag},
 				},
 				{
 					Action:             resourceInfo,
 					CustomHelpTemplate: helpTemplate,
 					Name:               "info",
 					Usage:              "obtains information about an existing Mutable Resource",
-					ArgsUsage:          "<Manifest Address or ENS domain>",
-					Description:        "obtains information about an existing Mutable Resource",
+					Description: `obtains information about an existing Mutable Resource
+					The topic can be specified directly with the --topic flag as an hex string
+					If no topic is specified, the default topic (zero) will be used
+					The --name flag can be used to specify subtopics with a specific name.
+					The --user flag allows to refer to a user other than yourself. If not specified,
+					it will then default to your local account (--bzzaccount)
+					If you have a manifest, you can specify it with --manifest instead of --topic / --name / ---user
+					to refer to the resource`,
+					Flags: []cli.Flag{SwarmResourceManifestFlag, SwarmResourceNameFlag, SwarmResourceTopicFlag, SwarmResourceUserFlag},
 				},
 			},
 		},
diff --git a/cmd/swarm/mru.go b/cmd/swarm/mru.go
index 6176b6d6c..cc7f634cb 100644
--- a/cmd/swarm/mru.go
+++ b/cmd/swarm/mru.go
@@ -19,10 +19,11 @@ package main
 
 import (
 	"fmt"
-	"strconv"
 	"strings"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/crypto"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
@@ -34,62 +35,38 @@ func NewGenericSigner(ctx *cli.Context) mru.Signer {
 	return mru.NewGenericSigner(getPrivKey(ctx))
 }
 
+func getTopic(ctx *cli.Context) (topic mru.Topic) {
+	var name = ctx.String(SwarmResourceNameFlag.Name)
+	var relatedTopic = ctx.String(SwarmResourceTopicFlag.Name)
+	var relatedTopicBytes []byte
+	var err error
+
+	if relatedTopic != "" {
+		relatedTopicBytes, err = hexutil.Decode(relatedTopic)
+		if err != nil {
+			utils.Fatalf("Error parsing topic: %s", err)
+		}
+	}
+
+	topic, err = mru.NewTopic(name, relatedTopicBytes)
+	if err != nil {
+		utils.Fatalf("Error parsing topic: %s", err)
+	}
+	return topic
+}
+
 // swarm resource create <frequency> [--name <name>] [--data <0x Hexdata> [--multihash=false]]
 // swarm resource update <Manifest Address or ENS domain> <0x Hexdata> [--multihash=false]
 // swarm resource info <Manifest Address or ENS domain>
 
 func resourceCreate(ctx *cli.Context) {
-	args := ctx.Args()
-
 	var (
-		bzzapi      = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client      = swarm.NewClient(bzzapi)
-		multihash   = ctx.Bool(SwarmResourceMultihashFlag.Name)
-		initialData = ctx.String(SwarmResourceDataOnCreateFlag.Name)
-		name        = ctx.String(SwarmResourceNameFlag.Name)
+		bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		client = swarm.NewClient(bzzapi)
 	)
 
-	if len(args) < 1 {
-		fmt.Println("Incorrect number of arguments")
-		cli.ShowCommandHelpAndExit(ctx, "create", 1)
-		return
-	}
-	signer := NewGenericSigner(ctx)
-	frequency, err := strconv.ParseUint(args[0], 10, 64)
-	if err != nil {
-		fmt.Printf("Frequency formatting error: %s\n", err.Error())
-		cli.ShowCommandHelpAndExit(ctx, "create", 1)
-		return
-	}
-
-	metadata := mru.ResourceMetadata{
-		Name:      name,
-		Frequency: frequency,
-		Owner:     signer.Address(),
-	}
-
-	var newResourceRequest *mru.Request
-	if initialData != "" {
-		initialDataBytes, err := hexutil.Decode(initialData)
-		if err != nil {
-			fmt.Printf("Error parsing data: %s\n", err.Error())
-			cli.ShowCommandHelpAndExit(ctx, "create", 1)
-			return
-		}
-		newResourceRequest, err = mru.NewCreateUpdateRequest(&metadata)
-		if err != nil {
-			utils.Fatalf("Error creating new resource request: %s", err)
-		}
-		newResourceRequest.SetData(initialDataBytes, multihash)
-		if err = newResourceRequest.Sign(signer); err != nil {
-			utils.Fatalf("Error signing resource update: %s", err.Error())
-		}
-	} else {
-		newResourceRequest, err = mru.NewCreateRequest(&metadata)
-		if err != nil {
-			utils.Fatalf("Error creating new resource request: %s", err)
-		}
-	}
+	newResourceRequest := mru.NewFirstRequest(getTopic(ctx))
+	newResourceRequest.View.User = resourceGetUser(ctx)
 
 	manifestAddress, err := client.CreateResource(newResourceRequest)
 	if err != nil {
@@ -104,32 +81,43 @@ func resourceUpdate(ctx *cli.Context) {
 	args := ctx.Args()
 
 	var (
-		bzzapi    = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client    = swarm.NewClient(bzzapi)
-		multihash = ctx.Bool(SwarmResourceMultihashFlag.Name)
+		bzzapi                  = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		client                  = swarm.NewClient(bzzapi)
+		manifestAddressOrDomain = ctx.String(SwarmResourceManifestFlag.Name)
 	)
 
-	if len(args) < 2 {
+	if len(args) < 1 {
 		fmt.Println("Incorrect number of arguments")
 		cli.ShowCommandHelpAndExit(ctx, "update", 1)
 		return
 	}
+
 	signer := NewGenericSigner(ctx)
-	manifestAddressOrDomain := args[0]
-	data, err := hexutil.Decode(args[1])
+
+	data, err := hexutil.Decode(args[0])
 	if err != nil {
 		utils.Fatalf("Error parsing data: %s", err.Error())
 		return
 	}
 
+	var updateRequest *mru.Request
+	var query *mru.Query
+
+	if manifestAddressOrDomain == "" {
+		query = new(mru.Query)
+		query.User = signer.Address()
+		query.Topic = getTopic(ctx)
+
+	}
+
 	// Retrieve resource status and metadata out of the manifest
-	updateRequest, err := client.GetResourceMetadata(manifestAddressOrDomain)
+	updateRequest, err = client.GetResourceMetadata(query, manifestAddressOrDomain)
 	if err != nil {
 		utils.Fatalf("Error retrieving resource status: %s", err.Error())
 	}
 
 	// set the new data
-	updateRequest.SetData(data, multihash)
+	updateRequest.SetData(data)
 
 	// sign update
 	if err = updateRequest.Sign(signer); err != nil {
@@ -146,17 +134,19 @@ func resourceUpdate(ctx *cli.Context) {
 
 func resourceInfo(ctx *cli.Context) {
 	var (
-		bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client = swarm.NewClient(bzzapi)
+		bzzapi                  = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		client                  = swarm.NewClient(bzzapi)
+		manifestAddressOrDomain = ctx.String(SwarmResourceManifestFlag.Name)
 	)
-	args := ctx.Args()
-	if len(args) < 1 {
-		fmt.Println("Incorrect number of arguments.")
-		cli.ShowCommandHelpAndExit(ctx, "info", 1)
-		return
+
+	var query *mru.Query
+	if manifestAddressOrDomain == "" {
+		query = new(mru.Query)
+		query.Topic = getTopic(ctx)
+		query.User = resourceGetUser(ctx)
 	}
-	manifestAddressOrDomain := args[0]
-	metadata, err := client.GetResourceMetadata(manifestAddressOrDomain)
+
+	metadata, err := client.GetResourceMetadata(query, manifestAddressOrDomain)
 	if err != nil {
 		utils.Fatalf("Error retrieving resource metadata: %s", err.Error())
 		return
@@ -167,3 +157,16 @@ func resourceInfo(ctx *cli.Context) {
 	}
 	fmt.Println(string(encodedMetadata))
 }
+
+func resourceGetUser(ctx *cli.Context) common.Address {
+	var user = ctx.String(SwarmResourceUserFlag.Name)
+	if user != "" {
+		return common.HexToAddress(user)
+	}
+	pk := getPrivKey(ctx)
+	if pk == nil {
+		utils.Fatalf("Cannot read private key. Must specify --user or --bzzaccount")
+	}
+	return crypto.PubkeyToAddress(pk.PublicKey)
+
+}
diff --git a/cmd/swarm/mru_test.go b/cmd/swarm/mru_test.go
new file mode 100644
index 000000000..142cf9cfd
--- /dev/null
+++ b/cmd/swarm/mru_test.go
@@ -0,0 +1,182 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/swarm/api"
+	"github.com/ethereum/go-ethereum/swarm/storage/mru/lookup"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
+
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/swarm/storage/mru"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/log"
+	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+	swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
+)
+
+func TestCLIResourceUpdate(t *testing.T) {
+
+	srv := testutil.NewTestSwarmServer(t, func(api *api.API) testutil.TestServer {
+		return swarmhttp.NewServer(api, "")
+	}, nil)
+	log.Info("starting 1 node cluster")
+	defer srv.Close()
+
+	// create a private key file for signing
+	pkfile, err := ioutil.TempFile("", "swarm-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer pkfile.Close()
+	defer os.Remove(pkfile.Name())
+
+	privkeyHex := "0000000000000000000000000000000000000000000000000000000000001979"
+	privKey, _ := crypto.HexToECDSA(privkeyHex)
+	address := crypto.PubkeyToAddress(privKey.PublicKey)
+
+	// save the private key to a file
+	_, err = io.WriteString(pkfile, privkeyHex)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// compose a topic. We'll be doing quotes about Miguel de Cervantes
+	var topic mru.Topic
+	subject := []byte("Miguel de Cervantes")
+	copy(topic[:], subject[:])
+	name := "quotes"
+
+	// prepare some data for the update
+	data := []byte("En boca cerrada no entran moscas")
+	hexData := hexutil.Encode(data)
+
+	flags := []string{
+		"--bzzapi", srv.URL,
+		"--bzzaccount", pkfile.Name(),
+		"resource", "update",
+		"--topic", topic.Hex(),
+		"--name", name,
+		hexData}
+
+	// create an update and expect an exit without errors
+	log.Info(fmt.Sprintf("updating a resource with 'swarm resource update'"))
+	cmd := runSwarm(t, flags...)
+	cmd.ExpectExit()
+
+	// now try to get the update using the client
+	client := swarm.NewClient(srv.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// build the same topic as before, this time
+	// we use NewTopic to create a topic automatically.
+	topic, err = mru.NewTopic(name, subject)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// View configures whose updates we will be looking up.
+	view := mru.View{
+		Topic: topic,
+		User:  address,
+	}
+
+	// Build a query to get the latest update
+	query := mru.NewQueryLatest(&view, lookup.NoClue)
+
+	// retrieve content!
+	reader, err := client.GetResource(query, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	retrieved, err := ioutil.ReadAll(reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// check we retrieved the sent information
+	if !bytes.Equal(data, retrieved) {
+		t.Fatalf("Received %s, expected %s", retrieved, data)
+	}
+
+	// Now retrieve info for the next update
+	flags = []string{
+		"--bzzapi", srv.URL,
+		"resource", "info",
+		"--topic", topic.Hex(),
+		"--user", address.Hex(),
+	}
+
+	log.Info(fmt.Sprintf("getting resource info with 'swarm resource info'"))
+	cmd = runSwarm(t, flags...)
+	_, matches := cmd.ExpectRegexp(`.*`) // regex hack to extract stdout
+	cmd.ExpectExit()
+
+	// verify we can deserialize the result as a valid JSON
+	var request mru.Request
+	err = json.Unmarshal([]byte(matches[0]), &request)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// make sure the retrieved view is the same
+	if request.View != view {
+		t.Fatalf("Expected view to be: %s, got %s", view, request.View)
+	}
+
+	// test publishing a manifest
+	flags = []string{
+		"--bzzapi", srv.URL,
+		"--bzzaccount", pkfile.Name(),
+		"resource", "create",
+		"--topic", topic.Hex(),
+	}
+
+	log.Info(fmt.Sprintf("Publishing manifest with 'swarm resource create'"))
+	cmd = runSwarm(t, flags...)
+	_, matches = cmd.ExpectRegexp(`[a-f\d]{64}`) // regex hack to extract stdout
+	cmd.ExpectExit()
+
+	manifestAddress := matches[0] // read the received resource manifest
+
+	// now attempt to lookup the latest update using a manifest instead
+	reader, err = client.GetResource(nil, manifestAddress)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	retrieved, err = ioutil.ReadAll(reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !bytes.Equal(data, retrieved) {
+		t.Fatalf("Received %s, expected %s", retrieved, data)
+	}
+}
diff --git a/cmd/swarm/swarm b/cmd/swarm/swarm
new file mode 100755
index 000000000..26952e479
--- /dev/null
+++ b/cmd/swarm/swarm