Swarm MRUs: Adaptive frequency / Predictable lookups / API simplification (#17559)

* swarm/storage/mru: Adaptive Frequency

swarm/storage/mru/lookup: fixed getBaseTime
Added NewEpoch constructor

swarm/api/client: better error handling in GetResource()


swarm/storage/mru: Renamed structures.
Renamed ResourceMetadata to ResourceID. 
Renamed ResourceID.Name to ResourceID.Topic

swarm/storage/mru: Added binarySerializer interface and test tools

swarm/storage/mru/lookup: Changed base time to time and + marshallers

swarm/storage/mru:  Added ResourceID (former resourceMetadata)

swarm/storage/mru: Added ResourceViewId and serialization tests

swarm/storage/mru/lookup: fixed epoch unmarshaller. Added Epoch Equals

swarm/storage/mru: Fixes as per review comments

cmd/swarm: reworded resource create/update help text regarding topic

swarm/storage/mru: Added UpdateLookup and serializer tests

swarm/storage/mru: Added UpdateHeader, serializers and tests

swarm/storage/mru: changed UpdateAddr / epoch to Base()

swarm/storage/mru: Added resourceUpdate serializer and tests

swarm/storage/mru: Added SignedResourceUpdate tests and serializers

swarm/storage/mru/lookup: fixed GetFirstEpoch bug

swarm/storage/mru: refactor, comments, cleanup

Also added tests for Topic
swarm/storage/mru: handler tests pass

swarm/storage/mru: all resource package tests pass

swarm/storage/mru: resource test pass after adding
timestamp checking support

swarm/storage/mru: Added JSON serializers to ResourceIDView structures

swarm/storage/mru: Sever, client, API test pass

swarm/storage/mru: server test pass

swarm/storage/mru: Added topic length check

swarm/storage/mru: removed some literals,
improved "previous lookup" test case

swarm/storage/mru: some fixes and comments as per review

swarm/storage/mru: first working version without metadata chunk

swarm/storage/mru: Various fixes as per review

swarm/storage/mru: client test pass

swarm/storage/mru: resource query strings and manifest-less queries


swarm/storage/mru: simplify naming

swarm/storage/mru: first autofreq working version



swarm/storage/mru: renamed ToValues to AppendValues

swarm/resource/mru: Added ToValues / FromValues for URL query strings

swarm/storage/mru: Changed POST resource to work with query strings.
No more JSON.

swarm/storage/mru: removed resourceid

swarm/storage/mru: Opened up structures

swarm/storage/mru: Merged Request and SignedResourceUpdate

swarm/storage/mru: removed initial data from CLI resource create

swarm/storage/mru: Refactor Topic as a direct fixed-length array

swarm/storage/mru/lookup: Comprehensive GetNextLevel tests

swarm/storage/mru: Added comments

Added length checks in Topic
swarm/storage/mru: fixes in tests and some code comments

swarm/storage/mru/lookup: new optimized lookup algorithm

swarm/api: moved getResourceView to api out of server

swarm/storage/mru: Lookup algorithm working

swarm/storage/mru: comments and renamed NewLookupParams

Deleted commented code


swarm/storage/mru/lookup: renamed Epoch.LaterThan to After

swarm/storage/mru/lookup: Comments and tidying naming



swarm/storage/mru: fix lookup algorithm

swarm/storage/mru: exposed lookup hint
removed updateheader

swarm/storage/mru/lookup: changed GetNextEpoch for initial values

swarm/storage/mru: resource tests pass

swarm/storage/mru: valueSerializer interface and tests



swarm/storage/mru/lookup: Comments, improvements, fixes, more tests

swarm/storage/mru: renamed UpdateLookup to ID, LookupParams to Query

swarm/storage/mru: renamed query receiver var



swarm/cmd: MRU CLI tests

* cmd/swarm: remove rogue fmt

* swarm/storage/mru: Add version / header for future use

* swarm/storage/mru: Fixes/comments as per review

cmd/swarm: remove rogue fmt

swarm/storage/mru: Add version / header for future use-

* swarm/storage/mru: fix linter errors

* cmd/swarm: Speeded up TestCLIResourceUpdate

1 files changed, 75 insertions, 84 deletions

diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go
index 4a3ca0429..8ba4e55c3 100644
--- a/swarm/api/http/server_test.go
+++ b/swarm/api/http/server_test.go
@@ -30,12 +30,15 @@ import (
 	"math/big"
 	"mime/multipart"
 	"net/http"
+	"net/url"
 	"os"
 	"strconv"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/ethereum/go-ethereum/swarm/storage/mru/lookup"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
@@ -121,8 +124,8 @@ func TestBzzResourceMultihash(t *testing.T) {
 
 	// add the data our multihash aliased manifest will point to
 	databytes := "bar"
-	url := fmt.Sprintf("%s/bzz:/", srv.URL)
-	resp, err := http.Post(url, "text/plain", bytes.NewReader([]byte(databytes)))
+	testBzzUrl := fmt.Sprintf("%s/bzz:/", srv.URL)
+	resp, err := http.Post(testBzzUrl, "text/plain", bytes.NewReader([]byte(databytes)))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -140,33 +143,27 @@ func TestBzzResourceMultihash(t *testing.T) {
 
 	log.Info("added data", "manifest", string(b), "data", common.ToHex(mh))
 
-	// our mutable resource "name"
-	keybytes := "foo.eth"
+	topic, _ := mru.NewTopic("foo.eth", nil)
+	updateRequest := mru.NewFirstRequest(topic)
 
-	updateRequest, err := mru.NewCreateUpdateRequest(&mru.ResourceMetadata{
-		Name:      keybytes,
-		Frequency: 13,
-		StartTime: srv.GetCurrentTime(),
-		Owner:     signer.Address(),
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-	updateRequest.SetData(mh, true)
+	updateRequest.SetData(mh)
 
 	if err := updateRequest.Sign(signer); err != nil {
 		t.Fatal(err)
 	}
 	log.Info("added data", "manifest", string(b), "data", common.ToHex(mh))
 
-	body, err := updateRequest.MarshalJSON()
+	testUrl, err := url.Parse(fmt.Sprintf("%s/bzz-resource:/", srv.URL))
 	if err != nil {
 		t.Fatal(err)
 	}
+	query := testUrl.Query()
+	body := updateRequest.AppendValues(query) // this adds all query parameters and returns the data to be posted
+	query.Set("manifest", "1")                // indicate we want a manifest back
+	testUrl.RawQuery = query.Encode()
 
 	// create the multihash update
-	url = fmt.Sprintf("%s/bzz-resource:/", srv.URL)
-	resp, err = http.Post(url, "application/json", bytes.NewReader(body))
+	resp, err = http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -184,14 +181,14 @@ func TestBzzResourceMultihash(t *testing.T) {
 		t.Fatalf("data %s could not be unmarshaled: %v", b, err)
 	}
 
-	correctManifestAddrHex := "6d3bc4664c97d8b821cb74bcae43f592494fb46d2d9cd31e69f3c7c802bbbd8e"
+	correctManifestAddrHex := "6ef40ba1492cf2a029dc9a8b5896c822cf689d3cd010842f4f1744e6db8824bd"
 	if rsrcResp.Hex() != correctManifestAddrHex {
 		t.Fatalf("Response resource key mismatch, expected '%s', got '%s'", correctManifestAddrHex, rsrcResp.Hex())
 	}
 
 	// get bzz manifest transparent resource resolve
-	url = fmt.Sprintf("%s/bzz:/%s", srv.URL, rsrcResp)
-	resp, err = http.Get(url)
+	testBzzUrl = fmt.Sprintf("%s/bzz:/%s", srv.URL, rsrcResp)
+	resp, err = http.Get(testBzzUrl)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -215,39 +212,38 @@ func TestBzzResource(t *testing.T) {
 
 	defer srv.Close()
 
-	// our mutable resource "name"
-	keybytes := "foo.eth"
-
 	// data of update 1
-	databytes := make([]byte, 666)
-	_, err := rand.Read(databytes)
+	update1Data := make([]byte, 666)
+	update1Timestamp := srv.CurrentTime
+	_, err := rand.Read(update1Data)
 	if err != nil {
 		t.Fatal(err)
 	}
+	//data for update 2
+	update2Data := []byte("foo")
 
-	updateRequest, err := mru.NewCreateUpdateRequest(&mru.ResourceMetadata{
-		Name:      keybytes,
-		Frequency: 13,
-		StartTime: srv.GetCurrentTime(),
-		Owner:     signer.Address(),
-	})
+	topic, _ := mru.NewTopic("foo.eth", nil)
+	updateRequest := mru.NewFirstRequest(topic)
 	if err != nil {
 		t.Fatal(err)
 	}
-	updateRequest.SetData(databytes, false)
+	updateRequest.SetData(update1Data)
 
 	if err := updateRequest.Sign(signer); err != nil {
 		t.Fatal(err)
 	}
 
-	body, err := updateRequest.MarshalJSON()
+	// creates resource and sets update 1
+	testUrl, err := url.Parse(fmt.Sprintf("%s/bzz-resource:/", srv.URL))
 	if err != nil {
 		t.Fatal(err)
 	}
+	urlQuery := testUrl.Query()
+	body := updateRequest.AppendValues(urlQuery) // this adds all query parameters
+	urlQuery.Set("manifest", "1")                // indicate we want a manifest back
+	testUrl.RawQuery = urlQuery.Encode()
 
-	// creates resource and sets update 1
-	url := fmt.Sprintf("%s/bzz-resource:/", srv.URL)
-	resp, err := http.Post(url, "application/json", bytes.NewReader(body))
+	resp, err := http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -265,14 +261,14 @@ func TestBzzResource(t *testing.T) {
 		t.Fatalf("data %s could not be unmarshaled: %v", b, err)
 	}
 
-	correctManifestAddrHex := "6d3bc4664c97d8b821cb74bcae43f592494fb46d2d9cd31e69f3c7c802bbbd8e"
+	correctManifestAddrHex := "6ef40ba1492cf2a029dc9a8b5896c822cf689d3cd010842f4f1744e6db8824bd"
 	if rsrcResp.Hex() != correctManifestAddrHex {
-		t.Fatalf("Response resource key mismatch, expected '%s', got '%s'", correctManifestAddrHex, rsrcResp.Hex())
+		t.Fatalf("Response resource manifest mismatch, expected '%s', got '%s'", correctManifestAddrHex, rsrcResp.Hex())
 	}
 
 	// get the manifest
-	url = fmt.Sprintf("%s/bzz-raw:/%s", srv.URL, rsrcResp)
-	resp, err = http.Get(url)
+	testRawUrl := fmt.Sprintf("%s/bzz-raw:/%s", srv.URL, rsrcResp)
+	resp, err = http.Get(testRawUrl)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -292,20 +288,20 @@ func TestBzzResource(t *testing.T) {
 	if len(manifest.Entries) != 1 {
 		t.Fatalf("Manifest has %d entries", len(manifest.Entries))
 	}
-	correctRootKeyHex := "68f7ba07ac8867a4c841a4d4320e3cdc549df23702dc7285fcb6acf65df48562"
-	if manifest.Entries[0].Hash != correctRootKeyHex {
-		t.Fatalf("Expected manifest path '%s', got '%s'", correctRootKeyHex, manifest.Entries[0].Hash)
+	correctViewHex := "0x666f6f2e65746800000000000000000000000000000000000000000000000000c96aaa54e2d44c299564da76e1cd3184a2386b8d"
+	if manifest.Entries[0].ResourceView.Hex() != correctViewHex {
+		t.Fatalf("Expected manifest Resource View '%s', got '%s'", correctViewHex, manifest.Entries[0].ResourceView.Hex())
 	}
 
 	// get bzz manifest transparent resource resolve
-	url = fmt.Sprintf("%s/bzz:/%s", srv.URL, rsrcResp)
-	resp, err = http.Get(url)
+	testBzzUrl := fmt.Sprintf("%s/bzz:/%s", srv.URL, rsrcResp)
+	resp, err = http.Get(testBzzUrl)
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusOK {
-		t.Fatalf("err %s", resp.Status)
+	if resp.StatusCode == http.StatusOK {
+		t.Fatal("Expected error status since resource is not multihash. Received 200 OK")
 	}
 	b, err = ioutil.ReadAll(resp.Body)
 	if err != nil {
@@ -313,8 +309,8 @@ func TestBzzResource(t *testing.T) {
 	}
 
 	// get non-existent name, should fail
-	url = fmt.Sprintf("%s/bzz-resource:/bar", srv.URL)
-	resp, err = http.Get(url)
+	testBzzResUrl := fmt.Sprintf("%s/bzz-resource:/bar", srv.URL)
+	resp, err = http.Get(testBzzResUrl)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -327,8 +323,8 @@ func TestBzzResource(t *testing.T) {
 
 	// get latest update (1.1) through resource directly
 	log.Info("get update latest = 1.1", "addr", correctManifestAddrHex)
-	url = fmt.Sprintf("%s/bzz-resource:/%s", srv.URL, correctManifestAddrHex)
-	resp, err = http.Get(url)
+	testBzzResUrl = fmt.Sprintf("%s/bzz-resource:/%s", srv.URL, correctManifestAddrHex)
+	resp, err = http.Get(testBzzResUrl)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -340,16 +336,18 @@ func TestBzzResource(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !bytes.Equal(databytes, b) {
-		t.Fatalf("Expected body '%x', got '%x'", databytes, b)
+	if !bytes.Equal(update1Data, b) {
+		t.Fatalf("Expected body '%x', got '%x'", update1Data, b)
 	}
 
 	// update 2
+	// Move the clock ahead 1 second
+	srv.CurrentTime++
 	log.Info("update 2")
 
 	// 1.- get metadata about this resource
-	url = fmt.Sprintf("%s/bzz-resource:/%s/", srv.URL, correctManifestAddrHex)
-	resp, err = http.Get(url + "meta")
+	testBzzResUrl = fmt.Sprintf("%s/bzz-resource:/%s/", srv.URL, correctManifestAddrHex)
+	resp, err = http.Get(testBzzResUrl + "?meta=1")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -365,17 +363,19 @@ func TestBzzResource(t *testing.T) {
 	if err = updateRequest.UnmarshalJSON(b); err != nil {
 		t.Fatalf("Error decoding resource metadata: %s", err)
 	}
-	data := []byte("foo")
-	updateRequest.SetData(data, false)
+	updateRequest.SetData(update2Data)
 	if err = updateRequest.Sign(signer); err != nil {
 		t.Fatal(err)
 	}
-	body, err = updateRequest.MarshalJSON()
+	testUrl, err = url.Parse(fmt.Sprintf("%s/bzz-resource:/", srv.URL))
 	if err != nil {
 		t.Fatal(err)
 	}
+	urlQuery = testUrl.Query()
+	body = updateRequest.AppendValues(urlQuery) // this adds all query parameters
+	testUrl.RawQuery = urlQuery.Encode()
 
-	resp, err = http.Post(url, "application/json", bytes.NewReader(body))
+	resp, err = http.Post(testUrl.String(), "application/octet-stream", bytes.NewReader(body))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -386,8 +386,8 @@ func TestBzzResource(t *testing.T) {
 
 	// get latest update (1.2) through resource directly
 	log.Info("get update 1.2")
-	url = fmt.Sprintf("%s/bzz-resource:/%s", srv.URL, correctManifestAddrHex)
-	resp, err = http.Get(url)
+	testBzzResUrl = fmt.Sprintf("%s/bzz-resource:/%s", srv.URL, correctManifestAddrHex)
+	resp, err = http.Get(testBzzResUrl)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -399,33 +399,23 @@ func TestBzzResource(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !bytes.Equal(data, b) {
-		t.Fatalf("Expected body '%x', got '%x'", data, b)
+	if !bytes.Equal(update2Data, b) {
+		t.Fatalf("Expected body '%x', got '%x'", update2Data, b)
 	}
 
-	// get latest update (1.2) with specified period
-	log.Info("get update latest = 1.2")
-	url = fmt.Sprintf("%s/bzz-resource:/%s/1", srv.URL, correctManifestAddrHex)
-	resp, err = http.Get(url)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusOK {
-		t.Fatalf("err %s", resp.Status)
-	}
-	b, err = ioutil.ReadAll(resp.Body)
+	// test manifest-less queries
+	log.Info("get first update in update1Timestamp via direct query")
+	query := mru.NewQuery(&updateRequest.View, update1Timestamp, lookup.NoClue)
+
+	urlq, err := url.Parse(fmt.Sprintf("%s/bzz-resource:/", srv.URL))
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !bytes.Equal(data, b) {
-		t.Fatalf("Expected body '%x', got '%x'", data, b)
-	}
 
-	// get first update (1.1) with specified period and version
-	log.Info("get first update 1.1")
-	url = fmt.Sprintf("%s/bzz-resource:/%s/1/1", srv.URL, correctManifestAddrHex)
-	resp, err = http.Get(url)
+	values := urlq.Query()
+	query.AppendValues(values) // this adds view query parameters
+	urlq.RawQuery = values.Encode()
+	resp, err = http.Get(urlq.String())
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -437,9 +427,10 @@ func TestBzzResource(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !bytes.Equal(databytes, b) {
-		t.Fatalf("Expected body '%x', got '%x'", databytes, b)
+	if !bytes.Equal(update1Data, b) {
+		t.Fatalf("Expected body '%x', got '%x'", update1Data, b)
 	}
+
 }
 
 func TestBzzGetPath(t *testing.T) {