Swarm MRUs: Adaptive frequency / Predictable lookups / API simplification (#17559)

* swarm/storage/mru: Adaptive Frequency

swarm/storage/mru/lookup: fixed getBaseTime
Added NewEpoch constructor

swarm/api/client: better error handling in GetResource()


swarm/storage/mru: Renamed structures.
Renamed ResourceMetadata to ResourceID. 
Renamed ResourceID.Name to ResourceID.Topic

swarm/storage/mru: Added binarySerializer interface and test tools

swarm/storage/mru/lookup: Changed base time to time and + marshallers

swarm/storage/mru:  Added ResourceID (former resourceMetadata)

swarm/storage/mru: Added ResourceViewId and serialization tests

swarm/storage/mru/lookup: fixed epoch unmarshaller. Added Epoch Equals

swarm/storage/mru: Fixes as per review comments

cmd/swarm: reworded resource create/update help text regarding topic

swarm/storage/mru: Added UpdateLookup and serializer tests

swarm/storage/mru: Added UpdateHeader, serializers and tests

swarm/storage/mru: changed UpdateAddr / epoch to Base()

swarm/storage/mru: Added resourceUpdate serializer and tests

swarm/storage/mru: Added SignedResourceUpdate tests and serializers

swarm/storage/mru/lookup: fixed GetFirstEpoch bug

swarm/storage/mru: refactor, comments, cleanup

Also added tests for Topic
swarm/storage/mru: handler tests pass

swarm/storage/mru: all resource package tests pass

swarm/storage/mru: resource test pass after adding
timestamp checking support

swarm/storage/mru: Added JSON serializers to ResourceIDView structures

swarm/storage/mru: Sever, client, API test pass

swarm/storage/mru: server test pass

swarm/storage/mru: Added topic length check

swarm/storage/mru: removed some literals,
improved "previous lookup" test case

swarm/storage/mru: some fixes and comments as per review

swarm/storage/mru: first working version without metadata chunk

swarm/storage/mru: Various fixes as per review

swarm/storage/mru: client test pass

swarm/storage/mru: resource query strings and manifest-less queries


swarm/storage/mru: simplify naming

swarm/storage/mru: first autofreq working version



swarm/storage/mru: renamed ToValues to AppendValues

swarm/resource/mru: Added ToValues / FromValues for URL query strings

swarm/storage/mru: Changed POST resource to work with query strings.
No more JSON.

swarm/storage/mru: removed resourceid

swarm/storage/mru: Opened up structures

swarm/storage/mru: Merged Request and SignedResourceUpdate

swarm/storage/mru: removed initial data from CLI resource create

swarm/storage/mru: Refactor Topic as a direct fixed-length array

swarm/storage/mru/lookup: Comprehensive GetNextLevel tests

swarm/storage/mru: Added comments

Added length checks in Topic
swarm/storage/mru: fixes in tests and some code comments

swarm/storage/mru/lookup: new optimized lookup algorithm

swarm/api: moved getResourceView to api out of server

swarm/storage/mru: Lookup algorithm working

swarm/storage/mru: comments and renamed NewLookupParams

Deleted commented code


swarm/storage/mru/lookup: renamed Epoch.LaterThan to After

swarm/storage/mru/lookup: Comments and tidying naming



swarm/storage/mru: fix lookup algorithm

swarm/storage/mru: exposed lookup hint
removed updateheader

swarm/storage/mru/lookup: changed GetNextEpoch for initial values

swarm/storage/mru: resource tests pass

swarm/storage/mru: valueSerializer interface and tests



swarm/storage/mru/lookup: Comments, improvements, fixes, more tests

swarm/storage/mru: renamed UpdateLookup to ID, LookupParams to Query

swarm/storage/mru: renamed query receiver var



swarm/cmd: MRU CLI tests

* cmd/swarm: remove rogue fmt

* swarm/storage/mru: Add version / header for future use

* swarm/storage/mru: Fixes/comments as per review

cmd/swarm: remove rogue fmt

swarm/storage/mru: Add version / header for future use-

* swarm/storage/mru: fix linter errors

* cmd/swarm: Speeded up TestCLIResourceUpdate

1 files changed, 58 insertions, 74 deletions

diff --git a/swarm/storage/mru/update.go b/swarm/storage/mru/update.go
index d1bd37ddf..6aa57fce1 100644
--- a/swarm/storage/mru/update.go
+++ b/swarm/storage/mru/update.go
@@ -17,36 +17,35 @@
 package mru
 
 import (
-	"encoding/binary"
-	"errors"
+	"fmt"
+	"strconv"
 
 	"github.com/ethereum/go-ethereum/swarm/chunk"
-	"github.com/ethereum/go-ethereum/swarm/log"
-	"github.com/ethereum/go-ethereum/swarm/multihash"
 )
 
-// resourceUpdate encapsulates the information sent as part of a resource update
-type resourceUpdate struct {
-	updateHeader        // metainformationa about this resource update
-	data         []byte // actual data payload
+// ProtocolVersion defines the current version of the protocol that will be included in each update message
+const ProtocolVersion uint8 = 0
+
+const headerLength = 8
+
+// Header defines a update message header including a protocol version byte
+type Header struct {
+	Version uint8                   // Protocol version
+	Padding [headerLength - 1]uint8 // reserved for future use
 }
 
-// Update chunk layout
-// Prefix:
-// 2 bytes updateHeaderLength
-// 2 bytes data length
-const chunkPrefixLength = 2 + 2
+// ResourceUpdate encapsulates the information sent as part of a resource update
+type ResourceUpdate struct {
+	Header Header //
+	ID            // Resource update identifying information
+	data   []byte // actual data payload
+}
 
-// Header: (see updateHeader)
-// Data:
-// data (datalength bytes)
-//
-// Minimum size is Header + 1 (minimum data length, enforced)
-const minimumUpdateDataLength = updateHeaderLength + 1
-const maxUpdateDataLength = chunk.DefaultSize - signatureLength - updateHeaderLength - chunkPrefixLength
+const minimumUpdateDataLength = idLength + headerLength + 1
+const maxUpdateDataLength = chunk.DefaultSize - signatureLength - idLength - headerLength
 
 // binaryPut serializes the resource update information into the given slice
-func (r *resourceUpdate) binaryPut(serializedData []byte) error {
+func (r *ResourceUpdate) binaryPut(serializedData []byte) error {
 	datalength := len(r.data)
 	if datalength == 0 {
 		return NewError(ErrInvalidValue, "cannot update a resource with no data")
@@ -60,26 +59,17 @@ func (r *resourceUpdate) binaryPut(serializedData []byte) error {
 		return NewErrorf(ErrInvalidValue, "slice passed to putBinary must be of exact size. Expected %d bytes", r.binaryLength())
 	}
 
-	if r.multihash {
-		if _, _, err := multihash.GetMultihashLength(r.data); err != nil {
-			return NewError(ErrInvalidValue, "Invalid multihash")
-		}
-	}
-
-	// Add prefix: updateHeaderLength and actual data length
-	cursor := 0
-	binary.LittleEndian.PutUint16(serializedData[cursor:], uint16(updateHeaderLength))
-	cursor += 2
+	var cursor int
+	// serialize Header
+	serializedData[cursor] = r.Header.Version
+	copy(serializedData[cursor+1:headerLength], r.Header.Padding[:headerLength-1])
+	cursor += headerLength
 
-	// data length
-	binary.LittleEndian.PutUint16(serializedData[cursor:], uint16(datalength))
-	cursor += 2
-
-	// serialize header (see updateHeader)
-	if err := r.updateHeader.binaryPut(serializedData[cursor : cursor+updateHeaderLength]); err != nil {
+	// serialize ID
+	if err := r.ID.binaryPut(serializedData[cursor : cursor+idLength]); err != nil {
 		return err
 	}
-	cursor += updateHeaderLength
+	cursor += idLength
 
 	// add the data
 	copy(serializedData[cursor:], r.data)
@@ -89,60 +79,54 @@ func (r *resourceUpdate) binaryPut(serializedData []byte) error {
 }
 
 // binaryLength returns the expected number of bytes this structure will take to encode
-func (r *resourceUpdate) binaryLength() int {
-	return chunkPrefixLength + updateHeaderLength + len(r.data)
+func (r *ResourceUpdate) binaryLength() int {
+	return idLength + headerLength + len(r.data)
 }
 
 // binaryGet populates this instance from the information contained in the passed byte slice
-func (r *resourceUpdate) binaryGet(serializedData []byte) error {
+func (r *ResourceUpdate) binaryGet(serializedData []byte) error {
 	if len(serializedData) < minimumUpdateDataLength {
 		return NewErrorf(ErrNothingToReturn, "chunk less than %d bytes cannot be a resource update chunk", minimumUpdateDataLength)
 	}
-	cursor := 0
-	declaredHeaderlength := binary.LittleEndian.Uint16(serializedData[cursor : cursor+2])
-	if declaredHeaderlength != updateHeaderLength {
-		return NewErrorf(ErrCorruptData, "Invalid header length. Expected %d, got %d", updateHeaderLength, declaredHeaderlength)
-	}
+	dataLength := len(serializedData) - idLength - headerLength
+	// at this point we can be satisfied that we have the correct data length to read
 
-	cursor += 2
-	datalength := int(binary.LittleEndian.Uint16(serializedData[cursor : cursor+2]))
-	cursor += 2
+	var cursor int
 
-	if chunkPrefixLength+updateHeaderLength+datalength+signatureLength != len(serializedData) {
-		return NewError(ErrNothingToReturn, "length specified in header is different than actual chunk size")
-	}
+	// deserialize Header
+	r.Header.Version = serializedData[cursor]                                      // extract the protocol version
+	copy(r.Header.Padding[:headerLength-1], serializedData[cursor+1:headerLength]) // extract the padding
+	cursor += headerLength
 
-	// at this point we can be satisfied that we have the correct data length to read
-	if err := r.updateHeader.binaryGet(serializedData[cursor : cursor+updateHeaderLength]); err != nil {
+	if err := r.ID.binaryGet(serializedData[cursor : cursor+idLength]); err != nil {
 		return err
 	}
-	cursor += updateHeaderLength
-
-	data := serializedData[cursor : cursor+datalength]
-	cursor += datalength
+	cursor += idLength
 
-	// if multihash content is indicated we check the validity of the multihash
-	if r.updateHeader.multihash {
-		mhLength, mhHeaderLength, err := multihash.GetMultihashLength(data)
-		if err != nil {
-			log.Error("multihash parse error", "err", err)
-			return err
-		}
-		if datalength != mhLength+mhHeaderLength {
-			log.Debug("multihash error", "datalength", datalength, "mhLength", mhLength, "mhHeaderLength", mhHeaderLength)
-			return errors.New("Corrupt multihash data")
-		}
-	}
+	data := serializedData[cursor : cursor+dataLength]
+	cursor += dataLength
 
 	// now that all checks have passed, copy data into structure
-	r.data = make([]byte, datalength)
+	r.data = make([]byte, dataLength)
 	copy(r.data, data)
 
 	return nil
 
 }
 
-// Multihash specifies whether the resource data should be interpreted as multihash
-func (r *resourceUpdate) Multihash() bool {
-	return r.multihash
+// FromValues deserializes this instance from a string key-value store
+// useful to parse query strings
+func (r *ResourceUpdate) FromValues(values Values, data []byte) error {
+	r.data = data
+	version, _ := strconv.ParseUint(values.Get("protocolVersion"), 10, 32)
+	r.Header.Version = uint8(version)
+	return r.ID.FromValues(values)
+}
+
+// AppendValues serializes this structure into the provided string key-value store
+// useful to build query strings
+func (r *ResourceUpdate) AppendValues(values Values) []byte {
+	r.ID.AppendValues(values)
+	values.Set("protocolVersion", fmt.Sprintf("%d", r.Header.Version))
+	return r.data
 }