1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
|
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package network
import (
"bytes"
"encoding/binary"
"fmt"
"time"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/swarm/storage"
)
//metrics variables
var (
syncReceiveCount = metrics.NewRegisteredCounter("network.sync.recv.count", nil)
syncReceiveIgnore = metrics.NewRegisteredCounter("network.sync.recv.ignore", nil)
syncSendCount = metrics.NewRegisteredCounter("network.sync.send.count", nil)
syncSendRefused = metrics.NewRegisteredCounter("network.sync.send.refused", nil)
syncSendNotFound = metrics.NewRegisteredCounter("network.sync.send.notfound", nil)
)
// Handler for storage/retrieval related protocol requests
// implements the StorageHandler interface used by the bzz protocol
type Depo struct {
hashfunc storage.SwarmHasher
localStore storage.ChunkStore
netStore storage.ChunkStore
}
func NewDepo(hash storage.SwarmHasher, localStore, remoteStore storage.ChunkStore) *Depo {
return &Depo{
hashfunc: hash,
localStore: localStore,
netStore: remoteStore, // entrypoint internal
}
}
// Handles UnsyncedKeysMsg after msg decoding - unsynced hashes upto sync state
// * the remote sync state is just stored and handled in protocol
// * filters through the new syncRequests and send the ones missing
// * back immediately as a deliveryRequest message
// * empty message just pings back for more (is this needed?)
// * strict signed sync states may be needed.
func (self *Depo) HandleUnsyncedKeysMsg(req *unsyncedKeysMsgData, p *peer) error {
unsynced := req.Unsynced
var missing []*syncRequest
var chunk *storage.Chunk
var err error
for _, req := range unsynced {
// skip keys that are found,
chunk, err = self.localStore.Get(req.Key[:])
if err != nil || chunk.SData == nil {
missing = append(missing, req)
}
}
log.Debug(fmt.Sprintf("Depo.HandleUnsyncedKeysMsg: received %v unsynced keys: %v missing. new state: %v", len(unsynced), len(missing), req.State))
log.Trace(fmt.Sprintf("Depo.HandleUnsyncedKeysMsg: received %v", unsynced))
// send delivery request with missing keys
err = p.deliveryRequest(missing)
if err != nil {
return err
}
// set peers state to persist
p.syncState = req.State
return nil
}
// Handles deliveryRequestMsg
// * serves actual chunks asked by the remote peer
// by pushing to the delivery queue (sync db) of the correct priority
// (remote peer is free to reprioritize)
// * the message implies remote peer wants more, so trigger for
// * new outgoing unsynced keys message is fired
func (self *Depo) HandleDeliveryRequestMsg(req *deliveryRequestMsgData, p *peer) error {
deliver := req.Deliver
// queue the actual delivery of a chunk ()
log.Trace(fmt.Sprintf("Depo.HandleDeliveryRequestMsg: received %v delivery requests: %v", len(deliver), deliver))
for _, sreq := range deliver {
// TODO: look up in cache here or in deliveries
// priorities are taken from the message so the remote party can
// reprioritise to at their leisure
// r = self.pullCached(sreq.Key) // pulls and deletes from cache
Push(p, sreq.Key, sreq.Priority)
}
// sends it out as unsyncedKeysMsg
p.syncer.sendUnsyncedKeys()
return nil
}
// the entrypoint for store requests coming from the bzz wire protocol
// if key found locally, return. otherwise
// remote is untrusted, so hash is verified and chunk passed on to NetStore
func (self *Depo) HandleStoreRequestMsg(req *storeRequestMsgData, p *peer) {
var islocal bool
req.from = p
chunk, err := self.localStore.Get(req.Key)
switch {
case err != nil:
log.Trace(fmt.Sprintf("Depo.handleStoreRequest: %v not found locally. create new chunk/request", req.Key))
// not found in memory cache, ie., a genuine store request
// create chunk
syncReceiveCount.Inc(1)
chunk = storage.NewChunk(req.Key, nil)
case chunk.SData == nil:
// found chunk in memory store, needs the data, validate now
log.Trace(fmt.Sprintf("Depo.HandleStoreRequest: %v. request entry found", req))
default:
// data is found, store request ignored
// this should update access count?
syncReceiveIgnore.Inc(1)
log.Trace(fmt.Sprintf("Depo.HandleStoreRequest: %v found locally. ignore.", req))
islocal = true
//return
}
hasher := self.hashfunc()
hasher.Write(req.SData)
if !bytes.Equal(hasher.Sum(nil), req.Key) {
// data does not validate, ignore
// TODO: peer should be penalised/dropped?
log.Warn(fmt.Sprintf("Depo.HandleStoreRequest: chunk invalid. store request ignored: %v", req))
return
}
if islocal {
return
}
// update chunk with size and data
chunk.SData = req.SData // protocol validates that SData is minimum 9 bytes long (int64 size + at least one byte of data)
chunk.Size = int64(binary.LittleEndian.Uint64(req.SData[0:8]))
log.Trace(fmt.Sprintf("delivery of %v from %v", chunk, p))
chunk.Source = p
self.netStore.Put(chunk)
}
// entrypoint for retrieve requests coming from the bzz wire protocol
// checks swap balance - return if peer has no credit
func (self *Depo) HandleRetrieveRequestMsg(req *retrieveRequestMsgData, p *peer) {
req.from = p
// swap - record credit for 1 request
// note that only charge actual reqsearches
var err error
if p.swap != nil {
err = p.swap.Add(1)
}
if err != nil {
log.Warn(fmt.Sprintf("Depo.HandleRetrieveRequest: %v - cannot process request: %v", req.Key.Log(), err))
return
}
// call storage.NetStore#Get which
// blocks until local retrieval finished
// launches cloud retrieval
chunk, _ := self.netStore.Get(req.Key)
req = self.strategyUpdateRequest(chunk.Req, req)
// check if we can immediately deliver
if chunk.SData != nil {
log.Trace(fmt.Sprintf("Depo.HandleRetrieveRequest: %v - content found, delivering...", req.Key.Log()))
if req.MaxSize == 0 || int64(req.MaxSize) >= chunk.Size {
sreq := &storeRequestMsgData{
Id: req.Id,
Key: chunk.Key,
SData: chunk.SData,
requestTimeout: req.timeout, //
}
syncSendCount.Inc(1)
p.syncer.addRequest(sreq, DeliverReq)
} else {
syncSendRefused.Inc(1)
log.Trace(fmt.Sprintf("Depo.HandleRetrieveRequest: %v - content found, not wanted", req.Key.Log()))
}
} else {
syncSendNotFound.Inc(1)
log.Trace(fmt.Sprintf("Depo.HandleRetrieveRequest: %v - content not found locally. asked swarm for help. will get back", req.Key.Log()))
}
}
// add peer request the chunk and decides the timeout for the response if still searching
func (self *Depo) strategyUpdateRequest(rs *storage.RequestStatus, origReq *retrieveRequestMsgData) (req *retrieveRequestMsgData) {
log.Trace(fmt.Sprintf("Depo.strategyUpdateRequest: key %v", origReq.Key.Log()))
// we do not create an alternative one
req = origReq
if rs != nil {
self.addRequester(rs, req)
req.setTimeout(self.searchTimeout(rs, req))
}
return
}
// decides the timeout promise sent with the immediate peers response to a retrieve request
// if timeout is explicitly set and expired
func (self *Depo) searchTimeout(rs *storage.RequestStatus, req *retrieveRequestMsgData) (timeout *time.Time) {
reqt := req.getTimeout()
t := time.Now().Add(searchTimeout)
if reqt != nil && reqt.Before(t) {
return reqt
} else {
return &t
}
}
/*
adds a new peer to an existing open request
only add if less than requesterCount peers forwarded the same request id so far
note this is done irrespective of status (searching or found)
*/
func (self *Depo) addRequester(rs *storage.RequestStatus, req *retrieveRequestMsgData) {
log.Trace(fmt.Sprintf("Depo.addRequester: key %v - add peer to req.Id %v", req.Key.Log(), req.Id))
list := rs.Requesters[req.Id]
rs.Requesters[req.Id] = append(list, req)
}
|
