From 3ff2f756368f2d8ec0d1d9d25f6ba9cdabd7383e Mon Sep 17 00:00:00 2001 From: Balint Gabor Date: Thu, 13 Sep 2018 11:42:19 +0200 Subject: swarm: Chunk refactor (#17659) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Janos Guljas Co-authored-by: Balint Gabor Co-authored-by: Anton Evangelatov Co-authored-by: Viktor Trón --- swarm/api/manifest.go | 66 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 27 deletions(-) (limited to 'swarm/api/manifest.go') diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index a1329a800..d44ad2277 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -69,9 +69,12 @@ func (a *API) NewManifest(ctx context.Context, toEncrypt bool) (storage.Address, if err != nil { return nil, err } - key, wait, err := a.Store(ctx, bytes.NewReader(data), int64(len(data)), toEncrypt) - wait(ctx) - return key, err + addr, wait, err := a.Store(ctx, bytes.NewReader(data), int64(len(data)), toEncrypt) + if err != nil { + return nil, err + } + err = wait(ctx) + return addr, err } // Manifest hack for supporting Mutable Resource Updates from the bzz: scheme @@ -87,8 +90,12 @@ func (a *API) NewResourceManifest(ctx context.Context, resourceAddr string) (sto if err != nil { return nil, err } - key, _, err := a.Store(ctx, bytes.NewReader(data), int64(len(data)), false) - return key, err + addr, wait, err := a.Store(ctx, bytes.NewReader(data), int64(len(data)), false) + if err != nil { + return nil, err + } + err = wait(ctx) + return addr, err } // ManifestWriter is used to add and remove entries from an underlying manifest @@ -106,21 +113,26 @@ func (a *API) NewManifestWriter(ctx context.Context, addr storage.Address, quitC return &ManifestWriter{a, trie, quitC}, nil } -// AddEntry stores the given data and adds the resulting key to the manifest -func (m *ManifestWriter) AddEntry(ctx context.Context, data io.Reader, e *ManifestEntry) (key storage.Address, err error) { +// AddEntry stores the given data and adds the resulting address to the manifest +func (m *ManifestWriter) AddEntry(ctx context.Context, data io.Reader, e *ManifestEntry) (addr storage.Address, err error) { entry := newManifestTrieEntry(e, nil) if data != nil { - key, _, err = m.api.Store(ctx, data, e.Size, m.trie.encrypted) + var wait func(context.Context) error + addr, wait, err = m.api.Store(ctx, data, e.Size, m.trie.encrypted) + if err != nil { + return nil, err + } + err = wait(ctx) if err != nil { return nil, err } - entry.Hash = key.Hex() + entry.Hash = addr.Hex() } if entry.Hash == "" { - return key, errors.New("missing entry hash") + return addr, errors.New("missing entry hash") } m.trie.addEntry(entry, m.quitC) - return key, nil + return addr, nil } // RemoveEntry removes the given path from the manifest @@ -129,7 +141,7 @@ func (m *ManifestWriter) RemoveEntry(path string) error { return nil } -// Store stores the manifest, returning the resulting storage key +// Store stores the manifest, returning the resulting storage address func (m *ManifestWriter) Store() (storage.Address, error) { return m.trie.ref, m.trie.recalcAndStore() } @@ -211,51 +223,51 @@ type manifestTrieEntry struct { subtrie *manifestTrie } -func loadManifest(ctx context.Context, fileStore *storage.FileStore, hash storage.Address, quitC chan bool, decrypt DecryptFunc) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand - log.Trace("manifest lookup", "key", hash) +func loadManifest(ctx context.Context, fileStore *storage.FileStore, addr storage.Address, quitC chan bool, decrypt DecryptFunc) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand + log.Trace("manifest lookup", "addr", addr) // retrieve manifest via FileStore - manifestReader, isEncrypted := fileStore.Retrieve(ctx, hash) - log.Trace("reader retrieved", "key", hash) - return readManifest(manifestReader, hash, fileStore, isEncrypted, quitC, decrypt) + manifestReader, isEncrypted := fileStore.Retrieve(ctx, addr) + log.Trace("reader retrieved", "addr", addr) + return readManifest(manifestReader, addr, fileStore, isEncrypted, quitC, decrypt) } -func readManifest(mr storage.LazySectionReader, hash storage.Address, fileStore *storage.FileStore, isEncrypted bool, quitC chan bool, decrypt DecryptFunc) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand +func readManifest(mr storage.LazySectionReader, addr storage.Address, fileStore *storage.FileStore, isEncrypted bool, quitC chan bool, decrypt DecryptFunc) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand // TODO check size for oversized manifests size, err := mr.Size(mr.Context(), quitC) if err != nil { // size == 0 // can't determine size means we don't have the root chunk - log.Trace("manifest not found", "key", hash) + log.Trace("manifest not found", "addr", addr) err = fmt.Errorf("Manifest not Found") return } if size > manifestSizeLimit { - log.Warn("manifest exceeds size limit", "key", hash, "size", size, "limit", manifestSizeLimit) + log.Warn("manifest exceeds size limit", "addr", addr, "size", size, "limit", manifestSizeLimit) err = fmt.Errorf("Manifest size of %v bytes exceeds the %v byte limit", size, manifestSizeLimit) return } manifestData := make([]byte, size) read, err := mr.Read(manifestData) if int64(read) < size { - log.Trace("manifest not found", "key", hash) + log.Trace("manifest not found", "addr", addr) if err == nil { err = fmt.Errorf("Manifest retrieval cut short: read %v, expect %v", read, size) } return } - log.Debug("manifest retrieved", "key", hash) + log.Debug("manifest retrieved", "addr", addr) var man struct { Entries []*manifestTrieEntry `json:"entries"` } err = json.Unmarshal(manifestData, &man) if err != nil { - err = fmt.Errorf("Manifest %v is malformed: %v", hash.Log(), err) - log.Trace("malformed manifest", "key", hash) + err = fmt.Errorf("Manifest %v is malformed: %v", addr.Log(), err) + log.Trace("malformed manifest", "addr", addr) return } - log.Trace("manifest entries", "key", hash, "len", len(man.Entries)) + log.Trace("manifest entries", "addr", addr, "len", len(man.Entries)) trie = &manifestTrie{ fileStore: fileStore, @@ -406,12 +418,12 @@ func (mt *manifestTrie) recalcAndStore() error { sr := bytes.NewReader(manifest) ctx := context.TODO() - key, wait, err2 := mt.fileStore.Store(ctx, sr, int64(len(manifest)), mt.encrypted) + addr, wait, err2 := mt.fileStore.Store(ctx, sr, int64(len(manifest)), mt.encrypted) if err2 != nil { return err2 } err2 = wait(ctx) - mt.ref = key + mt.ref = addr return err2 } -- cgit v1.2.3