aboutsummaryrefslogtreecommitdiffstats
path: root/swarm/api/manifest.go
diff options
context:
space:
mode:
Diffstat (limited to 'swarm/api/manifest.go')
-rw-r--r--swarm/api/manifest.go584
1 files changed, 0 insertions, 584 deletions
diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go
deleted file mode 100644
index d753b3f2e..000000000
--- a/swarm/api/manifest.go
+++ /dev/null
@@ -1,584 +0,0 @@
-// Copyright 2016 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package api
-
-import (
- "bytes"
- "context"
- "encoding/json"
- "errors"
- "fmt"
- "io"
- "net/http"
- "strings"
- "time"
-
- "github.com/ethereum/go-ethereum/swarm/storage/feed"
-
- "github.com/ethereum/go-ethereum/common"
- "github.com/ethereum/go-ethereum/swarm/log"
- "github.com/ethereum/go-ethereum/swarm/storage"
-)
-
-const (
- ManifestType = "application/bzz-manifest+json"
- FeedContentType = "application/bzz-feed"
-
- manifestSizeLimit = 5 * 1024 * 1024
-)
-
-// Manifest represents a swarm manifest
-type Manifest struct {
- Entries []ManifestEntry `json:"entries,omitempty"`
-}
-
-// ManifestEntry represents an entry in a swarm manifest
-type ManifestEntry struct {
- Hash string `json:"hash,omitempty"`
- Path string `json:"path,omitempty"`
- ContentType string `json:"contentType,omitempty"`
- Mode int64 `json:"mode,omitempty"`
- Size int64 `json:"size,omitempty"`
- ModTime time.Time `json:"mod_time,omitempty"`
- Status int `json:"status,omitempty"`
- Access *AccessEntry `json:"access,omitempty"`
- Feed *feed.Feed `json:"feed,omitempty"`
-}
-
-// ManifestList represents the result of listing files in a manifest
-type ManifestList struct {
- CommonPrefixes []string `json:"common_prefixes,omitempty"`
- Entries []*ManifestEntry `json:"entries,omitempty"`
-}
-
-// NewManifest creates and stores a new, empty manifest
-func (a *API) NewManifest(ctx context.Context, toEncrypt bool) (storage.Address, error) {
- var manifest Manifest
- data, err := json.Marshal(&manifest)
- if err != nil {
- return nil, err
- }
- addr, wait, err := a.Store(ctx, bytes.NewReader(data), int64(len(data)), toEncrypt)
- if err != nil {
- return nil, err
- }
- err = wait(ctx)
- return addr, err
-}
-
-// Manifest hack for supporting Swarm feeds from the bzz: scheme
-// see swarm/api/api.go:API.Get() for more information
-func (a *API) NewFeedManifest(ctx context.Context, feed *feed.Feed) (storage.Address, error) {
- var manifest Manifest
- entry := ManifestEntry{
- Feed: feed,
- ContentType: FeedContentType,
- }
- manifest.Entries = append(manifest.Entries, entry)
- data, err := json.Marshal(&manifest)
- if err != nil {
- return nil, err
- }
- addr, wait, err := a.Store(ctx, bytes.NewReader(data), int64(len(data)), false)
- if err != nil {
- return nil, err
- }
- err = wait(ctx)
- return addr, err
-}
-
-// ManifestWriter is used to add and remove entries from an underlying manifest
-type ManifestWriter struct {
- api *API
- trie *manifestTrie
- quitC chan bool
-}
-
-func (a *API) NewManifestWriter(ctx context.Context, addr storage.Address, quitC chan bool) (*ManifestWriter, error) {
- trie, err := loadManifest(ctx, a.fileStore, addr, quitC, NOOPDecrypt)
- if err != nil {
- return nil, fmt.Errorf("error loading manifest %s: %s", addr, err)
- }
- return &ManifestWriter{a, trie, quitC}, nil
-}
-
-// AddEntry stores the given data and adds the resulting address to the manifest
-func (m *ManifestWriter) AddEntry(ctx context.Context, data io.Reader, e *ManifestEntry) (addr storage.Address, err error) {
- entry := newManifestTrieEntry(e, nil)
- if data != nil {
- var wait func(context.Context) error
- addr, wait, err = m.api.Store(ctx, data, e.Size, m.trie.encrypted)
- if err != nil {
- return nil, err
- }
- err = wait(ctx)
- if err != nil {
- return nil, err
- }
- entry.Hash = addr.Hex()
- }
- if entry.Hash == "" {
- return addr, errors.New("missing entry hash")
- }
- m.trie.addEntry(entry, m.quitC)
- return addr, nil
-}
-
-// RemoveEntry removes the given path from the manifest
-func (m *ManifestWriter) RemoveEntry(path string) error {
- m.trie.deleteEntry(path, m.quitC)
- return nil
-}
-
-// Store stores the manifest, returning the resulting storage address
-func (m *ManifestWriter) Store() (storage.Address, error) {
- return m.trie.ref, m.trie.recalcAndStore()
-}
-
-// ManifestWalker is used to recursively walk the entries in the manifest and
-// all of its submanifests
-type ManifestWalker struct {
- api *API
- trie *manifestTrie
- quitC chan bool
-}
-
-func (a *API) NewManifestWalker(ctx context.Context, addr storage.Address, decrypt DecryptFunc, quitC chan bool) (*ManifestWalker, error) {
- trie, err := loadManifest(ctx, a.fileStore, addr, quitC, decrypt)
- if err != nil {
- return nil, fmt.Errorf("error loading manifest %s: %s", addr, err)
- }
- return &ManifestWalker{a, trie, quitC}, nil
-}
-
-// ErrSkipManifest is used as a return value from WalkFn to indicate that the
-// manifest should be skipped
-var ErrSkipManifest = errors.New("skip this manifest")
-
-// WalkFn is the type of function called for each entry visited by a recursive
-// manifest walk
-type WalkFn func(entry *ManifestEntry) error
-
-// Walk recursively walks the manifest calling walkFn for each entry in the
-// manifest, including submanifests
-func (m *ManifestWalker) Walk(walkFn WalkFn) error {
- return m.walk(m.trie, "", walkFn)
-}
-
-func (m *ManifestWalker) walk(trie *manifestTrie, prefix string, walkFn WalkFn) error {
- for _, entry := range &trie.entries {
- if entry == nil {
- continue
- }
- entry.Path = prefix + entry.Path
- err := walkFn(&entry.ManifestEntry)
- if err != nil {
- if entry.ContentType == ManifestType && err == ErrSkipManifest {
- continue
- }
- return err
- }
- if entry.ContentType != ManifestType {
- continue
- }
- if err := trie.loadSubTrie(entry, nil); err != nil {
- return err
- }
- if err := m.walk(entry.subtrie, entry.Path, walkFn); err != nil {
- return err
- }
- }
- return nil
-}
-
-type manifestTrie struct {
- fileStore *storage.FileStore
- entries [257]*manifestTrieEntry // indexed by first character of basePath, entries[256] is the empty basePath entry
- ref storage.Address // if ref != nil, it is stored
- encrypted bool
- decrypt DecryptFunc
-}
-
-func newManifestTrieEntry(entry *ManifestEntry, subtrie *manifestTrie) *manifestTrieEntry {
- return &manifestTrieEntry{
- ManifestEntry: *entry,
- subtrie: subtrie,
- }
-}
-
-type manifestTrieEntry struct {
- ManifestEntry
-
- subtrie *manifestTrie
-}
-
-func loadManifest(ctx context.Context, fileStore *storage.FileStore, addr storage.Address, quitC chan bool, decrypt DecryptFunc) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
- log.Trace("manifest lookup", "addr", addr)
- // retrieve manifest via FileStore
- manifestReader, isEncrypted := fileStore.Retrieve(ctx, addr)
- log.Trace("reader retrieved", "addr", addr)
- return readManifest(manifestReader, addr, fileStore, isEncrypted, quitC, decrypt)
-}
-
-func readManifest(mr storage.LazySectionReader, addr storage.Address, fileStore *storage.FileStore, isEncrypted bool, quitC chan bool, decrypt DecryptFunc) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
- // TODO check size for oversized manifests
- size, err := mr.Size(mr.Context(), quitC)
- if err != nil { // size == 0
- // can't determine size means we don't have the root chunk
- log.Trace("manifest not found", "addr", addr)
- err = fmt.Errorf("Manifest not Found")
- return
- }
- if size > manifestSizeLimit {
- log.Warn("manifest exceeds size limit", "addr", addr, "size", size, "limit", manifestSizeLimit)
- err = fmt.Errorf("Manifest size of %v bytes exceeds the %v byte limit", size, manifestSizeLimit)
- return
- }
- manifestData := make([]byte, size)
- read, err := mr.Read(manifestData)
- if int64(read) < size {
- log.Trace("manifest not found", "addr", addr)
- if err == nil {
- err = fmt.Errorf("Manifest retrieval cut short: read %v, expect %v", read, size)
- }
- return
- }
-
- log.Debug("manifest retrieved", "addr", addr)
- var man struct {
- Entries []*manifestTrieEntry `json:"entries"`
- }
- err = json.Unmarshal(manifestData, &man)
- if err != nil {
- err = fmt.Errorf("Manifest %v is malformed: %v", addr.Log(), err)
- log.Trace("malformed manifest", "addr", addr)
- return
- }
-
- log.Trace("manifest entries", "addr", addr, "len", len(man.Entries))
-
- trie = &manifestTrie{
- fileStore: fileStore,
- encrypted: isEncrypted,
- decrypt: decrypt,
- }
- for _, entry := range man.Entries {
- err = trie.addEntry(entry, quitC)
- if err != nil {
- return
- }
- }
- return
-}
-
-func (mt *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) error {
- mt.ref = nil // trie modified, hash needs to be re-calculated on demand
-
- if entry.ManifestEntry.Access != nil {
- if mt.decrypt == nil {
- return errors.New("dont have decryptor")
- }
-
- err := mt.decrypt(&entry.ManifestEntry)
- if err != nil {
- return err
- }
- }
-
- if len(entry.Path) == 0 {
- mt.entries[256] = entry
- return nil
- }
-
- b := entry.Path[0]
- oldentry := mt.entries[b]
- if (oldentry == nil) || (oldentry.Path == entry.Path && oldentry.ContentType != ManifestType) {
- mt.entries[b] = entry
- return nil
- }
-
- cpl := 0
- for (len(entry.Path) > cpl) && (len(oldentry.Path) > cpl) && (entry.Path[cpl] == oldentry.Path[cpl]) {
- cpl++
- }
-
- if (oldentry.ContentType == ManifestType) && (cpl == len(oldentry.Path)) {
- if mt.loadSubTrie(oldentry, quitC) != nil {
- return nil
- }
- entry.Path = entry.Path[cpl:]
- oldentry.subtrie.addEntry(entry, quitC)
- oldentry.Hash = ""
- return nil
- }
-
- commonPrefix := entry.Path[:cpl]
-
- subtrie := &manifestTrie{
- fileStore: mt.fileStore,
- encrypted: mt.encrypted,
- }
- entry.Path = entry.Path[cpl:]
- oldentry.Path = oldentry.Path[cpl:]
- subtrie.addEntry(entry, quitC)
- subtrie.addEntry(oldentry, quitC)
-
- mt.entries[b] = newManifestTrieEntry(&ManifestEntry{
- Path: commonPrefix,
- ContentType: ManifestType,
- }, subtrie)
- return nil
-}
-
-func (mt *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) {
- for _, e := range &mt.entries {
- if e != nil {
- cnt++
- entry = e
- }
- }
- return
-}
-
-func (mt *manifestTrie) deleteEntry(path string, quitC chan bool) {
- mt.ref = nil // trie modified, hash needs to be re-calculated on demand
-
- if len(path) == 0 {
- mt.entries[256] = nil
- return
- }
-
- b := path[0]
- entry := mt.entries[b]
- if entry == nil {
- return
- }
- if entry.Path == path {
- mt.entries[b] = nil
- return
- }
-
- epl := len(entry.Path)
- if (entry.ContentType == ManifestType) && (len(path) >= epl) && (path[:epl] == entry.Path) {
- if mt.loadSubTrie(entry, quitC) != nil {
- return
- }
- entry.subtrie.deleteEntry(path[epl:], quitC)
- entry.Hash = ""
- // remove subtree if it has less than 2 elements
- cnt, lastentry := entry.subtrie.getCountLast()
- if cnt < 2 {
- if lastentry != nil {
- lastentry.Path = entry.Path + lastentry.Path
- }
- mt.entries[b] = lastentry
- }
- }
-}
-
-func (mt *manifestTrie) recalcAndStore() error {
- if mt.ref != nil {
- return nil
- }
-
- var buffer bytes.Buffer
- buffer.WriteString(`{"entries":[`)
-
- list := &Manifest{}
- for _, entry := range &mt.entries {
- if entry != nil {
- if entry.Hash == "" { // TODO: paralellize
- err := entry.subtrie.recalcAndStore()
- if err != nil {
- return err
- }
- entry.Hash = entry.subtrie.ref.Hex()
- }
- list.Entries = append(list.Entries, entry.ManifestEntry)
- }
-
- }
-
- manifest, err := json.Marshal(list)
- if err != nil {
- return err
- }
-
- sr := bytes.NewReader(manifest)
- ctx := context.TODO()
- addr, wait, err2 := mt.fileStore.Store(ctx, sr, int64(len(manifest)), mt.encrypted)
- if err2 != nil {
- return err2
- }
- err2 = wait(ctx)
- mt.ref = addr
- return err2
-}
-
-func (mt *manifestTrie) loadSubTrie(entry *manifestTrieEntry, quitC chan bool) (err error) {
- if entry.ManifestEntry.Access != nil {
- if mt.decrypt == nil {
- return errors.New("dont have decryptor")
- }
-
- err := mt.decrypt(&entry.ManifestEntry)
- if err != nil {
- return err
- }
- }
-
- if entry.subtrie == nil {
- hash := common.Hex2Bytes(entry.Hash)
- entry.subtrie, err = loadManifest(context.TODO(), mt.fileStore, hash, quitC, mt.decrypt)
- entry.Hash = "" // might not match, should be recalculated
- }
- return
-}
-
-func (mt *manifestTrie) listWithPrefixInt(prefix, rp string, quitC chan bool, cb func(entry *manifestTrieEntry, suffix string)) error {
- plen := len(prefix)
- var start, stop int
- if plen == 0 {
- start = 0
- stop = 256
- } else {
- start = int(prefix[0])
- stop = start
- }
-
- for i := start; i <= stop; i++ {
- select {
- case <-quitC:
- return fmt.Errorf("aborted")
- default:
- }
- entry := mt.entries[i]
- if entry != nil {
- epl := len(entry.Path)
- if entry.ContentType == ManifestType {
- l := plen
- if epl < l {
- l = epl
- }
- if prefix[:l] == entry.Path[:l] {
- err := mt.loadSubTrie(entry, quitC)
- if err != nil {
- return err
- }
- err = entry.subtrie.listWithPrefixInt(prefix[l:], rp+entry.Path[l:], quitC, cb)
- if err != nil {
- return err
- }
- }
- } else {
- if (epl >= plen) && (prefix == entry.Path[:plen]) {
- cb(entry, rp+entry.Path[plen:])
- }
- }
- }
- }
- return nil
-}
-
-func (mt *manifestTrie) listWithPrefix(prefix string, quitC chan bool, cb func(entry *manifestTrieEntry, suffix string)) (err error) {
- return mt.listWithPrefixInt(prefix, "", quitC, cb)
-}
-
-func (mt *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *manifestTrieEntry, pos int) {
- log.Trace(fmt.Sprintf("findPrefixOf(%s)", path))
-
- if len(path) == 0 {
- return mt.entries[256], 0
- }
-
- //see if first char is in manifest entries
- b := path[0]
- entry = mt.entries[b]
- if entry == nil {
- return mt.entries[256], 0
- }
-
- epl := len(entry.Path)
- log.Trace(fmt.Sprintf("path = %v entry.Path = %v epl = %v", path, entry.Path, epl))
- if len(path) <= epl {
- if entry.Path[:len(path)] == path {
- if entry.ContentType == ManifestType {
- err := mt.loadSubTrie(entry, quitC)
- if err == nil && entry.subtrie != nil {
- subentries := entry.subtrie.entries
- for i := 0; i < len(subentries); i++ {
- sub := subentries[i]
- if sub != nil && sub.Path == "" {
- return sub, len(path)
- }
- }
- }
- entry.Status = http.StatusMultipleChoices
- }
- pos = len(path)
- return
- }
- return nil, 0
- }
- if path[:epl] == entry.Path {
- log.Trace(fmt.Sprintf("entry.ContentType = %v", entry.ContentType))
- //the subentry is a manifest, load subtrie
- if entry.ContentType == ManifestType && (strings.Contains(entry.Path, path) || strings.Contains(path, entry.Path)) {
- err := mt.loadSubTrie(entry, quitC)
- if err != nil {
- return nil, 0
- }
- sub, pos := entry.subtrie.findPrefixOf(path[epl:], quitC)
- if sub != nil {
- entry = sub
- pos += epl
- return sub, pos
- } else if path == entry.Path {
- entry.Status = http.StatusMultipleChoices
- }
-
- } else {
- //entry is not a manifest, return it
- if path != entry.Path {
- return nil, 0
- }
- }
- }
- return nil, 0
-}
-
-// file system manifest always contains regularized paths
-// no leading or trailing slashes, only single slashes inside
-func RegularSlashes(path string) (res string) {
- for i := 0; i < len(path); i++ {
- if (path[i] != '/') || ((i > 0) && (path[i-1] != '/')) {
- res = res + path[i:i+1]
- }
- }
- if (len(res) > 0) && (res[len(res)-1] == '/') {
- res = res[:len(res)-1]
- }
- return
-}
-
-func (mt *manifestTrie) getEntry(spath string) (entry *manifestTrieEntry, fullpath string) {
- path := RegularSlashes(spath)
- var pos int
- quitC := make(chan bool)
- entry, pos = mt.findPrefixOf(path, quitC)
- return entry, path[:pos]
-}
generated by cgit v1.2.3 (git 2.25.1) at 2025-01-19 19:12:53 +0800