path: root/swarm/api/manifest.go

// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.

package api

import (
    "bytes"
    "context"
    "encoding/json"
    "errors"
    "fmt"
    "io"
    "net/http"
    "strings"
    "time"

    "github.com/ethereum/go-ethereum/swarm/storage/mru"

    "github.com/ethereum/go-ethereum/common"
    "github.com/ethereum/go-ethereum/swarm/log"
    "github.com/ethereum/go-ethereum/swarm/storage"
)

const (
    ManifestType        = "application/bzz-manifest+json"
    ResourceContentType = "application/bzz-resource"

    manifestSizeLimit = 5 * 1024 * 1024
)

// Manifest represents a swarm manifest
type Manifest struct {
    Entries []ManifestEntry `json:"entries,omitempty"`
}

// ManifestEntry represents an entry in a swarm manifest
type ManifestEntry struct {
    Hash         string       `json:"hash,omitempty"`
    Path         string       `json:"path,omitempty"`
    ContentType  string       `json:"contentType,omitempty"`
    Mode         int64        `json:"mode,omitempty"`
    Size         int64        `json:"size,omitempty"`
    ModTime      time.Time    `json:"mod_time,omitempty"`
    Status       int          `json:"status,omitempty"`
    Access       *AccessEntry `json:"access,omitempty"`
    ResourceView *mru.Feed    `json:"resourceView,omitempty"`
}

// ManifestList represents the result of listing files in a manifest
type ManifestList struct {
    CommonPrefixes []string         `json:"common_prefixes,omitempty"`
    Entries        []*ManifestEntry `json:"entries,omitempty"`
}

// NewManifest creates and stores a new, empty manifest
func (a *API) NewManifest(ctx context.Context, toEncrypt bool) (storage.Address, error) {
    var manifest Manifest
    data, err := json.Marshal(&manifest)
    if err != nil {
        return nil, err
    }
    addr, wait, err := a.Store(ctx, bytes.NewReader(data), int64(len(data)), toEncrypt)
    if err != nil {
        return nil, err
    }
    err = wait(ctx)
    return addr, err
}

// Manifest hack for supporting Mutable Resource Updates from the bzz: scheme
// see swarm/api/api.go:API.Get() for more information
func (a *API) NewResourceManifest(ctx context.Context, view *mru.Feed) (storage.Address, error) {
    var manifest Manifest
    entry := ManifestEntry{
        ResourceView: view,
        ContentType:  ResourceContentType,
    }
    manifest.Entries = append(manifest.Entries, entry)
    data, err := json.Marshal(&manifest)
    if err != nil {
        return nil, err
    }
    addr, wait, err := a.Store(ctx, bytes.NewReader(data), int64(len(data)), false)
    if err != nil {
        return nil, err
    }
    err = wait(ctx)
    return addr, err
}

// ManifestWriter is used to add and remove entries from an underlying manifest
type ManifestWriter struct {
    api   *API
    trie  *manifestTrie
    quitC chan bool
}

func (a *API) NewManifestWriter(ctx context.Context, addr storage.Address, quitC chan bool) (*ManifestWriter, error) {
    trie, err := loadManifest(ctx, a.fileStore, addr, quitC, NOOPDecrypt)
    if err != nil {
        return nil, fmt.Errorf("error loading manifest %s: %s", addr, err)
    }
    return &ManifestWriter{a, trie, quitC}, nil
}

// AddEntry stores the given data and adds the resulting address to the manifest
func (m *ManifestWriter) AddEntry(ctx context.Context, data io.Reader, e *ManifestEntry) (addr storage.Address, err error) {
    entry := newManifestTrieEntry(e, nil)
    if data != nil {
        var wait func(context.Context) error
        addr, wait, err = m.api.Store(ctx, data, e.Size, m.trie.encrypted)
        if err != nil {
            return nil, err
        }
        err = wait(ctx)
        if err != nil {
            return nil, err
        }
        entry.Hash = addr.Hex()
    }
    if entry.Hash == "" {
        return addr, errors.New("missing entry hash")
    }
    m.trie.addEntry(entry, m.quitC)
    return addr, nil
}

// RemoveEntry removes the given path from the manifest
func (m *ManifestWriter) RemoveEntry(path string) error {
    m.trie.deleteEntry(path, m.quitC)
    return nil
}

// Store stores the manifest, returning the resulting storage address
func (m *ManifestWriter) Store() (storage.Address, error) {
    return m.trie.ref, m.trie.recalcAndStore()
}

// ManifestWalker is used to recursively walk the entries in the manifest and
// all of its submanifests
type ManifestWalker struct {
    api   *API
    trie  *manifestTrie
    quitC chan bool
}

func (a *API) NewManifestWalker(ctx context.Context, addr storage.Address, decrypt DecryptFunc, quitC chan bool) (*ManifestWalker, error) {
    trie, err := loadManifest(ctx, a.fileStore, addr, quitC, decrypt)
    if err != nil {
        return nil, fmt.Errorf("error loading manifest %s: %s", addr, err)
    }
    return &ManifestWalker{a, trie, quitC}, nil
}

// ErrSkipManifest is used as a return value from WalkFn to indicate that the
// manifest should be skipped
var ErrSkipManifest = errors.New("skip this manifest")

// WalkFn is the type of function called for each entry visited by a recursive
// manifest walk
type WalkFn func(entry *ManifestEntry) error

// Walk recursively walks the manifest calling walkFn for each entry in the
// manifest, including submanifests
func (m *ManifestWalker) Walk(walkFn WalkFn) error {
    return m.walk(m.trie, "", walkFn)
}

func (m *ManifestWalker) walk(trie *manifestTrie, prefix string, walkFn WalkFn) error {
    for _, entry := range &trie.entries {
        if entry == nil {
            continue
        }
        entry.Path = prefix + entry.Path
        err := walkFn(&entry.ManifestEntry)
        if err != nil {
            if entry.ContentType == ManifestType && err == ErrSkipManifest {
                continue
            }
            return err
        }
        if entry.ContentType != ManifestType {
            continue
        }
        if err := trie.loadSubTrie(entry, nil); err != nil {
            return err
        }
        if err := m.walk(entry.subtrie, entry.Path, walkFn); err != nil {
            return err
        }
    }
    return nil
}

type manifestTrie struct {
    fileStore *storage.FileStore
    entries   [257]*manifestTrieEntry // indexed by first character of basePath, entries[256] is the empty basePath entry
    ref       storage.Address         // if ref != nil, it is stored
    encrypted bool
    decrypt   DecryptFunc
}

func newManifestTrieEntry(entry *ManifestEntry, subtrie *manifestTrie) *manifestTrieEntry {
    return &manifestTrieEntry{
        ManifestEntry: *entry,
        subtrie:       subtrie,
    }
}

type manifestTrieEntry struct {
    ManifestEntry

    subtrie *manifestTrie
}

func loadManifest(ctx context.Context, fileStore *storage.FileStore, addr storage.Address, quitC chan bool, decrypt DecryptFunc) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
    log.Trace("manifest lookup", "addr", addr)
    // retrieve manifest via FileStore
    manifestReader, isEncrypted := fileStore.Retrieve(ctx, addr)
    log.Trace("reader retrieved", "addr", addr)
    return readManifest(manifestReader, addr, fileStore, isEncrypted, quitC, decrypt)
}

func readManifest(mr storage.LazySectionReader, addr storage.Address, fileStore *storage.FileStore, isEncrypted bool, quitC chan bool, decrypt DecryptFunc) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand

    // TODO check size for oversized manifests
    size, err := mr.Size(mr.Context(), quitC)
    if err != nil { // size == 0
        // can't determine size means we don't have the root chunk
        log.Trace("manifest not found", "addr", addr)
        err = fmt.Errorf("Manifest not Found")
        return
    }
    if size > manifestSizeLimit {
        log.Warn("manifest exceeds size limit", "addr", addr, "size", size, "limit", manifestSizeLimit)
        err = fmt.Errorf("Manifest size of %v bytes exceeds the %v byte limit", size, manifestSizeLimit)
        return
    }
    manifestData := make([]byte, size)
    read, err := mr.Read(manifestData)
    if int64(read) < size {
        log.Trace("manifest not found", "addr", addr)
        if err == nil {
            err = fmt.Errorf("Manifest retrieval cut short: read %v, expect %v", read, size)
        }
        return
    }

    log.Debug("manifest retrieved", "addr", addr)
    var man struct {
        Entries []*manifestTrieEntry `json:"entries"`
    }
    err = json.Unmarshal(manifestData, &man)
    if err != nil {
        err = fmt.Errorf("Manifest %v is malformed: %v", addr.Log(), err)
        log.Trace("malformed manifest", "addr", addr)
        return
    }

    log.Trace("manifest entries", "addr", addr, "len", len(man.Entries))

    trie = &manifestTrie{
        fileStore: fileStore,
        encrypted: isEncrypted,
        decrypt:   decrypt,
    }
    for _, entry := range man.Entries {
        err = trie.addEntry(entry, quitC)
        if err != nil {
            return
        }
    }
    return
}

func (mt *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) error {
    mt.ref = nil // trie modified, hash needs to be re-calculated on demand

    if entry.ManifestEntry.Access != nil {
        if mt.decrypt == nil {
            return errors.New("dont have decryptor")
        }

        err := mt.decrypt(&entry.ManifestEntry)
        if err != nil {
            return err
        }
    }

    if len(entry.Path) == 0 {
        mt.entries[256] = entry
        return nil
    }

    b := entry.Path[0]
    oldentry := mt.entries[b]
    if (oldentry == nil) || (oldentry.Path == entry.Path && oldentry.ContentType != ManifestType) {
        mt.entries[b] = entry
        return nil
    }

    cpl := 0
    for (len(entry.Path) > cpl) && (len(oldentry.Path) > cpl) && (entry.Path[cpl] == oldentry.Path[cpl]) {
        cpl++
    }

    if (oldentry.ContentType == ManifestType) && (cpl == len(oldentry.Path)) {
        if mt.loadSubTrie(oldentry, quitC) != nil {
            return nil
        }
        entry.Path = entry.Path[cpl:]
        oldentry.subtrie.addEntry(entry, quitC)
        oldentry.Hash = ""
        return nil
    }

    commonPrefix := entry.Path[:cpl]

    subtrie := &manifestTrie{
        fileStore: mt.fileStore,
        encrypted: mt.encrypted,
    }
    entry.Path = entry.Path[cpl:]
    oldentry.Path = oldentry.Path[cpl:]
    subtrie.addEntry(entry, quitC)
    subtrie.addEntry(oldentry, quitC)

    mt.entries[b] = newManifestTrieEntry(&ManifestEntry{
        Path:        commonPrefix,
        ContentType: ManifestType,
    }, subtrie)
    return nil
}

func (mt *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) {
    for _, e := range &mt.entries {
        if e != nil {
            cnt++
            entry = e
        }
    }
    return
}

func (mt *manifestTrie) deleteEntry(path string, quitC chan bool) {
    mt.ref = nil // trie modified, hash needs to be re-calculated on demand

    if len(path) == 0 {
        mt.entries[256] = nil
        return
    }

    b := path[0]
    entry := mt.entries[b]
    if entry == nil {
        return
    }
    if entry.Path == path {
        mt.entries[b] = nil
        return
    }

    epl := len(entry.Path)
    if (entry.ContentType == ManifestType) && (len(path) >= epl) && (path[:epl] == entry.Path) {
        if mt.loadSubTrie(entry, quitC) != nil {
            return
        }
        entry.subtrie.deleteEntry(path[epl:], quitC)
        entry.Hash = ""
        // remove subtree if it has less than 2 elements
        cnt, lastentry := entry.subtrie.getCountLast()
        if cnt < 2 {
            if lastentry != nil {
                lastentry.Path = entry.Path + lastentry.Path
            }
            mt.entries[b] = lastentry
        }
    }
}

func (mt *manifestTrie) recalcAndStore() error {
    if mt.ref != nil {
        return nil
    }

    var buffer bytes.Buffer
    buffer.WriteString(`{"entries":[`)

    list := &Manifest{}
    for _, entry := range &mt.entries {
        if entry != nil {
            if entry.Hash == "" { // TODO: paralellize
                err := entry.subtrie.recalcAndStore()
                if err != nil {
                    return err
                }
                entry.Hash = entry.subtrie.ref.Hex()
            }
            list.Entries = append(list.Entries, entry.ManifestEntry)
        }

    }

    manifest, err := json.Marshal(list)
    if err != nil {
        return err
    }

    sr := bytes.NewReader(manifest)
    ctx := context.TODO()
    addr, wait, err2 := mt.fileStore.Store(ctx, sr, int64(len(manifest)), mt.encrypted)
    if err2 != nil {
        return err2
    }
    err2 = wait(ctx)
    mt.ref = addr
    return err2
}

func (mt *manifestTrie) loadSubTrie(entry *manifestTrieEntry, quitC chan bool) (err error) {
    if entry.ManifestEntry.Access != nil {
        if mt.decrypt == nil {
            return errors.New("dont have decryptor")
        }

        err := mt.decrypt(&entry.ManifestEntry)
        if err != nil {
            return err
        }
    }

    if entry.subtrie == nil {
        hash := common.Hex2Bytes(entry.Hash)
        entry.subtrie, err = loadManifest(context.TODO(), mt.fileStore, hash, quitC, mt.decrypt)
        entry.Hash = "" // might not match, should be recalculated
    }
    return
}

func (mt *manifestTrie) listWithPrefixInt(prefix, rp string, quitC chan bool, cb func(entry *manifestTrieEntry, suffix string)) error {
    plen := len(prefix)
    var start, stop int
    if plen == 0 {
        start = 0
        stop = 256
    } else {
        start = int(prefix[0])
        stop = start
    }

    for i := start; i <= stop; i++ {
        select {
        case <-quitC:
            return fmt.Errorf("aborted")
        default:
        }
        entry := mt.entries[i]
        if entry != nil {
            epl := len(entry.Path)
            if entry.ContentType == ManifestType {
                l := plen
                if epl < l {
                    l = epl
                }
                if prefix[:l] == entry.Path[:l] {
                    err := mt.loadSubTrie(entry, quitC)
                    if err != nil {
                        return err
                    }
                    err = entry.subtrie.listWithPrefixInt(prefix[l:], rp+entry.Path[l:], quitC, cb)
                    if err != nil {
                        return err
                    }
                }
            } else {
                if (epl >= plen) && (prefix == entry.Path[:plen]) {
                    cb(entry, rp+entry.Path[plen:])
                }
            }
        }
    }
    return nil
}

func (mt *manifestTrie) listWithPrefix(prefix string, quitC chan bool, cb func(entry *manifestTrieEntry, suffix string)) (err error) {
    return mt.listWithPrefixInt(prefix, "", quitC, cb)
}

func (mt *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *manifestTrieEntry, pos int) {
    log.Trace(fmt.Sprintf("findPrefixOf(%s)", path))

    if len(path) == 0 {
        return mt.entries[256], 0
    }

    //see if first char is in manifest entries
    b := path[0]
    entry = mt.entries[b]
    if entry == nil {
        return mt.entries[256], 0
    }

    epl := len(entry.Path)
    log.Trace(fmt.Sprintf("path = %v  entry.Path = %v  epl = %v", path, entry.Path, epl))
    if len(path) <= epl {
        if entry.Path[:len(path)] == path {
            if entry.ContentType == ManifestType {
                err := mt.loadSubTrie(entry, quitC)
                if err == nil && entry.subtrie != nil {
                    subentries := entry.subtrie.entries
                    for i := 0; i < len(subentries); i++ {
                        sub := subentries[i]
                        if sub != nil && sub.Path == "" {
                            return sub, len(path)
                        }
                    }
                }
                entry.Status = http.StatusMultipleChoices
            }
            pos = len(path)
            return
        }
        return nil, 0
    }
    if path[:epl] == entry.Path {
        log.Trace(fmt.Sprintf("entry.ContentType = %v", entry.ContentType))
        //the subentry is a manifest, load subtrie
        if entry.ContentType == ManifestType && (strings.Contains(entry.Path, path) || strings.Contains(path, entry.Path)) {
            err := mt.loadSubTrie(entry, quitC)
            if err != nil {
                return nil, 0
            }
            sub, pos := entry.subtrie.findPrefixOf(path[epl:], quitC)
            if sub != nil {
                entry = sub
                pos += epl
                return sub, pos
            } else if path == entry.Path {
                entry.Status = http.StatusMultipleChoices
            }

        } else {
            //entry is not a manifest, return it
            if path != entry.Path {
                return nil, 0
            }
            pos = epl
        }
    }
    return nil, 0
}

// file system manifest always contains regularized paths
// no leading or trailing slashes, only single slashes inside
func RegularSlashes(path string) (res string) {
    for i := 0; i < len(path); i++ {
        if (path[i] != '/') || ((i > 0) && (path[i-1] != '/')) {
            res = res + path[i:i+1]
        }
    }
    if (len(res) > 0) && (res[len(res)-1] == '/') {
        res = res[:len(res)-1]
    }
    return
}

func (mt *manifestTrie) getEntry(spath string) (entry *manifestTrieEntry, fullpath string) {
    path := RegularSlashes(spath)
    var pos int
    quitC := make(chan bool)
    entry, pos = mt.findPrefixOf(path, quitC)
    return entry, path[:pos]
}