aboutsummaryrefslogblamecommitdiffstats
path: root/swarm/api/api.go
blob: 0cf12fdbed6f6147c380ee7ac4702b46954c8794 (plain) (tree)




















                                                                                  
                  
              



                 
               


                       


                                                
                                                 
                                                       

 
                                                        
 


















                                                                                      



                                            
                                                                      
                                













                                                                                        

















                                                                                  






                                                                       
                                                       











                                                                        
                             



                                             




                                  
                                                 









                                        


















                                                                             






                                                                           











                                                                                                     
                                                         
                              
                                                          
 

                                                                  
                                                         

                                                                                                    
                 
                                                      
         



                                                                   
                                             


                                                                                      
         





                                                   
                                     
                               
         
                                              


                                                               
                                                                        
                          



                                                                   
                                 
                               




                                                                                                   
                                 
                               

                 
                       


                                                            
                                                    
                                                             
                                                                                                                               
                          
                                                     
                       
                                     
                                            
                                                                        


                      
                                                    
 
                                       
 


                                                  
                                                         
                                            





                                                                                              
                
                                            
                                     
                                                                           
                                                



              
                                                                                                      
                             
                                
                                                       
                       
                                    
                               
         
                              
                                                             
                                          
                                                 

                                        




                                             
                                                     
                                    
                               
         
                             
 

                                                                                                                     
                              


                                          
                                     



                                      
                                     

















                                                                       
                                     




                                                                 
                                     




                                   
                                     








                                                                                           
                             


                                          
                                    



                                      
                                    









                                                    
                                    




                                                        
                                    




                                  
                                    







                                                                                                                                                                                             
                                 



























                                                                         
                                        



                                      
                                        









                                                    
                                        




                                                        
                                        












                                                                       
                                        




                                   
                                        








                                                                                                                                                   
 



















                                                                                                      


                                                                                                

                                         
// Copyright 2016 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.

package api

import (
    "fmt"
    "io"
    "net/http"
    "path"
    "regexp"
    "strings"
    "sync"

    "bytes"
    "mime"
    "path/filepath"
    "time"

    "github.com/ethereum/go-ethereum/common"
    "github.com/ethereum/go-ethereum/log"
    "github.com/ethereum/go-ethereum/metrics"
    "github.com/ethereum/go-ethereum/swarm/storage"
)

var hashMatcher = regexp.MustCompile("^[0-9A-Fa-f]{64}")

//setup metrics
var (
    apiResolveCount    = metrics.NewRegisteredCounter("api.resolve.count", nil)
    apiResolveFail     = metrics.NewRegisteredCounter("api.resolve.fail", nil)
    apiPutCount        = metrics.NewRegisteredCounter("api.put.count", nil)
    apiPutFail         = metrics.NewRegisteredCounter("api.put.fail", nil)
    apiGetCount        = metrics.NewRegisteredCounter("api.get.count", nil)
    apiGetNotFound     = metrics.NewRegisteredCounter("api.get.notfound", nil)
    apiGetHttp300      = metrics.NewRegisteredCounter("api.get.http.300", nil)
    apiModifyCount     = metrics.NewRegisteredCounter("api.modify.count", nil)
    apiModifyFail      = metrics.NewRegisteredCounter("api.modify.fail", nil)
    apiAddFileCount    = metrics.NewRegisteredCounter("api.addfile.count", nil)
    apiAddFileFail     = metrics.NewRegisteredCounter("api.addfile.fail", nil)
    apiRmFileCount     = metrics.NewRegisteredCounter("api.removefile.count", nil)
    apiRmFileFail      = metrics.NewRegisteredCounter("api.removefile.fail", nil)
    apiAppendFileCount = metrics.NewRegisteredCounter("api.appendfile.count", nil)
    apiAppendFileFail  = metrics.NewRegisteredCounter("api.appendfile.fail", nil)
)

type Resolver interface {
    Resolve(string) (common.Hash, error)
}

// NoResolverError is returned by MultiResolver.Resolve if no resolver
// can be found for the address.
type NoResolverError struct {
    TLD string
}

func NewNoResolverError(tld string) *NoResolverError {
    return &NoResolverError{TLD: tld}
}

func (e *NoResolverError) Error() string {
    if e.TLD == "" {
        return "no ENS resolver"
    }
    return fmt.Sprintf("no ENS endpoint configured to resolve .%s TLD names", e.TLD)
}

// MultiResolver is used to resolve URL addresses based on their TLDs.
// Each TLD can have multiple resolvers, and the resoluton from the
// first one in the sequence will be returned.
type MultiResolver struct {
    resolvers map[string][]Resolver
}

// MultiResolverOption sets options for MultiResolver and is used as
// arguments for its constructor.
type MultiResolverOption func(*MultiResolver)

// MultiResolverOptionWithResolver adds a Resolver to a list of resolvers
// for a specific TLD. If TLD is an empty string, the resolver will be added
// to the list of default resolver, the ones that will be used for resolution
// of addresses which do not have their TLD resolver specified.
func MultiResolverOptionWithResolver(r Resolver, tld string) MultiResolverOption {
    return func(m *MultiResolver) {
        m.resolvers[tld] = append(m.resolvers[tld], r)
    }
}

// NewMultiResolver creates a new instance of MultiResolver.
func NewMultiResolver(opts ...MultiResolverOption) (m *MultiResolver) {
    m = &MultiResolver{
        resolvers: make(map[string][]Resolver),
    }
    for _, o := range opts {
        o(m)
    }
    return m
}

// Resolve resolves address by choosing a Resolver by TLD.
// If there are more default Resolvers, or for a specific TLD,
// the Hash from the the first one which does not return error
// will be returned.
func (m MultiResolver) Resolve(addr string) (h common.Hash, err error) {
    rs := m.resolvers[""]
    tld := path.Ext(addr)
    if tld != "" {
        tld = tld[1:]
        rstld, ok := m.resolvers[tld]
        if ok {
            rs = rstld
        }
    }
    if rs == nil {
        return h, NewNoResolverError(tld)
    }
    for _, r := range rs {
        h, err = r.Resolve(addr)
        if err == nil {
            return
        }
    }
    return
}

/*
Api implements webserver/file system related content storage and retrieval
on top of the dpa
it is the public interface of the dpa which is included in the ethereum stack
*/
type Api struct {
    dpa *storage.DPA
    dns Resolver
}

//the api constructor initialises
func NewApi(dpa *storage.DPA, dns Resolver) (self *Api) {
    self = &Api{
        dpa: dpa,
        dns: dns,
    }
    return
}

// to be used only in TEST
func (self *Api) Upload(uploadDir, index string) (hash string, err error) {
    fs := NewFileSystem(self)
    hash, err = fs.Upload(uploadDir, index)
    return hash, err
}

// DPA reader API
func (self *Api) Retrieve(key storage.Key) storage.LazySectionReader {
    return self.dpa.Retrieve(key)
}

func (self *Api) Store(data io.Reader, size int64, wg *sync.WaitGroup) (key storage.Key, err error) {
    return self.dpa.Store(data, size, wg, nil)
}

type ErrResolve error

// DNS Resolver
func (self *Api) Resolve(uri *URI) (storage.Key, error) {
    apiResolveCount.Inc(1)
    log.Trace(fmt.Sprintf("Resolving : %v", uri.Addr))

    // if the URI is immutable, check if the address is a hash
    isHash := hashMatcher.MatchString(uri.Addr)
    if uri.Immutable() || uri.DeprecatedImmutable() {
        if !isHash {
            return nil, fmt.Errorf("immutable address not a content hash: %q", uri.Addr)
        }
        return common.Hex2Bytes(uri.Addr), nil
    }

    // if DNS is not configured, check if the address is a hash
    if self.dns == nil {
        if !isHash {
            apiResolveFail.Inc(1)
            return nil, fmt.Errorf("no DNS to resolve name: %q", uri.Addr)
        }
        return common.Hex2Bytes(uri.Addr), nil
    }

    // try and resolve the address
    resolved, err := self.dns.Resolve(uri.Addr)
    if err == nil {
        return resolved[:], nil
    } else if !isHash {
        apiResolveFail.Inc(1)
        return nil, err
    }
    return common.Hex2Bytes(uri.Addr), nil
}

// Put provides singleton manifest creation on top of dpa store
func (self *Api) Put(content, contentType string) (storage.Key, error) {
    apiPutCount.Inc(1)
    r := strings.NewReader(content)
    wg := &sync.WaitGroup{}
    key, err := self.dpa.Store(r, int64(len(content)), wg, nil)
    if err != nil {
        apiPutFail.Inc(1)
        return nil, err
    }
    manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType)
    r = strings.NewReader(manifest)
    key, err = self.dpa.Store(r, int64(len(manifest)), wg, nil)
    if err != nil {
        apiPutFail.Inc(1)
        return nil, err
    }
    wg.Wait()
    return key, nil
}

// Get uses iterative manifest retrieval and prefix matching
// to resolve basePath to content using dpa retrieve
// it returns a section reader, mimeType, status and an error
func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionReader, mimeType string, status int, err error) {
    apiGetCount.Inc(1)
    trie, err := loadManifest(self.dpa, key, nil)
    if err != nil {
        apiGetNotFound.Inc(1)
        status = http.StatusNotFound
        log.Warn(fmt.Sprintf("loadManifestTrie error: %v", err))
        return
    }

    log.Trace(fmt.Sprintf("getEntry(%s)", path))

    entry, _ := trie.getEntry(path)

    if entry != nil {
        key = common.Hex2Bytes(entry.Hash)
        status = entry.Status
        if status == http.StatusMultipleChoices {
            apiGetHttp300.Inc(1)
            return
        } else {
            mimeType = entry.ContentType
            log.Trace(fmt.Sprintf("content lookup key: '%v' (%v)", key, mimeType))
            reader = self.dpa.Retrieve(key)
        }
    } else {
        status = http.StatusNotFound
        apiGetNotFound.Inc(1)
        err = fmt.Errorf("manifest entry for '%s' not found", path)
        log.Warn(fmt.Sprintf("%v", err))
    }
    return
}

func (self *Api) Modify(key storage.Key, path, contentHash, contentType string) (storage.Key, error) {
    apiModifyCount.Inc(1)
    quitC := make(chan bool)
    trie, err := loadManifest(self.dpa, key, quitC)
    if err != nil {
        apiModifyFail.Inc(1)
        return nil, err
    }
    if contentHash != "" {
        entry := newManifestTrieEntry(&ManifestEntry{
            Path:        path,
            ContentType: contentType,
        }, nil)
        entry.Hash = contentHash
        trie.addEntry(entry, quitC)
    } else {
        trie.deleteEntry(path, quitC)
    }

    if err := trie.recalcAndStore(); err != nil {
        apiModifyFail.Inc(1)
        return nil, err
    }
    return trie.hash, nil
}

func (self *Api) AddFile(mhash, path, fname string, content []byte, nameresolver bool) (storage.Key, string, error) {
    apiAddFileCount.Inc(1)

    uri, err := Parse("bzz:/" + mhash)
    if err != nil {
        apiAddFileFail.Inc(1)
        return nil, "", err
    }
    mkey, err := self.Resolve(uri)
    if err != nil {
        apiAddFileFail.Inc(1)
        return nil, "", err
    }

    // trim the root dir we added
    if path[:1] == "/" {
        path = path[1:]
    }

    entry := &ManifestEntry{
        Path:        filepath.Join(path, fname),
        ContentType: mime.TypeByExtension(filepath.Ext(fname)),
        Mode:        0700,
        Size:        int64(len(content)),
        ModTime:     time.Now(),
    }

    mw, err := self.NewManifestWriter(mkey, nil)
    if err != nil {
        apiAddFileFail.Inc(1)
        return nil, "", err
    }

    fkey, err := mw.AddEntry(bytes.NewReader(content), entry)
    if err != nil {
        apiAddFileFail.Inc(1)
        return nil, "", err
    }

    newMkey, err := mw.Store()
    if err != nil {
        apiAddFileFail.Inc(1)
        return nil, "", err

    }

    return fkey, newMkey.String(), nil

}

func (self *Api) RemoveFile(mhash, path, fname string, nameresolver bool) (string, error) {
    apiRmFileCount.Inc(1)

    uri, err := Parse("bzz:/" + mhash)
    if err != nil {
        apiRmFileFail.Inc(1)
        return "", err
    }
    mkey, err := self.Resolve(uri)
    if err != nil {
        apiRmFileFail.Inc(1)
        return "", err
    }

    // trim the root dir we added
    if path[:1] == "/" {
        path = path[1:]
    }

    mw, err := self.NewManifestWriter(mkey, nil)
    if err != nil {
        apiRmFileFail.Inc(1)
        return "", err
    }

    err = mw.RemoveEntry(filepath.Join(path, fname))
    if err != nil {
        apiRmFileFail.Inc(1)
        return "", err
    }

    newMkey, err := mw.Store()
    if err != nil {
        apiRmFileFail.Inc(1)
        return "", err

    }

    return newMkey.String(), nil
}

func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, content []byte, oldKey storage.Key, offset int64, addSize int64, nameresolver bool) (storage.Key, string, error) {
    apiAppendFileCount.Inc(1)

    buffSize := offset + addSize
    if buffSize < existingSize {
        buffSize = existingSize
    }

    buf := make([]byte, buffSize)

    oldReader := self.Retrieve(oldKey)
    io.ReadAtLeast(oldReader, buf, int(offset))

    newReader := bytes.NewReader(content)
    io.ReadAtLeast(newReader, buf[offset:], int(addSize))

    if buffSize < existingSize {
        io.ReadAtLeast(oldReader, buf[addSize:], int(buffSize))
    }

    combinedReader := bytes.NewReader(buf)
    totalSize := int64(len(buf))

    // TODO(jmozah): to append using pyramid chunker when it is ready
    //oldReader := self.Retrieve(oldKey)
    //newReader := bytes.NewReader(content)
    //combinedReader := io.MultiReader(oldReader, newReader)

    uri, err := Parse("bzz:/" + mhash)
    if err != nil {
        apiAppendFileFail.Inc(1)
        return nil, "", err
    }
    mkey, err := self.Resolve(uri)
    if err != nil {
        apiAppendFileFail.Inc(1)
        return nil, "", err
    }

    // trim the root dir we added
    if path[:1] == "/" {
        path = path[1:]
    }

    mw, err := self.NewManifestWriter(mkey, nil)
    if err != nil {
        apiAppendFileFail.Inc(1)
        return nil, "", err
    }

    err = mw.RemoveEntry(filepath.Join(path, fname))
    if err != nil {
        apiAppendFileFail.Inc(1)
        return nil, "", err
    }

    entry := &ManifestEntry{
        Path:        filepath.Join(path, fname),
        ContentType: mime.TypeByExtension(filepath.Ext(fname)),
        Mode:        0700,
        Size:        totalSize,
        ModTime:     time.Now(),
    }

    fkey, err := mw.AddEntry(io.Reader(combinedReader), entry)
    if err != nil {
        apiAppendFileFail.Inc(1)
        return nil, "", err
    }

    newMkey, err := mw.Store()
    if err != nil {
        apiAppendFileFail.Inc(1)
        return nil, "", err

    }

    return fkey, newMkey.String(), nil

}

func (self *Api) BuildDirectoryTree(mhash string, nameresolver bool) (key storage.Key, manifestEntryMap map[string]*manifestTrieEntry, err error) {

    uri, err := Parse("bzz:/" + mhash)
    if err != nil {
        return nil, nil, err
    }
    key, err = self.Resolve(uri)
    if err != nil {
        return nil, nil, err
    }

    quitC := make(chan bool)
    rootTrie, err := loadManifest(self.dpa, key, quitC)
    if err != nil {
        return nil, nil, fmt.Errorf("can't load manifest %v: %v", key.String(), err)
    }

    manifestEntryMap = map[string]*manifestTrieEntry{}
    err = rootTrie.listWithPrefix(uri.Path, quitC, func(entry *manifestTrieEntry, suffix string) {
        manifestEntryMap[suffix] = entry
    })

    if err != nil {
        return nil, nil, fmt.Errorf("list with prefix failed %v: %v", key.String(), err)
    }
    return key, manifestEntryMap, nil
}