From 4d300e4dece56535f56ccc32330340ce89e42581 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9ETH=CE=9E=D0=AFSPH=CE=9E=D0=AF=CE=9E?= <{viktor.tron,nagydani,zsfelfoldi}@gmail.com> Date: Mon, 29 Aug 2016 21:18:00 +0200 Subject: swarm: plan bee for content storage and distribution on web3 This change imports the Swarm protocol codebase. Compared to the 'swarm' branch, a few mostly cosmetic changes had to be made: * The various redundant log message prefixes are gone. * All files now have LGPLv3 license headers. * Minor code changes were needed to please go vet and make the tests pass on Windows. * Further changes were required to adapt to the go-ethereum develop branch and its new Go APIs. Some code has not (yet) been brought over: * swarm/cmd/bzzhash: will reappear as cmd/bzzhash later * swarm/cmd/bzzup.sh: will be reimplemented in cmd/bzzup * swarm/cmd/makegenesis: will reappear somehow * swarm/examples/album: will move to a separate repository * swarm/examples/filemanager: ditto * swarm/examples/files: will not be merged * swarm/test/*: will not be merged * swarm/services/swear: will reappear as contracts/swear when needed --- swarm/api/api.go | 191 +++++++++++++++++++ swarm/api/api_test.go | 117 ++++++++++++ swarm/api/config.go | 132 +++++++++++++ swarm/api/config_test.go | 124 +++++++++++++ swarm/api/filesystem.go | 283 ++++++++++++++++++++++++++++ swarm/api/filesystem_test.go | 187 +++++++++++++++++++ swarm/api/http/roundtripper.go | 69 +++++++ swarm/api/http/roundtripper_test.go | 68 +++++++ swarm/api/http/server.go | 286 +++++++++++++++++++++++++++++ swarm/api/manifest.go | 336 ++++++++++++++++++++++++++++++++++ swarm/api/manifest_test.go | 80 ++++++++ swarm/api/storage.go | 70 +++++++ swarm/api/storage_test.go | 49 +++++ swarm/api/testapi.go | 46 +++++ swarm/api/testdata/test0/img/logo.png | Bin 0 -> 18136 bytes swarm/api/testdata/test0/index.css | 9 + swarm/api/testdata/test0/index.html | 10 + 17 files changed, 2057 insertions(+) create mode 100644 swarm/api/api.go create mode 100644 swarm/api/api_test.go create mode 100644 swarm/api/config.go create mode 100644 swarm/api/config_test.go create mode 100644 swarm/api/filesystem.go create mode 100644 swarm/api/filesystem_test.go create mode 100644 swarm/api/http/roundtripper.go create mode 100644 swarm/api/http/roundtripper_test.go create mode 100644 swarm/api/http/server.go create mode 100644 swarm/api/manifest.go create mode 100644 swarm/api/manifest_test.go create mode 100644 swarm/api/storage.go create mode 100644 swarm/api/storage_test.go create mode 100644 swarm/api/testapi.go create mode 100644 swarm/api/testdata/test0/img/logo.png create mode 100644 swarm/api/testdata/test0/index.css create mode 100644 swarm/api/testdata/test0/index.html (limited to 'swarm/api') diff --git a/swarm/api/api.go b/swarm/api/api.go new file mode 100644 index 000000000..673cff350 --- /dev/null +++ b/swarm/api/api.go @@ -0,0 +1,191 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package api + +import ( + "fmt" + "io" + "regexp" + "strings" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/logger" + "github.com/ethereum/go-ethereum/logger/glog" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +var ( + hashMatcher = regexp.MustCompile("^[0-9A-Fa-f]{64}") + slashes = regexp.MustCompile("/+") + domainAndVersion = regexp.MustCompile("[@:;,]+") +) + +type Resolver interface { + Resolve(string) (common.Hash, error) +} + +/* +Api implements webserver/file system related content storage and retrieval +on top of the dpa +it is the public interface of the dpa which is included in the ethereum stack +*/ +type Api struct { + dpa *storage.DPA + dns Resolver +} + +//the api constructor initialises +func NewApi(dpa *storage.DPA, dns Resolver) (self *Api) { + self = &Api{ + dpa: dpa, + dns: dns, + } + return +} + +// DPA reader API +func (self *Api) Retrieve(key storage.Key) storage.LazySectionReader { + return self.dpa.Retrieve(key) +} + +func (self *Api) Store(data io.Reader, size int64, wg *sync.WaitGroup) (key storage.Key, err error) { + return self.dpa.Store(data, size, wg, nil) +} + +type ErrResolve error + +// DNS Resolver +func (self *Api) Resolve(hostPort string, nameresolver bool) (storage.Key, error) { + if hashMatcher.MatchString(hostPort) || self.dns == nil { + glog.V(logger.Detail).Infof("host is a contentHash: '%v'", hostPort) + return storage.Key(common.Hex2Bytes(hostPort)), nil + } + if !nameresolver { + return nil, fmt.Errorf("'%s' is not a content hash value.", hostPort) + } + contentHash, err := self.dns.Resolve(hostPort) + if err != nil { + err = ErrResolve(err) + glog.V(logger.Warn).Infof("DNS error : %v", err) + } + glog.V(logger.Detail).Infof("host lookup: %v -> %v", err) + return contentHash[:], err +} + +func parse(uri string) (hostPort, path string) { + parts := slashes.Split(uri, 3) + var i int + if len(parts) == 0 { + return + } + // beginning with slash is now optional + for len(parts[i]) == 0 { + i++ + } + hostPort = parts[i] + for i < len(parts)-1 { + i++ + if len(path) > 0 { + path = path + "/" + parts[i] + } else { + path = parts[i] + } + } + glog.V(logger.Debug).Infof("host: '%s', path '%s' requested.", hostPort, path) + return +} + +func (self *Api) parseAndResolve(uri string, nameresolver bool) (key storage.Key, hostPort, path string, err error) { + hostPort, path = parse(uri) + //resolving host and port + contentHash, err := self.Resolve(hostPort, nameresolver) + glog.V(logger.Debug).Infof("Resolved '%s' to contentHash: '%s', path: '%s'", uri, contentHash, path) + return contentHash[:], hostPort, path, err +} + +// Put provides singleton manifest creation on top of dpa store +func (self *Api) Put(content, contentType string) (string, error) { + r := strings.NewReader(content) + wg := &sync.WaitGroup{} + key, err := self.dpa.Store(r, int64(len(content)), wg, nil) + if err != nil { + return "", err + } + manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType) + r = strings.NewReader(manifest) + key, err = self.dpa.Store(r, int64(len(manifest)), wg, nil) + if err != nil { + return "", err + } + wg.Wait() + return key.String(), nil +} + +// Get uses iterative manifest retrieval and prefix matching +// to resolve path to content using dpa retrieve +// it returns a section reader, mimeType, status and an error +func (self *Api) Get(uri string, nameresolver bool) (reader storage.LazySectionReader, mimeType string, status int, err error) { + + key, _, path, err := self.parseAndResolve(uri, nameresolver) + quitC := make(chan bool) + trie, err := loadManifest(self.dpa, key, quitC) + if err != nil { + glog.V(logger.Warn).Infof("loadManifestTrie error: %v", err) + return + } + + glog.V(logger.Detail).Infof("getEntry(%s)", path) + entry, _ := trie.getEntry(path) + if entry != nil { + key = common.Hex2Bytes(entry.Hash) + status = entry.Status + mimeType = entry.ContentType + glog.V(logger.Detail).Infof("content lookup key: '%v' (%v)", key, mimeType) + reader = self.dpa.Retrieve(key) + } else { + err = fmt.Errorf("manifest entry for '%s' not found", path) + glog.V(logger.Warn).Infof("%v", err) + } + return +} + +func (self *Api) Modify(uri, contentHash, contentType string, nameresolver bool) (newRootHash string, err error) { + root, _, path, err := self.parseAndResolve(uri, nameresolver) + quitC := make(chan bool) + trie, err := loadManifest(self.dpa, root, quitC) + if err != nil { + return + } + + if contentHash != "" { + entry := &manifestTrieEntry{ + Path: path, + Hash: contentHash, + ContentType: contentType, + } + trie.addEntry(entry, quitC) + } else { + trie.deleteEntry(path, quitC) + } + + err = trie.recalcAndStore() + if err != nil { + return + } + return trie.hash.String(), nil +} diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go new file mode 100644 index 000000000..b09811959 --- /dev/null +++ b/swarm/api/api_test.go @@ -0,0 +1,117 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package api + +import ( + "io" + "io/ioutil" + "os" + "testing" + + "github.com/ethereum/go-ethereum/logger" + "github.com/ethereum/go-ethereum/logger/glog" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +func testApi(t *testing.T, f func(*Api)) { + datadir, err := ioutil.TempDir("", "bzz-test") + if err != nil { + t.Fatalf("unable to create temp dir: %v", err) + } + os.RemoveAll(datadir) + defer os.RemoveAll(datadir) + dpa, err := storage.NewLocalDPA(datadir) + if err != nil { + return + } + api := NewApi(dpa, nil) + dpa.Start() + f(api) + dpa.Stop() +} + +type testResponse struct { + reader storage.LazySectionReader + *Response +} + +func checkResponse(t *testing.T, resp *testResponse, exp *Response) { + + if resp.MimeType != exp.MimeType { + t.Errorf("incorrect mimeType. expected '%s', got '%s'", exp.MimeType, resp.MimeType) + } + if resp.Status != exp.Status { + t.Errorf("incorrect status. expected '%d', got '%d'", exp.Status, resp.Status) + } + if resp.Size != exp.Size { + t.Errorf("incorrect size. expected '%d', got '%d'", exp.Size, resp.Size) + } + if resp.reader != nil { + content := make([]byte, resp.Size) + read, _ := resp.reader.Read(content) + if int64(read) != exp.Size { + t.Errorf("incorrect content length. expected '%d...', got '%d...'", read, exp.Size) + } + resp.Content = string(content) + } + if resp.Content != exp.Content { + // if !bytes.Equal(resp.Content, exp.Content) + t.Errorf("incorrect content. expected '%s...', got '%s...'", string(exp.Content), string(resp.Content)) + } +} + +// func expResponse(content []byte, mimeType string, status int) *Response { +func expResponse(content string, mimeType string, status int) *Response { + glog.V(logger.Detail).Infof("expected content (%v): %v ", len(content), content) + return &Response{mimeType, status, int64(len(content)), content} +} + +// func testGet(t *testing.T, api *Api, bzzhash string) *testResponse { +func testGet(t *testing.T, api *Api, bzzhash string) *testResponse { + reader, mimeType, status, err := api.Get(bzzhash, true) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + quitC := make(chan bool) + size, err := reader.Size(quitC) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + glog.V(logger.Detail).Infof("reader size: %v ", size) + s := make([]byte, size) + _, err = reader.Read(s) + if err != io.EOF { + t.Fatalf("unexpected error: %v", err) + } + reader.Seek(0, 0) + return &testResponse{reader, &Response{mimeType, status, size, string(s)}} + // return &testResponse{reader, &Response{mimeType, status, reader.Size(), nil}} +} + +func TestApiPut(t *testing.T) { + testApi(t, func(api *Api) { + content := "hello" + exp := expResponse(content, "text/plain", 0) + // exp := expResponse([]byte(content), "text/plain", 0) + bzzhash, err := api.Put(content, exp.MimeType) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + resp := testGet(t, api, bzzhash) + checkResponse(t, resp, exp) + }) +} diff --git a/swarm/api/config.go b/swarm/api/config.go new file mode 100644 index 000000000..730755c43 --- /dev/null +++ b/swarm/api/config.go @@ -0,0 +1,132 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package api + +import ( + "crypto/ecdsa" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/swarm/network" + "github.com/ethereum/go-ethereum/swarm/services/swap" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +const ( + port = "8500" +) + +// by default ens root is north internal +var ( + toyNetEnsRoot = common.HexToAddress("0xd344889e0be3e9ef6c26b0f60ef66a32e83c1b69") +) + +// separate bzz directories +// allow several bzz nodes running in parallel +type Config struct { + // serialised/persisted fields + *storage.StoreParams + *storage.ChunkerParams + *network.HiveParams + Swap *swap.SwapParams + *network.SyncParams + Path string + Port string + PublicKey string + BzzKey string + EnsRoot common.Address +} + +// config is agnostic to where private key is coming from +// so managing accounts is outside swarm and left to wrappers +func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey) (self *Config, err error) { + + address := crypto.PubkeyToAddress(prvKey.PublicKey) // default beneficiary address + dirpath := filepath.Join(path, common.Bytes2Hex(address.Bytes())) + err = os.MkdirAll(dirpath, os.ModePerm) + if err != nil { + return + } + confpath := filepath.Join(dirpath, "config.json") + var data []byte + pubkey := crypto.FromECDSAPub(&prvKey.PublicKey) + pubkeyhex := common.ToHex(pubkey) + keyhex := crypto.Sha3Hash(pubkey).Hex() + + self = &Config{ + SyncParams: network.NewSyncParams(dirpath), + HiveParams: network.NewHiveParams(dirpath), + ChunkerParams: storage.NewChunkerParams(), + StoreParams: storage.NewStoreParams(dirpath), + Port: port, + Path: dirpath, + Swap: swap.DefaultSwapParams(contract, prvKey), + PublicKey: pubkeyhex, + BzzKey: keyhex, + EnsRoot: toyNetEnsRoot, + } + data, err = ioutil.ReadFile(confpath) + if err != nil { + if !os.IsNotExist(err) { + return + } + // file does not exist + // write out config file + err = self.Save() + if err != nil { + err = fmt.Errorf("error writing config: %v", err) + } + return + } + // file exists, deserialise + err = json.Unmarshal(data, self) + if err != nil { + return nil, fmt.Errorf("unable to parse config: %v", err) + } + // check public key + if pubkeyhex != self.PublicKey { + return nil, fmt.Errorf("public key does not match the one in the config file %v != %v", pubkeyhex, self.PublicKey) + } + if keyhex != self.BzzKey { + return nil, fmt.Errorf("bzz key does not match the one in the config file %v != %v", keyhex, self.BzzKey) + } + self.Swap.SetKey(prvKey) + + if (self.EnsRoot == common.Address{}) { + self.EnsRoot = toyNetEnsRoot + } + + return +} + +func (self *Config) Save() error { + data, err := json.MarshalIndent(self, "", " ") + if err != nil { + return err + } + err = os.MkdirAll(self.Path, os.ModePerm) + if err != nil { + return err + } + confpath := filepath.Join(self.Path, "config.json") + return ioutil.WriteFile(confpath, data, os.ModePerm) +} diff --git a/swarm/api/config_test.go b/swarm/api/config_test.go new file mode 100644 index 000000000..874701119 --- /dev/null +++ b/swarm/api/config_test.go @@ -0,0 +1,124 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package api + +import ( + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" +) + +var ( + hexprvkey = "65138b2aa745041b372153550584587da326ab440576b2a1191dd95cee30039c" + defaultConfig = `{ + "ChunkDbPath": "` + filepath.Join("TMPDIR", "0d2f62485607cf38d9d795d93682a517661e513e", "chunks") + `", + "DbCapacity": 5000000, + "CacheCapacity": 5000, + "Radius": 0, + "Branches": 128, + "Hash": "SHA3", + "CallInterval": 3000000000, + "KadDbPath": "` + filepath.Join("TMPDIR", "0d2f62485607cf38d9d795d93682a517661e513e", "bzz-peers.json") + `", + "MaxProx": 8, + "ProxBinSize": 2, + "BucketSize": 4, + "PurgeInterval": 151200000000000, + "InitialRetryInterval": 42000000, + "MaxIdleInterval": 42000000000, + "ConnRetryExp": 2, + "Swap": { + "BuyAt": 20000000000, + "SellAt": 20000000000, + "PayAt": 100, + "DropAt": 10000, + "AutoCashInterval": 300000000000, + "AutoCashThreshold": 50000000000000, + "AutoDepositInterval": 300000000000, + "AutoDepositThreshold": 50000000000000, + "AutoDepositBuffer": 100000000000000, + "PublicKey": "0x045f5cfd26692e48d0017d380349bcf50982488bc11b5145f3ddf88b24924299048450542d43527fbe29a5cb32f38d62755393ac002e6bfdd71b8d7ba725ecd7a3", + "Contract": "0x0000000000000000000000000000000000000000", + "Beneficiary": "0x0d2f62485607cf38d9d795d93682a517661e513e" + }, + "RequestDbPath": "` + filepath.Join("TMPDIR", "0d2f62485607cf38d9d795d93682a517661e513e", "requests") + `", + "RequestDbBatchSize": 512, + "KeyBufferSize": 1024, + "SyncBatchSize": 128, + "SyncBufferSize": 128, + "SyncCacheSize": 1024, + "SyncPriorities": [ + 2, + 1, + 1, + 0, + 0 + ], + "SyncModes": [ + true, + true, + true, + true, + false + ], + "Path": "` + filepath.Join("TMPDIR", "0d2f62485607cf38d9d795d93682a517661e513e") + `", + "Port": "8500", + "PublicKey": "0x045f5cfd26692e48d0017d380349bcf50982488bc11b5145f3ddf88b24924299048450542d43527fbe29a5cb32f38d62755393ac002e6bfdd71b8d7ba725ecd7a3", + "BzzKey": "0xe861964402c0b78e2d44098329b8545726f215afa737d803714a4338552fcb81", + "EnsRoot": "0xd344889e0be3e9ef6c26b0f60ef66a32e83c1b69" +}` +) + +func TestConfigWriteRead(t *testing.T) { + tmp, err := ioutil.TempDir(os.TempDir(), "bzz-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmp) + + prvkey := crypto.ToECDSA(common.Hex2Bytes(hexprvkey)) + orig, err := NewConfig(tmp, common.Address{}, prvkey) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + account := crypto.PubkeyToAddress(prvkey.PublicKey) + dirpath := filepath.Join(tmp, common.Bytes2Hex(account.Bytes())) + confpath := filepath.Join(dirpath, "config.json") + data, err := ioutil.ReadFile(confpath) + if err != nil { + t.Fatalf("default config file cannot be read: %v", err) + } + exp := strings.Replace(defaultConfig, "TMPDIR", tmp, -1) + exp = strings.Replace(exp, "\\", "\\\\", -1) + + if string(data) != exp { + t.Fatalf("default config mismatch:\nexpected: %v\ngot: %v", exp, string(data)) + } + + conf, err := NewConfig(tmp, common.Address{}, prvkey) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if conf.Swap.Beneficiary.Hex() != orig.Swap.Beneficiary.Hex() { + t.Fatalf("expected beneficiary from loaded config %v to match original %v", conf.Swap.Beneficiary.Hex(), orig.Swap.Beneficiary.Hex()) + } + +} diff --git a/swarm/api/filesystem.go b/swarm/api/filesystem.go new file mode 100644 index 000000000..428f3e3ac --- /dev/null +++ b/swarm/api/filesystem.go @@ -0,0 +1,283 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package api + +import ( + "bufio" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/logger" + "github.com/ethereum/go-ethereum/logger/glog" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +const maxParallelFiles = 5 + +type FileSystem struct { + api *Api +} + +func NewFileSystem(api *Api) *FileSystem { + return &FileSystem{api} +} + +// Upload replicates a local directory as a manifest file and uploads it +// using dpa store +// TODO: localpath should point to a manifest +func (self *FileSystem) Upload(lpath, index string) (string, error) { + var list []*manifestTrieEntry + localpath, err := filepath.Abs(filepath.Clean(lpath)) + if err != nil { + return "", err + } + + f, err := os.Open(localpath) + if err != nil { + return "", err + } + stat, err := f.Stat() + if err != nil { + return "", err + } + + var start int + if stat.IsDir() { + start = len(localpath) + glog.V(logger.Debug).Infof("uploading '%s'", localpath) + err = filepath.Walk(localpath, func(path string, info os.FileInfo, err error) error { + if (err == nil) && !info.IsDir() { + //fmt.Printf("lp %s path %s\n", localpath, path) + if len(path) <= start { + return fmt.Errorf("Path is too short") + } + if path[:start] != localpath { + return fmt.Errorf("Path prefix of '%s' does not match localpath '%s'", path, localpath) + } + entry := &manifestTrieEntry{ + Path: filepath.ToSlash(path), + } + list = append(list, entry) + } + return err + }) + if err != nil { + return "", err + } + } else { + dir := filepath.Dir(localpath) + start = len(dir) + if len(localpath) <= start { + return "", fmt.Errorf("Path is too short") + } + if localpath[:start] != dir { + return "", fmt.Errorf("Path prefix of '%s' does not match dir '%s'", localpath, dir) + } + entry := &manifestTrieEntry{ + Path: filepath.ToSlash(localpath), + } + list = append(list, entry) + } + + cnt := len(list) + errors := make([]error, cnt) + done := make(chan bool, maxParallelFiles) + dcnt := 0 + awg := &sync.WaitGroup{} + + for i, entry := range list { + if i >= dcnt+maxParallelFiles { + <-done + dcnt++ + } + awg.Add(1) + go func(i int, entry *manifestTrieEntry, done chan bool) { + f, err := os.Open(entry.Path) + if err == nil { + stat, _ := f.Stat() + var hash storage.Key + wg := &sync.WaitGroup{} + hash, err = self.api.dpa.Store(f, stat.Size(), wg, nil) + if hash != nil { + list[i].Hash = hash.String() + } + wg.Wait() + awg.Done() + if err == nil { + first512 := make([]byte, 512) + fread, _ := f.ReadAt(first512, 0) + if fread > 0 { + mimeType := http.DetectContentType(first512[:fread]) + if filepath.Ext(entry.Path) == ".css" { + mimeType = "text/css" + } + list[i].ContentType = mimeType + } + } + f.Close() + } + errors[i] = err + done <- true + }(i, entry, done) + } + for dcnt < cnt { + <-done + dcnt++ + } + + trie := &manifestTrie{ + dpa: self.api.dpa, + } + quitC := make(chan bool) + for i, entry := range list { + if errors[i] != nil { + return "", errors[i] + } + entry.Path = RegularSlashes(entry.Path[start:]) + if entry.Path == index { + ientry := &manifestTrieEntry{ + Path: "", + Hash: entry.Hash, + ContentType: entry.ContentType, + } + trie.addEntry(ientry, quitC) + } + trie.addEntry(entry, quitC) + } + + err2 := trie.recalcAndStore() + var hs string + if err2 == nil { + hs = trie.hash.String() + } + awg.Wait() + return hs, err2 +} + +// Download replicates the manifest path structure on the local filesystem +// under localpath +func (self *FileSystem) Download(bzzpath, localpath string) error { + lpath, err := filepath.Abs(filepath.Clean(localpath)) + if err != nil { + return err + } + err = os.MkdirAll(lpath, os.ModePerm) + if err != nil { + return err + } + + //resolving host and port + key, _, path, err := self.api.parseAndResolve(bzzpath, true) + if err != nil { + return err + } + + if len(path) > 0 { + path += "/" + } + + quitC := make(chan bool) + trie, err := loadManifest(self.api.dpa, key, quitC) + if err != nil { + glog.V(logger.Warn).Infof("fs.Download: loadManifestTrie error: %v", err) + return err + } + + type downloadListEntry struct { + key storage.Key + path string + } + + var list []*downloadListEntry + var mde error + + prevPath := lpath + err = trie.listWithPrefix(path, quitC, func(entry *manifestTrieEntry, suffix string) { + glog.V(logger.Detail).Infof("fs.Download: %#v", entry) + + key = common.Hex2Bytes(entry.Hash) + path := lpath + "/" + suffix + dir := filepath.Dir(path) + if dir != prevPath { + mde = os.MkdirAll(dir, os.ModePerm) + prevPath = dir + } + if (mde == nil) && (path != dir+"/") { + list = append(list, &downloadListEntry{key: key, path: path}) + } + }) + if err != nil { + return err + } + + wg := sync.WaitGroup{} + errC := make(chan error) + done := make(chan bool, maxParallelFiles) + for i, entry := range list { + select { + case done <- true: + wg.Add(1) + case <-quitC: + return fmt.Errorf("aborted") + } + go func(i int, entry *downloadListEntry) { + defer wg.Done() + f, err := os.Create(entry.path) // TODO: path separators + if err == nil { + + reader := self.api.dpa.Retrieve(entry.key) + writer := bufio.NewWriter(f) + size, err := reader.Size(quitC) + if err == nil { + _, err = io.CopyN(writer, reader, size) // TODO: handle errors + err2 := writer.Flush() + if err == nil { + err = err2 + } + err2 = f.Close() + if err == nil { + err = err2 + } + } + } + if err != nil { + select { + case errC <- err: + case <-quitC: + } + return + } + <-done + }(i, entry) + } + go func() { + wg.Wait() + close(errC) + }() + select { + case err = <-errC: + return err + case <-quitC: + return fmt.Errorf("aborted") + } + +} diff --git a/swarm/api/filesystem_test.go b/swarm/api/filesystem_test.go new file mode 100644 index 000000000..f6657aede --- /dev/null +++ b/swarm/api/filesystem_test.go @@ -0,0 +1,187 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package api + +import ( + "bytes" + "io/ioutil" + "os" + "path/filepath" + "sync" + "testing" +) + +var testDownloadDir, _ = ioutil.TempDir(os.TempDir(), "bzz-test") + +func testFileSystem(t *testing.T, f func(*FileSystem)) { + testApi(t, func(api *Api) { + f(NewFileSystem(api)) + }) +} + +func readPath(t *testing.T, parts ...string) string { + file := filepath.Join(parts...) + content, err := ioutil.ReadFile(file) + + if err != nil { + t.Fatalf("unexpected error reading '%v': %v", file, err) + } + return string(content) +} + +func TestApiDirUpload0(t *testing.T) { + testFileSystem(t, func(fs *FileSystem) { + api := fs.api + bzzhash, err := fs.Upload(filepath.Join("testdata", "test0"), "") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + content := readPath(t, "testdata", "test0", "index.html") + resp := testGet(t, api, bzzhash+"/index.html") + exp := expResponse(content, "text/html; charset=utf-8", 0) + checkResponse(t, resp, exp) + + content = readPath(t, "testdata", "test0", "index.css") + resp = testGet(t, api, bzzhash+"/index.css") + exp = expResponse(content, "text/css", 0) + checkResponse(t, resp, exp) + + _, _, _, err = api.Get(bzzhash, true) + if err == nil { + t.Fatalf("expected error: %v", err) + } + + downloadDir := filepath.Join(testDownloadDir, "test0") + defer os.RemoveAll(downloadDir) + err = fs.Download(bzzhash, downloadDir) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + newbzzhash, err := fs.Upload(downloadDir, "") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if bzzhash != newbzzhash { + t.Fatalf("download %v reuploaded has incorrect hash, expected %v, got %v", downloadDir, bzzhash, newbzzhash) + } + }) +} + +func TestApiDirUploadModify(t *testing.T) { + testFileSystem(t, func(fs *FileSystem) { + api := fs.api + bzzhash, err := fs.Upload(filepath.Join("testdata", "test0"), "") + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + + bzzhash, err = api.Modify(bzzhash+"/index.html", "", "", true) + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + index, err := ioutil.ReadFile(filepath.Join("testdata", "test0", "index.html")) + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + wg := &sync.WaitGroup{} + hash, err := api.Store(bytes.NewReader(index), int64(len(index)), wg) + wg.Wait() + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + bzzhash, err = api.Modify(bzzhash+"/index2.html", hash.Hex(), "text/html; charset=utf-8", true) + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + bzzhash, err = api.Modify(bzzhash+"/img/logo.png", hash.Hex(), "text/html; charset=utf-8", true) + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + + content := readPath(t, "testdata", "test0", "index.html") + resp := testGet(t, api, bzzhash+"/index2.html") + exp := expResponse(content, "text/html; charset=utf-8", 0) + checkResponse(t, resp, exp) + + resp = testGet(t, api, bzzhash+"/img/logo.png") + exp = expResponse(content, "text/html; charset=utf-8", 0) + checkResponse(t, resp, exp) + + content = readPath(t, "testdata", "test0", "index.css") + resp = testGet(t, api, bzzhash+"/index.css") + exp = expResponse(content, "text/css", 0) + + _, _, _, err = api.Get(bzzhash, true) + if err == nil { + t.Errorf("expected error: %v", err) + } + }) +} + +func TestApiDirUploadWithRootFile(t *testing.T) { + testFileSystem(t, func(fs *FileSystem) { + api := fs.api + bzzhash, err := fs.Upload(filepath.Join("testdata", "test0"), "index.html") + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + + content := readPath(t, "testdata", "test0", "index.html") + resp := testGet(t, api, bzzhash) + exp := expResponse(content, "text/html; charset=utf-8", 0) + checkResponse(t, resp, exp) + }) +} + +func TestApiFileUpload(t *testing.T) { + testFileSystem(t, func(fs *FileSystem) { + api := fs.api + bzzhash, err := fs.Upload(filepath.Join("testdata", "test0", "index.html"), "") + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + + content := readPath(t, "testdata", "test0", "index.html") + resp := testGet(t, api, bzzhash+"/index.html") + exp := expResponse(content, "text/html; charset=utf-8", 0) + checkResponse(t, resp, exp) + }) +} + +func TestApiFileUploadWithRootFile(t *testing.T) { + testFileSystem(t, func(fs *FileSystem) { + api := fs.api + bzzhash, err := fs.Upload(filepath.Join("testdata", "test0", "index.html"), "index.html") + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + + content := readPath(t, "testdata", "test0", "index.html") + resp := testGet(t, api, bzzhash) + exp := expResponse(content, "text/html; charset=utf-8", 0) + checkResponse(t, resp, exp) + }) +} diff --git a/swarm/api/http/roundtripper.go b/swarm/api/http/roundtripper.go new file mode 100644 index 000000000..a3a644b73 --- /dev/null +++ b/swarm/api/http/roundtripper.go @@ -0,0 +1,69 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package http + +import ( + "fmt" + "net/http" + + "github.com/ethereum/go-ethereum/logger" + "github.com/ethereum/go-ethereum/logger/glog" +) + +/* +http roundtripper to register for bzz url scheme +see https://github.com/ethereum/go-ethereum/issues/2040 +Usage: + +import ( + "github.com/ethereum/go-ethereum/common/httpclient" + "github.com/ethereum/go-ethereum/swarm/api/http" +) +client := httpclient.New() +// for (private) swarm proxy running locally +client.RegisterScheme("bzz", &http.RoundTripper{Port: port}) +client.RegisterScheme("bzzi", &http.RoundTripper{Port: port}) +client.RegisterScheme("bzzr", &http.RoundTripper{Port: port}) + +The port you give the Roundtripper is the port the swarm proxy is listening on. +If Host is left empty, localhost is assumed. + +Using a public gateway, the above few lines gives you the leanest +bzz-scheme aware read-only http client. You really only ever need this +if you need go-native swarm access to bzz addresses, e.g., +github.com/ethereum/go-ethereum/common/natspec + +*/ + +type RoundTripper struct { + Host string + Port string +} + +func (self *RoundTripper) RoundTrip(req *http.Request) (resp *http.Response, err error) { + host := self.Host + if len(host) == 0 { + host = "localhost" + } + url := fmt.Sprintf("http://%s:%s/%s:/%s/%s", host, self.Port, req.Proto, req.URL.Host, req.URL.Path) + glog.V(logger.Info).Infof("roundtripper: proxying request '%s' to '%s'", req.RequestURI, url) + reqProxy, err := http.NewRequest(req.Method, url, req.Body) + if err != nil { + return nil, err + } + return http.DefaultClient.Do(reqProxy) +} diff --git a/swarm/api/http/roundtripper_test.go b/swarm/api/http/roundtripper_test.go new file mode 100644 index 000000000..9afad20ae --- /dev/null +++ b/swarm/api/http/roundtripper_test.go @@ -0,0 +1,68 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package http + +import ( + "io/ioutil" + "net/http" + "strings" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common/httpclient" +) + +const port = "3222" + +func TestRoundTripper(t *testing.T) { + serveMux := http.NewServeMux() + serveMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + if r.Method == "GET" { + w.Header().Set("Content-Type", "text/plain") + http.ServeContent(w, r, "", time.Unix(0, 0), strings.NewReader(r.RequestURI)) + } else { + http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed) + } + }) + go http.ListenAndServe(":"+port, serveMux) + + rt := &RoundTripper{Port: port} + client := httpclient.New("/") + client.RegisterProtocol("bzz", rt) + + resp, err := client.Client().Get("bzz://test.com/path") + if err != nil { + t.Errorf("expected no error, got %v", err) + return + } + + defer func() { + if resp != nil { + resp.Body.Close() + } + }() + + content, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Errorf("expected no error, got %v", err) + return + } + if string(content) != "/HTTP/1.1:/test.com/path" { + t.Errorf("incorrect response from http server: expected '%v', got '%v'", "/HTTP/1.1:/test.com/path", string(content)) + } + +} diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go new file mode 100644 index 000000000..a35672687 --- /dev/null +++ b/swarm/api/http/server.go @@ -0,0 +1,286 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +/* +A simple http server interface to Swarm +*/ +package http + +import ( + "bytes" + "io" + "net/http" + "regexp" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/logger" + "github.com/ethereum/go-ethereum/logger/glog" + "github.com/ethereum/go-ethereum/swarm/api" +) + +const ( + rawType = "application/octet-stream" +) + +var ( + // accepted protocols: bzz (traditional), bzzi (immutable) and bzzr (raw) + bzzPrefix = regexp.MustCompile("^/+bzz[ir]?:/+") + trailingSlashes = regexp.MustCompile("/+$") + rootDocumentUri = regexp.MustCompile("^/+bzz[i]?:/+[^/]+$") + // forever = func() time.Time { return time.Unix(0, 0) } + forever = time.Now +) + +type sequentialReader struct { + reader io.Reader + pos int64 + ahead map[int64](chan bool) + lock sync.Mutex +} + +// browser API for registering bzz url scheme handlers: +// https://developer.mozilla.org/en/docs/Web-based_protocol_handlers +// electron (chromium) api for registering bzz url scheme handlers: +// https://github.com/atom/electron/blob/master/docs/api/protocol.md + +// starts up http server +func StartHttpServer(api *api.Api, port string) { + serveMux := http.NewServeMux() + serveMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + handler(w, r, api) + }) + go http.ListenAndServe(":"+port, serveMux) + glog.V(logger.Info).Infof("Swarm HTTP proxy started on localhost:%s", port) +} + +func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { + requestURL := r.URL + // This is wrong + // if requestURL.Host == "" { + // var err error + // requestURL, err = url.Parse(r.Referer() + requestURL.String()) + // if err != nil { + // http.Error(w, err.Error(), http.StatusBadRequest) + // return + // } + // } + glog.V(logger.Debug).Infof("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, requestURL.Host, requestURL.Path, r.Referer(), r.Header.Get("Accept")) + uri := requestURL.Path + var raw, nameresolver bool + var proto string + + // HTTP-based URL protocol handler + glog.V(logger.Debug).Infof("BZZ request URI: '%s'", uri) + + path := bzzPrefix.ReplaceAllStringFunc(uri, func(p string) string { + proto = p + return "" + }) + + // protocol identification (ugly) + if proto == "" { + if glog.V(logger.Error) { + glog.Errorf( + "[BZZ] Swarm: Protocol error in request `%s`.", + uri, + ) + http.Error(w, "BZZ protocol error", http.StatusBadRequest) + return + } + } + if len(proto) > 4 { + raw = proto[1:5] == "bzzr" + nameresolver = proto[1:5] != "bzzi" + } + + glog.V(logger.Debug).Infof( + "[BZZ] Swarm: %s request over protocol %s '%s' received.", + r.Method, proto, path, + ) + + switch { + case r.Method == "POST" || r.Method == "PUT": + key, err := a.Store(r.Body, r.ContentLength, nil) + if err == nil { + glog.V(logger.Debug).Infof("Content for %v stored", key.Log()) + } else { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if r.Method == "POST" { + if raw { + w.Header().Set("Content-Type", "text/plain") + http.ServeContent(w, r, "", time.Now(), bytes.NewReader([]byte(common.Bytes2Hex(key)))) + } else { + http.Error(w, "No POST to "+uri+" allowed.", http.StatusBadRequest) + return + } + } else { + // PUT + if raw { + http.Error(w, "No PUT to /raw allowed.", http.StatusBadRequest) + return + } else { + path = api.RegularSlashes(path) + mime := r.Header.Get("Content-Type") + // TODO proper root hash separation + glog.V(logger.Debug).Infof("Modify '%s' to store %v as '%s'.", path, key.Log(), mime) + newKey, err := a.Modify(path, common.Bytes2Hex(key), mime, nameresolver) + if err == nil { + glog.V(logger.Debug).Infof("Swarm replaced manifest by '%s'", newKey) + w.Header().Set("Content-Type", "text/plain") + http.ServeContent(w, r, "", time.Now(), bytes.NewReader([]byte(newKey))) + } else { + http.Error(w, "PUT to "+path+"failed.", http.StatusBadRequest) + return + } + } + } + case r.Method == "DELETE": + if raw { + http.Error(w, "No DELETE to /raw allowed.", http.StatusBadRequest) + return + } else { + path = api.RegularSlashes(path) + glog.V(logger.Debug).Infof("Delete '%s'.", path) + newKey, err := a.Modify(path, "", "", nameresolver) + if err == nil { + glog.V(logger.Debug).Infof("Swarm replaced manifest by '%s'", newKey) + w.Header().Set("Content-Type", "text/plain") + http.ServeContent(w, r, "", time.Now(), bytes.NewReader([]byte(newKey))) + } else { + http.Error(w, "DELETE to "+path+"failed.", http.StatusBadRequest) + return + } + } + case r.Method == "GET" || r.Method == "HEAD": + path = trailingSlashes.ReplaceAllString(path, "") + if raw { + // resolving host + key, err := a.Resolve(path, nameresolver) + if err != nil { + glog.V(logger.Error).Infof("%v", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // retrieving content + reader := a.Retrieve(key) + quitC := make(chan bool) + size, err := reader.Size(quitC) + glog.V(logger.Debug).Infof("Reading %d bytes.", size) + + // setting mime type + qv := requestURL.Query() + mimeType := qv.Get("content_type") + if mimeType == "" { + mimeType = rawType + } + + w.Header().Set("Content-Type", mimeType) + http.ServeContent(w, r, uri, forever(), reader) + glog.V(logger.Debug).Infof("Serve raw content '%s' (%d bytes) as '%s'", uri, size, mimeType) + + // retrieve path via manifest + } else { + glog.V(logger.Debug).Infof("Structured GET request '%s' received.", uri) + // add trailing slash, if missing + if rootDocumentUri.MatchString(uri) { + http.Redirect(w, r, path+"/", http.StatusFound) + return + } + reader, mimeType, status, err := a.Get(path, nameresolver) + if err != nil { + if _, ok := err.(api.ErrResolve); ok { + glog.V(logger.Debug).Infof("%v", err) + status = http.StatusBadRequest + } else { + glog.V(logger.Debug).Infof("error retrieving '%s': %v", uri, err) + status = http.StatusNotFound + } + http.Error(w, err.Error(), status) + return + } + // set mime type and status headers + w.Header().Set("Content-Type", mimeType) + if status > 0 { + w.WriteHeader(status) + } else { + status = 200 + } + quitC := make(chan bool) + size, err := reader.Size(quitC) + glog.V(logger.Debug).Infof("Served '%s' (%d bytes) as '%s' (status code: %v)", uri, size, mimeType, status) + + http.ServeContent(w, r, path, forever(), reader) + + } + default: + http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed) + } +} + +func (self *sequentialReader) ReadAt(target []byte, off int64) (n int, err error) { + self.lock.Lock() + // assert self.pos <= off + if self.pos > off { + glog.V(logger.Error).Infof("non-sequential read attempted from sequentialReader; %d > %d", + self.pos, off) + panic("Non-sequential read attempt") + } + if self.pos != off { + glog.V(logger.Debug).Infof("deferred read in POST at position %d, offset %d.", + self.pos, off) + wait := make(chan bool) + self.ahead[off] = wait + self.lock.Unlock() + if <-wait { + // failed read behind + n = 0 + err = io.ErrUnexpectedEOF + return + } + self.lock.Lock() + } + localPos := 0 + for localPos < len(target) { + n, err = self.reader.Read(target[localPos:]) + localPos += n + glog.V(logger.Debug).Infof("Read %d bytes into buffer size %d from POST, error %v.", + n, len(target), err) + if err != nil { + glog.V(logger.Debug).Infof("POST stream's reading terminated with %v.", err) + for i := range self.ahead { + self.ahead[i] <- true + delete(self.ahead, i) + } + self.lock.Unlock() + return localPos, err + } + self.pos += int64(n) + } + wait := self.ahead[self.pos] + if wait != nil { + glog.V(logger.Debug).Infof("deferred read in POST at position %d triggered.", + self.pos) + delete(self.ahead, self.pos) + close(wait) + } + self.lock.Unlock() + return localPos, err +} diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go new file mode 100644 index 000000000..a289c01f9 --- /dev/null +++ b/swarm/api/manifest.go @@ -0,0 +1,336 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package api + +import ( + "bytes" + "encoding/json" + "fmt" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/logger" + "github.com/ethereum/go-ethereum/logger/glog" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +const ( + manifestType = "application/bzz-manifest+json" +) + +type manifestTrie struct { + dpa *storage.DPA + entries [257]*manifestTrieEntry // indexed by first character of path, entries[256] is the empty path entry + hash storage.Key // if hash != nil, it is stored +} + +type manifestJSON struct { + Entries []*manifestTrieEntry `json:"entries"` +} + +type manifestTrieEntry struct { + Path string `json:"path"` + Hash string `json:"hash"` // for manifest content type, empty until subtrie is evaluated + ContentType string `json:"contentType"` + Status int `json:"status"` + subtrie *manifestTrie +} + +func loadManifest(dpa *storage.DPA, hash storage.Key, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand + + glog.V(logger.Detail).Infof("manifest lookup key: '%v'.", hash.Log()) + // retrieve manifest via DPA + manifestReader := dpa.Retrieve(hash) + return readManifest(manifestReader, hash, dpa, quitC) +} + +func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dpa *storage.DPA, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand + + // TODO check size for oversized manifests + size, err := manifestReader.Size(quitC) + manifestData := make([]byte, size) + read, err := manifestReader.Read(manifestData) + if int64(read) < size { + glog.V(logger.Detail).Infof("Manifest %v not found.", hash.Log()) + if err == nil { + err = fmt.Errorf("Manifest retrieval cut short: read %v, expect %v", read, size) + } + return + } + + glog.V(logger.Detail).Infof("Manifest %v retrieved", hash.Log()) + man := manifestJSON{} + err = json.Unmarshal(manifestData, &man) + if err != nil { + err = fmt.Errorf("Manifest %v is malformed: %v", hash.Log(), err) + glog.V(logger.Detail).Infof("%v", err) + return + } + + glog.V(logger.Detail).Infof("Manifest %v has %d entries.", hash.Log(), len(man.Entries)) + + trie = &manifestTrie{ + dpa: dpa, + } + for _, entry := range man.Entries { + trie.addEntry(entry, quitC) + } + return +} + +func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) { + self.hash = nil // trie modified, hash needs to be re-calculated on demand + + if len(entry.Path) == 0 { + self.entries[256] = entry + return + } + + b := byte(entry.Path[0]) + if (self.entries[b] == nil) || (self.entries[b].Path == entry.Path) { + self.entries[b] = entry + return + } + + oldentry := self.entries[b] + cpl := 0 + for (len(entry.Path) > cpl) && (len(oldentry.Path) > cpl) && (entry.Path[cpl] == oldentry.Path[cpl]) { + cpl++ + } + + if (oldentry.ContentType == manifestType) && (cpl == len(oldentry.Path)) { + if self.loadSubTrie(oldentry, quitC) != nil { + return + } + entry.Path = entry.Path[cpl:] + oldentry.subtrie.addEntry(entry, quitC) + oldentry.Hash = "" + return + } + + commonPrefix := entry.Path[:cpl] + + subtrie := &manifestTrie{ + dpa: self.dpa, + } + entry.Path = entry.Path[cpl:] + oldentry.Path = oldentry.Path[cpl:] + subtrie.addEntry(entry, quitC) + subtrie.addEntry(oldentry, quitC) + + self.entries[b] = &manifestTrieEntry{ + Path: commonPrefix, + Hash: "", + ContentType: manifestType, + subtrie: subtrie, + } +} + +func (self *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) { + for _, e := range self.entries { + if e != nil { + cnt++ + entry = e + } + } + return +} + +func (self *manifestTrie) deleteEntry(path string, quitC chan bool) { + self.hash = nil // trie modified, hash needs to be re-calculated on demand + + if len(path) == 0 { + self.entries[256] = nil + return + } + + b := byte(path[0]) + entry := self.entries[b] + if entry == nil { + return + } + if entry.Path == path { + self.entries[b] = nil + return + } + + epl := len(entry.Path) + if (entry.ContentType == manifestType) && (len(path) >= epl) && (path[:epl] == entry.Path) { + if self.loadSubTrie(entry, quitC) != nil { + return + } + entry.subtrie.deleteEntry(path[epl:], quitC) + entry.Hash = "" + // remove subtree if it has less than 2 elements + cnt, lastentry := entry.subtrie.getCountLast() + if cnt < 2 { + if lastentry != nil { + lastentry.Path = entry.Path + lastentry.Path + } + self.entries[b] = lastentry + } + } +} + +func (self *manifestTrie) recalcAndStore() error { + if self.hash != nil { + return nil + } + + var buffer bytes.Buffer + buffer.WriteString(`{"entries":[`) + + list := &manifestJSON{} + for _, entry := range self.entries { + if entry != nil { + if entry.Hash == "" { // TODO: paralellize + err := entry.subtrie.recalcAndStore() + if err != nil { + return err + } + entry.Hash = entry.subtrie.hash.String() + } + list.Entries = append(list.Entries, entry) + } + } + + manifest, err := json.Marshal(list) + if err != nil { + return err + } + + sr := bytes.NewReader(manifest) + wg := &sync.WaitGroup{} + key, err2 := self.dpa.Store(sr, int64(len(manifest)), wg, nil) + wg.Wait() + self.hash = key + return err2 +} + +func (self *manifestTrie) loadSubTrie(entry *manifestTrieEntry, quitC chan bool) (err error) { + if entry.subtrie == nil { + hash := common.Hex2Bytes(entry.Hash) + entry.subtrie, err = loadManifest(self.dpa, hash, quitC) + entry.Hash = "" // might not match, should be recalculated + } + return +} + +func (self *manifestTrie) listWithPrefixInt(prefix, rp string, quitC chan bool, cb func(entry *manifestTrieEntry, suffix string)) error { + plen := len(prefix) + var start, stop int + if plen == 0 { + start = 0 + stop = 256 + } else { + start = int(prefix[0]) + stop = start + } + + for i := start; i <= stop; i++ { + select { + case <-quitC: + return fmt.Errorf("aborted") + default: + } + entry := self.entries[i] + if entry != nil { + epl := len(entry.Path) + if entry.ContentType == manifestType { + l := plen + if epl < l { + l = epl + } + if prefix[:l] == entry.Path[:l] { + err := self.loadSubTrie(entry, quitC) + if err != nil { + return err + } + err = entry.subtrie.listWithPrefixInt(prefix[l:], rp+entry.Path[l:], quitC, cb) + if err != nil { + return err + } + } + } else { + if (epl >= plen) && (prefix == entry.Path[:plen]) { + cb(entry, rp+entry.Path[plen:]) + } + } + } + } + return nil +} + +func (self *manifestTrie) listWithPrefix(prefix string, quitC chan bool, cb func(entry *manifestTrieEntry, suffix string)) (err error) { + return self.listWithPrefixInt(prefix, "", quitC, cb) +} + +func (self *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *manifestTrieEntry, pos int) { + + glog.V(logger.Detail).Infof("findPrefixOf(%s)", path) + + if len(path) == 0 { + return self.entries[256], 0 + } + + b := byte(path[0]) + entry = self.entries[b] + if entry == nil { + return self.entries[256], 0 + } + epl := len(entry.Path) + glog.V(logger.Detail).Infof("path = %v entry.Path = %v epl = %v", path, entry.Path, epl) + if (len(path) >= epl) && (path[:epl] == entry.Path) { + glog.V(logger.Detail).Infof("entry.ContentType = %v", entry.ContentType) + if entry.ContentType == manifestType { + if self.loadSubTrie(entry, quitC) != nil { + return nil, 0 + } + entry, pos = entry.subtrie.findPrefixOf(path[epl:], quitC) + if entry != nil { + pos += epl + } + } else { + pos = epl + } + } else { + entry = nil + } + return +} + +// file system manifest always contains regularized paths +// no leading or trailing slashes, only single slashes inside +func RegularSlashes(path string) (res string) { + for i := 0; i < len(path); i++ { + if (path[i] != '/') || ((i > 0) && (path[i-1] != '/')) { + res = res + path[i:i+1] + } + } + if (len(res) > 0) && (res[len(res)-1] == '/') { + res = res[:len(res)-1] + } + return +} + +func (self *manifestTrie) getEntry(spath string) (entry *manifestTrieEntry, fullpath string) { + path := RegularSlashes(spath) + var pos int + quitC := make(chan bool) + entry, pos = self.findPrefixOf(path, quitC) + return entry, path[:pos] +} diff --git a/swarm/api/manifest_test.go b/swarm/api/manifest_test.go new file mode 100644 index 000000000..20b8117c6 --- /dev/null +++ b/swarm/api/manifest_test.go @@ -0,0 +1,80 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package api + +import ( + // "encoding/json" + "fmt" + "io" + "strings" + "testing" + + "github.com/ethereum/go-ethereum/swarm/storage" +) + +func manifest(paths ...string) (manifestReader storage.LazySectionReader) { + var entries []string + for _, path := range paths { + entry := fmt.Sprintf(`{"path":"%s"}`, path) + entries = append(entries, entry) + } + manifest := fmt.Sprintf(`{"entries":[%s]}`, strings.Join(entries, ",")) + return &storage.LazyTestSectionReader{ + SectionReader: io.NewSectionReader(strings.NewReader(manifest), 0, int64(len(manifest))), + } +} + +func testGetEntry(t *testing.T, path, match string, paths ...string) *manifestTrie { + quitC := make(chan bool) + trie, err := readManifest(manifest(paths...), nil, nil, quitC) + if err != nil { + t.Errorf("unexpected error making manifest: %v", err) + } + checkEntry(t, path, match, trie) + return trie +} + +func checkEntry(t *testing.T, path, match string, trie *manifestTrie) { + entry, fullpath := trie.getEntry(path) + if match == "-" && entry != nil { + t.Errorf("expected no match for '%s', got '%s'", path, fullpath) + } else if entry == nil { + if match != "-" { + t.Errorf("expected entry '%s' to match '%s', got no match", match, path) + } + } else if fullpath != match { + t.Errorf("incorrect entry retrieved for '%s'. expected path '%v', got '%s'", path, match, fullpath) + } +} + +func TestGetEntry(t *testing.T) { + // file system manifest always contains regularized paths + testGetEntry(t, "a", "a", "a") + testGetEntry(t, "b", "-", "a") + testGetEntry(t, "/a//", "a", "a") + // fallback + testGetEntry(t, "/a", "", "") + testGetEntry(t, "/a/b", "a/b", "a/b") + // longest/deepest math + testGetEntry(t, "a/b", "-", "a", "a/ba", "a/b/c") + testGetEntry(t, "a/b", "a/b", "a", "a/b", "a/bb", "a/b/c") + testGetEntry(t, "//a//b//", "a/b", "a", "a/b", "a/bb", "a/b/c") +} + +func TestDeleteEntry(t *testing.T) { + +} diff --git a/swarm/api/storage.go b/swarm/api/storage.go new file mode 100644 index 000000000..31b484675 --- /dev/null +++ b/swarm/api/storage.go @@ -0,0 +1,70 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package api + +type Response struct { + MimeType string + Status int + Size int64 + // Content []byte + Content string +} + +// implements a service +type Storage struct { + api *Api +} + +func NewStorage(api *Api) *Storage { + return &Storage{api} +} + +// Put uploads the content to the swarm with a simple manifest speficying +// its content type +func (self *Storage) Put(content, contentType string) (string, error) { + return self.api.Put(content, contentType) +} + +// Get retrieves the content from bzzpath and reads the response in full +// It returns the Response object, which serialises containing the +// response body as the value of the Content field +// NOTE: if error is non-nil, sResponse may still have partial content +// the actual size of which is given in len(resp.Content), while the expected +// size is resp.Size +func (self *Storage) Get(bzzpath string) (*Response, error) { + reader, mimeType, status, err := self.api.Get(bzzpath, true) + if err != nil { + return nil, err + } + quitC := make(chan bool) + expsize, err := reader.Size(quitC) + if err != nil { + return nil, err + } + body := make([]byte, expsize) + size, err := reader.Read(body) + if int64(size) == expsize { + err = nil + } + return &Response{mimeType, status, expsize, string(body[:size])}, err +} + +// Modify(rootHash, path, contentHash, contentType) takes th e manifest trie rooted in rootHash, +// and merge on to it. creating an entry w conentType (mime) +func (self *Storage) Modify(rootHash, path, contentHash, contentType string) (newRootHash string, err error) { + return self.api.Modify(rootHash+"/"+path, contentHash, contentType, true) +} diff --git a/swarm/api/storage_test.go b/swarm/api/storage_test.go new file mode 100644 index 000000000..72caf52df --- /dev/null +++ b/swarm/api/storage_test.go @@ -0,0 +1,49 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package api + +import ( + "testing" +) + +func testStorage(t *testing.T, f func(*Storage)) { + testApi(t, func(api *Api) { + f(NewStorage(api)) + }) +} + +func TestStoragePutGet(t *testing.T) { + testStorage(t, func(api *Storage) { + content := "hello" + exp := expResponse(content, "text/plain", 0) + // exp := expResponse([]byte(content), "text/plain", 0) + bzzhash, err := api.Put(content, exp.MimeType) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + // to check put against the Api#Get + resp0 := testGet(t, api.api, bzzhash) + checkResponse(t, resp0, exp) + + // check storage#Get + resp, err := api.Get(bzzhash) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + checkResponse(t, &testResponse{nil, resp}, exp) + }) +} diff --git a/swarm/api/testapi.go b/swarm/api/testapi.go new file mode 100644 index 000000000..6631196c1 --- /dev/null +++ b/swarm/api/testapi.go @@ -0,0 +1,46 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package api + +import ( + "github.com/ethereum/go-ethereum/swarm/network" +) + +type Control struct { + api *Api + hive *network.Hive +} + +func NewControl(api *Api, hive *network.Hive) *Control { + return &Control{api, hive} +} + +func (self *Control) BlockNetworkRead(on bool) { + self.hive.BlockNetworkRead(on) +} + +func (self *Control) SyncEnabled(on bool) { + self.hive.SyncEnabled(on) +} + +func (self *Control) SwapEnabled(on bool) { + self.hive.SwapEnabled(on) +} + +func (self *Control) Hive() string { + return self.hive.String() +} diff --git a/swarm/api/testdata/test0/img/logo.png b/swarm/api/testdata/test0/img/logo.png new file mode 100644 index 000000000..e0fb15ab3 Binary files /dev/null and b/swarm/api/testdata/test0/img/logo.png differ diff --git a/swarm/api/testdata/test0/index.css b/swarm/api/testdata/test0/index.css new file mode 100644 index 000000000..67cb8d0ff --- /dev/null +++ b/swarm/api/testdata/test0/index.css @@ -0,0 +1,9 @@ +h1 { + color: black; + font-size: 12px; + background-color: orange; + border: 4px solid black; +} +body { + background-color: orange +} \ No newline at end of file diff --git a/swarm/api/testdata/test0/index.html b/swarm/api/testdata/test0/index.html new file mode 100644 index 000000000..321e910d7 --- /dev/null +++ b/swarm/api/testdata/test0/index.html @@ -0,0 +1,10 @@ + + + + + + +

Swarm Test

+ Ethereum logo + + \ No newline at end of file -- cgit v1.2.3