From 057af8c5c842714feff675faeab089e497ec1739 Mon Sep 17 00:00:00 2001 From: Janos Guljas
Date: Fri, 1 Dec 2017 10:32:14 +0100 Subject: swarm: add CLI --ens-endpoint flag (#14386) Implement a CLI flag that can be repeated to allow multiple ENS resolvers for different TLDs. --- swarm/api/api.go | 64 +++++++++++++++++++++++++++ swarm/api/api_test.go | 117 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 181 insertions(+) (limited to 'swarm/api') diff --git a/swarm/api/api.go b/swarm/api/api.go index 79de29a1c..d0c3754e0 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -17,6 +17,7 @@ package api import ( + "errors" "fmt" "io" "net/http" @@ -40,6 +41,69 @@ type Resolver interface { Resolve(string) (common.Hash, error) } +// errNoResolver is returned by MultiResolver.Resolve if no resolver +// can be found for the address. +var errNoResolver = errors.New("no resolver") + +// MultiResolver is used to resolve URL addresses based on their TLDs. +// Each TLD can have multiple resolvers, and the resoluton from the +// first one in the sequence will be returned. +type MultiResolver struct { + resolvers map[string][]Resolver +} + +// MultiResolverOption sets options for MultiResolver and is used as +// arguments for its constructor. +type MultiResolverOption func(*MultiResolver) + +// MultiResolverOptionWithResolver adds a Resolver to a list of resolvers +// for a specific TLD. If TLD is an empty string, the resolver will be added +// to the list of default resolver, the ones that will be used for resolution +// of addresses which do not have their TLD resolver specified. +func MultiResolverOptionWithResolver(r Resolver, tld string) MultiResolverOption { + return func(m *MultiResolver) { + if _, ok := m.resolvers[tld]; !ok { + m.resolvers[tld] = []Resolver{} + } + m.resolvers[tld] = append(m.resolvers[tld], r) + } +} + +// NewMultiResolver creates a new instance of MultiResolver. +func NewMultiResolver(opts ...MultiResolverOption) (m *MultiResolver) { + m = &MultiResolver{ + resolvers: map[string][]Resolver{}, + } + for _, o := range opts { + o(m) + } + return m +} + +// Resolve resolves address by choosing a Resolver by TLD. +// If there are more default Resolvers, or for a specific TLD, +// the Hash from the the first one which does not return error +// will be returned. +func (m MultiResolver) Resolve(addr string) (h common.Hash, err error) { + rs, _ := m.resolvers[""] + if i := strings.LastIndex(addr, "."); i >= 0 { + rstld, ok := m.resolvers[addr[i+1:]] + if ok { + rs = rstld + } + } + if rs == nil { + return h, errNoResolver + } + for _, r := range rs { + h, err = r.Resolve(addr) + if err == nil { + return + } + } + return +} + /* Api implements webserver/file system related content storage and retrieval on top of the dpa diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index f9caed27f..36cbdb83c 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -237,3 +237,120 @@ func TestAPIResolve(t *testing.T) { }) } } + +func TestMultiResolver(t *testing.T) { + doesntResolve := newTestResolver("") + + ethAddr := "swarm.eth" + ethHash := "0x2222222222222222222222222222222222222222222222222222222222222222" + ethResolve := newTestResolver(ethHash) + + testAddr := "swarm.test" + testHash := "0x1111111111111111111111111111111111111111111111111111111111111111" + testResolve := newTestResolver(testHash) + + tests := []struct { + desc string + r Resolver + addr string + result string + err error + }{ + { + desc: "No resolvers, returns error", + r: NewMultiResolver(), + err: errNoResolver, + }, + { + desc: "One default resolver, returns resolved address", + r: NewMultiResolver(MultiResolverOptionWithResolver(ethResolve, "")), + addr: ethAddr, + result: ethHash, + }, + { + desc: "Two default resolvers, returns resolved address", + r: NewMultiResolver( + MultiResolverOptionWithResolver(ethResolve, ""), + MultiResolverOptionWithResolver(ethResolve, ""), + ), + addr: ethAddr, + result: ethHash, + }, + { + desc: "Two default resolvers, first doesn't resolve, returns resolved address", + r: NewMultiResolver( + MultiResolverOptionWithResolver(doesntResolve, ""), + MultiResolverOptionWithResolver(ethResolve, ""), + ), + addr: ethAddr, + result: ethHash, + }, + { + desc: "Default resolver doesn't resolve, tld resolver resolve, returns resolved address", + r: NewMultiResolver( + MultiResolverOptionWithResolver(doesntResolve, ""), + MultiResolverOptionWithResolver(ethResolve, "eth"), + ), + addr: ethAddr, + result: ethHash, + }, + { + desc: "Three TLD resolvers, third resolves, returns resolved address", + r: NewMultiResolver( + MultiResolverOptionWithResolver(doesntResolve, "eth"), + MultiResolverOptionWithResolver(doesntResolve, "eth"), + MultiResolverOptionWithResolver(ethResolve, "eth"), + ), + addr: ethAddr, + result: ethHash, + }, + { + desc: "One TLD resolver doesn't resolve, returns error", + r: NewMultiResolver( + MultiResolverOptionWithResolver(doesntResolve, ""), + MultiResolverOptionWithResolver(ethResolve, "eth"), + ), + addr: ethAddr, + result: ethHash, + }, + { + desc: "One defautl and one TLD resolver, all doesn't resolve, returns error", + r: NewMultiResolver( + MultiResolverOptionWithResolver(doesntResolve, ""), + MultiResolverOptionWithResolver(doesntResolve, "eth"), + ), + addr: ethAddr, + result: ethHash, + err: errors.New(`DNS name not found: "swarm.eth"`), + }, + { + desc: "Two TLD resolvers, both resolve, returns resolved address", + r: NewMultiResolver( + MultiResolverOptionWithResolver(ethResolve, "eth"), + MultiResolverOptionWithResolver(testResolve, "test"), + ), + addr: testAddr, + result: testHash, + }, + } + for _, x := range tests { + t.Run(x.desc, func(t *testing.T) { + res, err := x.r.Resolve(x.addr) + if err == nil { + if x.err != nil { + t.Fatalf("expected error %q, got result %q", x.err, res.Hex()) + } + if res.Hex() != x.result { + t.Fatalf("expected result %q, got %q", x.result, res.Hex()) + } + } else { + if x.err == nil { + t.Fatalf("expected no error, got %q", err) + } + if err.Error() != x.err.Error() { + t.Fatalf("expected error %q, got %q", x.err, err) + } + } + }) + } +} -- cgit v1.2.3 From 34edbc8868b84e15d7102a1164c7af8ef4873882 Mon Sep 17 00:00:00 2001 From: Janos Guljas
Date: Mon, 4 Dec 2017 22:29:37 +0100 Subject: swarm/api: remove unneeded assignment in MultiResolverOptionWithResolver --- swarm/api/api.go | 3 --- 1 file changed, 3 deletions(-) (limited to 'swarm/api') diff --git a/swarm/api/api.go b/swarm/api/api.go index d0c3754e0..3682ab309 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -62,9 +62,6 @@ type MultiResolverOption func(*MultiResolver) // of addresses which do not have their TLD resolver specified. func MultiResolverOptionWithResolver(r Resolver, tld string) MultiResolverOption { return func(m *MultiResolver) { - if _, ok := m.resolvers[tld]; !ok { - m.resolvers[tld] = []Resolver{} - } m.resolvers[tld] = append(m.resolvers[tld], r) } } -- cgit v1.2.3 From a758b5cf7ac3111c053bc1275ca849f7aae080e1 Mon Sep 17 00:00:00 2001 From: Janos Guljas
Date: Mon, 4 Dec 2017 22:31:00 +0100 Subject: swarm/api: initialize map with make to comply with the convention --- swarm/api/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'swarm/api') diff --git a/swarm/api/api.go b/swarm/api/api.go index 3682ab309..d6e9d3732 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -69,7 +69,7 @@ func MultiResolverOptionWithResolver(r Resolver, tld string) MultiResolverOption // NewMultiResolver creates a new instance of MultiResolver. func NewMultiResolver(opts ...MultiResolverOption) (m *MultiResolver) { m = &MultiResolver{ - resolvers: map[string][]Resolver{}, + resolvers: make(map[string][]Resolver), } for _, o := range opts { o(m) -- cgit v1.2.3 From 3732c15faa83e60024ef46b3def0c105c6912c47 Mon Sep 17 00:00:00 2001 From: Janos Guljas
Date: Mon, 4 Dec 2017 22:32:33 +0100 Subject: swarm/api: remove unneeded blank assignement --- swarm/api/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'swarm/api') diff --git a/swarm/api/api.go b/swarm/api/api.go index d6e9d3732..8bd386643 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -82,7 +82,7 @@ func NewMultiResolver(opts ...MultiResolverOption) (m *MultiResolver) { // the Hash from the the first one which does not return error // will be returned. func (m MultiResolver) Resolve(addr string) (h common.Hash, err error) { - rs, _ := m.resolvers[""] + rs := m.resolvers[""] if i := strings.LastIndex(addr, "."); i >= 0 { rstld, ok := m.resolvers[addr[i+1:]] if ok { -- cgit v1.2.3 From 1dc19de5da64962a98a37bbc7b93a3895d2eb6e6 Mon Sep 17 00:00:00 2001 From: Janos Guljas
Date: Mon, 4 Dec 2017 22:56:52 +0100 Subject: swarm/api: use path.Ext instead strings.LastIndex in MultiResolver.Resolve --- swarm/api/api.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'swarm/api') diff --git a/swarm/api/api.go b/swarm/api/api.go index 8bd386643..2d9db022e 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -21,6 +21,7 @@ import ( "fmt" "io" "net/http" + "path" "regexp" "strings" "sync" @@ -83,8 +84,8 @@ func NewMultiResolver(opts ...MultiResolverOption) (m *MultiResolver) { // will be returned. func (m MultiResolver) Resolve(addr string) (h common.Hash, err error) { rs := m.resolvers[""] - if i := strings.LastIndex(addr, "."); i >= 0 { - rstld, ok := m.resolvers[addr[i+1:]] + if ext := path.Ext(addr); ext != "" { + rstld, ok := m.resolvers[ext[1:]] if ok { rs = rstld } -- cgit v1.2.3 From c0a4d9e1e64a09a19484c8c12e24505d9bacbd57 Mon Sep 17 00:00:00 2001 From: Janos Guljas
Date: Mon, 18 Dec 2017 16:22:39 +0100 Subject: cmd/swarm, swarm: disable ENS API by default Specifying ENS API CLI flag, env variable or configuration field is required for ENS resolving. Backward compatibility is preserved with --ens-api="" CLI flag value. --- swarm/api/config.go | 2 -- 1 file changed, 2 deletions(-) (limited to 'swarm/api') diff --git a/swarm/api/config.go b/swarm/api/config.go index 3f1a2a074..6b224140a 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -48,7 +48,6 @@ type Config struct { *network.SyncParams Contract common.Address EnsRoot common.Address - EnsDisabled bool EnsAPIs []string Path string ListenAddr string @@ -78,7 +77,6 @@ func NewDefaultConfig() (self *Config) { Path: node.DefaultDataDir(), EnsAPIs: nil, EnsRoot: ens.TestNetAddress, - EnsDisabled: false, NetworkId: network.NetworkId, SwapEnabled: false, SyncEnabled: true, -- cgit v1.2.3 From 0d6a735a72340130acd6b7e536dad5d8bee40d84 Mon Sep 17 00:00:00 2001 From: Janos Guljas
Date: Mon, 18 Dec 2017 23:07:48 +0100 Subject: swarm/api: implement NoResolverError with information about TLD MultiResolver needs to provide information about TLD that has no resolver configured for. --- swarm/api/api.go | 26 ++++++++++++++++++++------ swarm/api/api_test.go | 10 +++++++++- 2 files changed, 29 insertions(+), 7 deletions(-) (limited to 'swarm/api') diff --git a/swarm/api/api.go b/swarm/api/api.go index 2d9db022e..c143d8a3f 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -17,7 +17,6 @@ package api import ( - "errors" "fmt" "io" "net/http" @@ -42,9 +41,22 @@ type Resolver interface { Resolve(string) (common.Hash, error) } -// errNoResolver is returned by MultiResolver.Resolve if no resolver +// NoResolverError is returned by MultiResolver.Resolve if no resolver // can be found for the address. -var errNoResolver = errors.New("no resolver") +type NoResolverError struct { + TLD string +} + +func NewNoResolverError(tld string) *NoResolverError { + return &NoResolverError{TLD: tld} +} + +func (e *NoResolverError) Error() string { + if e.TLD == "" { + return "no ENS resolver" + } + return fmt.Sprintf("no ENS endpoint configured to resolve .%s TLD names", e.TLD) +} // MultiResolver is used to resolve URL addresses based on their TLDs. // Each TLD can have multiple resolvers, and the resoluton from the @@ -84,14 +96,16 @@ func NewMultiResolver(opts ...MultiResolverOption) (m *MultiResolver) { // will be returned. func (m MultiResolver) Resolve(addr string) (h common.Hash, err error) { rs := m.resolvers[""] - if ext := path.Ext(addr); ext != "" { - rstld, ok := m.resolvers[ext[1:]] + tld := path.Ext(addr) + if tld != "" { + tld = tld[1:] + rstld, ok := m.resolvers[tld] if ok { rs = rstld } } if rs == nil { - return h, errNoResolver + return h, NewNoResolverError(tld) } for _, r := range rs { h, err = r.Resolve(addr) diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index 36cbdb83c..dbe7c1dac 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -259,7 +259,7 @@ func TestMultiResolver(t *testing.T) { { desc: "No resolvers, returns error", r: NewMultiResolver(), - err: errNoResolver, + err: NewNoResolverError(""), }, { desc: "One default resolver, returns resolved address", @@ -332,6 +332,14 @@ func TestMultiResolver(t *testing.T) { addr: testAddr, result: testHash, }, + { + desc: "One TLD resolver, no default resolver, returns error for different TLD", + r: NewMultiResolver( + MultiResolverOptionWithResolver(ethResolve, "eth"), + ), + addr: testAddr, + err: NewNoResolverError("test"), + }, } for _, x := range tests { t.Run(x.desc, func(t *testing.T) { -- cgit v1.2.3 From b677a07d36c957c4221bae952189559ac0c70537 Mon Sep 17 00:00:00 2001 From: Lewis Marshall
Date: Fri, 23 Feb 2018 13:09:01 +0000 Subject: swarm/api/http: Fix using deprecated bzzr scheme (#16152) Without this, deprecated bzzr requests just return an empty response. Signed-off-by: Lewis Marshall
--- swarm/api/http/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'swarm/api') diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 74341899d..b89487af7 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -349,7 +349,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { } switch { - case r.uri.Raw(): + case r.uri.Raw() || r.uri.DeprecatedRaw(): // allow the request to overwrite the content type using a query // parameter contentType := "application/octet-stream" -- cgit v1.2.3 From dcca613a0b4c6ce56e52f4607cf71f4f1338db8f Mon Sep 17 00:00:00 2001 From: Anton Evangelatov
Date: Fri, 23 Feb 2018 14:19:59 +0100 Subject: swarm: initial instrumentation (#15969) * swarm: initial instrumentation with go-metrics * swarm: initialise metrics collection and add ResettingTimer to HTTP requests * swarm: update metrics flags names. remove redundant Timer. * swarm: rename method for periodically updating gauges * swarm: finalise metrics after feedback * swarm/network: always init kad metrics containers * swarm/network: off-by-one index in metrics containers * swarm, metrics: resolved conflicts --- swarm/api/api.go | 53 +++++++++++++++++++++++++++++++++++++ swarm/api/http/error.go | 9 +++++++ swarm/api/http/server.go | 69 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+) (limited to 'swarm/api') diff --git a/swarm/api/api.go b/swarm/api/api.go index fdf76d390..0cf12fdbe 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -32,11 +32,31 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/swarm/storage" ) var hashMatcher = regexp.MustCompile("^[0-9A-Fa-f]{64}") +//setup metrics +var ( + apiResolveCount = metrics.NewRegisteredCounter("api.resolve.count", nil) + apiResolveFail = metrics.NewRegisteredCounter("api.resolve.fail", nil) + apiPutCount = metrics.NewRegisteredCounter("api.put.count", nil) + apiPutFail = metrics.NewRegisteredCounter("api.put.fail", nil) + apiGetCount = metrics.NewRegisteredCounter("api.get.count", nil) + apiGetNotFound = metrics.NewRegisteredCounter("api.get.notfound", nil) + apiGetHttp300 = metrics.NewRegisteredCounter("api.get.http.300", nil) + apiModifyCount = metrics.NewRegisteredCounter("api.modify.count", nil) + apiModifyFail = metrics.NewRegisteredCounter("api.modify.fail", nil) + apiAddFileCount = metrics.NewRegisteredCounter("api.addfile.count", nil) + apiAddFileFail = metrics.NewRegisteredCounter("api.addfile.fail", nil) + apiRmFileCount = metrics.NewRegisteredCounter("api.removefile.count", nil) + apiRmFileFail = metrics.NewRegisteredCounter("api.removefile.fail", nil) + apiAppendFileCount = metrics.NewRegisteredCounter("api.appendfile.count", nil) + apiAppendFileFail = metrics.NewRegisteredCounter("api.appendfile.fail", nil) +) + type Resolver interface { Resolve(string) (common.Hash, error) } @@ -155,6 +175,7 @@ type ErrResolve error // DNS Resolver func (self *Api) Resolve(uri *URI) (storage.Key, error) { + apiResolveCount.Inc(1) log.Trace(fmt.Sprintf("Resolving : %v", uri.Addr)) // if the URI is immutable, check if the address is a hash @@ -169,6 +190,7 @@ func (self *Api) Resolve(uri *URI) (storage.Key, error) { // if DNS is not configured, check if the address is a hash if self.dns == nil { if !isHash { + apiResolveFail.Inc(1) return nil, fmt.Errorf("no DNS to resolve name: %q", uri.Addr) } return common.Hex2Bytes(uri.Addr), nil @@ -179,6 +201,7 @@ func (self *Api) Resolve(uri *URI) (storage.Key, error) { if err == nil { return resolved[:], nil } else if !isHash { + apiResolveFail.Inc(1) return nil, err } return common.Hex2Bytes(uri.Addr), nil @@ -186,16 +209,19 @@ func (self *Api) Resolve(uri *URI) (storage.Key, error) { // Put provides singleton manifest creation on top of dpa store func (self *Api) Put(content, contentType string) (storage.Key, error) { + apiPutCount.Inc(1) r := strings.NewReader(content) wg := &sync.WaitGroup{} key, err := self.dpa.Store(r, int64(len(content)), wg, nil) if err != nil { + apiPutFail.Inc(1) return nil, err } manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType) r = strings.NewReader(manifest) key, err = self.dpa.Store(r, int64(len(manifest)), wg, nil) if err != nil { + apiPutFail.Inc(1) return nil, err } wg.Wait() @@ -206,8 +232,10 @@ func (self *Api) Put(content, contentType string) (storage.Key, error) { // to resolve basePath to content using dpa retrieve // it returns a section reader, mimeType, status and an error func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionReader, mimeType string, status int, err error) { + apiGetCount.Inc(1) trie, err := loadManifest(self.dpa, key, nil) if err != nil { + apiGetNotFound.Inc(1) status = http.StatusNotFound log.Warn(fmt.Sprintf("loadManifestTrie error: %v", err)) return @@ -221,6 +249,7 @@ func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionRe key = common.Hex2Bytes(entry.Hash) status = entry.Status if status == http.StatusMultipleChoices { + apiGetHttp300.Inc(1) return } else { mimeType = entry.ContentType @@ -229,6 +258,7 @@ func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionRe } } else { status = http.StatusNotFound + apiGetNotFound.Inc(1) err = fmt.Errorf("manifest entry for '%s' not found", path) log.Warn(fmt.Sprintf("%v", err)) } @@ -236,9 +266,11 @@ func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionRe } func (self *Api) Modify(key storage.Key, path, contentHash, contentType string) (storage.Key, error) { + apiModifyCount.Inc(1) quitC := make(chan bool) trie, err := loadManifest(self.dpa, key, quitC) if err != nil { + apiModifyFail.Inc(1) return nil, err } if contentHash != "" { @@ -253,19 +285,23 @@ func (self *Api) Modify(key storage.Key, path, contentHash, contentType string) } if err := trie.recalcAndStore(); err != nil { + apiModifyFail.Inc(1) return nil, err } return trie.hash, nil } func (self *Api) AddFile(mhash, path, fname string, content []byte, nameresolver bool) (storage.Key, string, error) { + apiAddFileCount.Inc(1) uri, err := Parse("bzz:/" + mhash) if err != nil { + apiAddFileFail.Inc(1) return nil, "", err } mkey, err := self.Resolve(uri) if err != nil { + apiAddFileFail.Inc(1) return nil, "", err } @@ -284,16 +320,19 @@ func (self *Api) AddFile(mhash, path, fname string, content []byte, nameresolver mw, err := self.NewManifestWriter(mkey, nil) if err != nil { + apiAddFileFail.Inc(1) return nil, "", err } fkey, err := mw.AddEntry(bytes.NewReader(content), entry) if err != nil { + apiAddFileFail.Inc(1) return nil, "", err } newMkey, err := mw.Store() if err != nil { + apiAddFileFail.Inc(1) return nil, "", err } @@ -303,13 +342,16 @@ func (self *Api) AddFile(mhash, path, fname string, content []byte, nameresolver } func (self *Api) RemoveFile(mhash, path, fname string, nameresolver bool) (string, error) { + apiRmFileCount.Inc(1) uri, err := Parse("bzz:/" + mhash) if err != nil { + apiRmFileFail.Inc(1) return "", err } mkey, err := self.Resolve(uri) if err != nil { + apiRmFileFail.Inc(1) return "", err } @@ -320,16 +362,19 @@ func (self *Api) RemoveFile(mhash, path, fname string, nameresolver bool) (strin mw, err := self.NewManifestWriter(mkey, nil) if err != nil { + apiRmFileFail.Inc(1) return "", err } err = mw.RemoveEntry(filepath.Join(path, fname)) if err != nil { + apiRmFileFail.Inc(1) return "", err } newMkey, err := mw.Store() if err != nil { + apiRmFileFail.Inc(1) return "", err } @@ -338,6 +383,7 @@ func (self *Api) RemoveFile(mhash, path, fname string, nameresolver bool) (strin } func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, content []byte, oldKey storage.Key, offset int64, addSize int64, nameresolver bool) (storage.Key, string, error) { + apiAppendFileCount.Inc(1) buffSize := offset + addSize if buffSize < existingSize { @@ -366,10 +412,12 @@ func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, conte uri, err := Parse("bzz:/" + mhash) if err != nil { + apiAppendFileFail.Inc(1) return nil, "", err } mkey, err := self.Resolve(uri) if err != nil { + apiAppendFileFail.Inc(1) return nil, "", err } @@ -380,11 +428,13 @@ func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, conte mw, err := self.NewManifestWriter(mkey, nil) if err != nil { + apiAppendFileFail.Inc(1) return nil, "", err } err = mw.RemoveEntry(filepath.Join(path, fname)) if err != nil { + apiAppendFileFail.Inc(1) return nil, "", err } @@ -398,11 +448,13 @@ func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, conte fkey, err := mw.AddEntry(io.Reader(combinedReader), entry) if err != nil { + apiAppendFileFail.Inc(1) return nil, "", err } newMkey, err := mw.Store() if err != nil { + apiAppendFileFail.Inc(1) return nil, "", err } @@ -412,6 +464,7 @@ func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, conte } func (self *Api) BuildDirectoryTree(mhash string, nameresolver bool) (key storage.Key, manifestEntryMap map[string]*manifestTrieEntry, err error) { + uri, err := Parse("bzz:/" + mhash) if err != nil { return nil, nil, err diff --git a/swarm/api/http/error.go b/swarm/api/http/error.go index dbd97182f..831cf23fe 100644 --- a/swarm/api/http/error.go +++ b/swarm/api/http/error.go @@ -29,12 +29,19 @@ import ( "time" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/swarm/api" ) //templateMap holds a mapping of an HTTP error code to a template var templateMap map[int]*template.Template +//metrics variables +var ( + htmlCounter = metrics.NewRegisteredCounter("api.http.errorpage.html.count", nil) + jsonCounter = metrics.NewRegisteredCounter("api.http.errorpage.json.count", nil) +) + //parameters needed for formatting the correct HTML page type ErrorParams struct { Msg string @@ -132,6 +139,7 @@ func respond(w http.ResponseWriter, r *http.Request, params *ErrorParams) { //return a HTML page func respondHtml(w http.ResponseWriter, params *ErrorParams) { + htmlCounter.Inc(1) err := params.template.Execute(w, params) if err != nil { log.Error(err.Error()) @@ -140,6 +148,7 @@ func respondHtml(w http.ResponseWriter, params *ErrorParams) { //return JSON func respondJson(w http.ResponseWriter, params *ErrorParams) { + jsonCounter.Inc(1) w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(params) } diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index b89487af7..6ebfc8059 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -37,11 +37,35 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/swarm/api" "github.com/ethereum/go-ethereum/swarm/storage" "github.com/rs/cors" ) +//setup metrics +var ( + postRawCount = metrics.NewRegisteredCounter("api.http.post.raw.count", nil) + postRawFail = metrics.NewRegisteredCounter("api.http.post.raw.fail", nil) + postFilesCount = metrics.NewRegisteredCounter("api.http.post.files.count", nil) + postFilesFail = metrics.NewRegisteredCounter("api.http.post.files.fail", nil) + deleteCount = metrics.NewRegisteredCounter("api.http.delete.count", nil) + deleteFail = metrics.NewRegisteredCounter("api.http.delete.fail", nil) + getCount = metrics.NewRegisteredCounter("api.http.get.count", nil) + getFail = metrics.NewRegisteredCounter("api.http.get.fail", nil) + getFileCount = metrics.NewRegisteredCounter("api.http.get.file.count", nil) + getFileNotFound = metrics.NewRegisteredCounter("api.http.get.file.notfound", nil) + getFileFail = metrics.NewRegisteredCounter("api.http.get.file.fail", nil) + getFilesCount = metrics.NewRegisteredCounter("api.http.get.files.count", nil) + getFilesFail = metrics.NewRegisteredCounter("api.http.get.files.fail", nil) + getListCount = metrics.NewRegisteredCounter("api.http.get.list.count", nil) + getListFail = metrics.NewRegisteredCounter("api.http.get.list.fail", nil) + requestCount = metrics.NewRegisteredCounter("http.request.count", nil) + htmlRequestCount = metrics.NewRegisteredCounter("http.request.html.count", nil) + jsonRequestCount = metrics.NewRegisteredCounter("http.request.json.count", nil) + requestTimer = metrics.NewRegisteredResettingTimer("http.request.time", nil) +) + // ServerConfig is the basic configuration needed for the HTTP server and also // includes CORS settings. type ServerConfig struct { @@ -89,18 +113,22 @@ type Request struct { // HandlePostRaw handles a POST request to a raw bzz-raw:/ URI, stores the request // body in swarm and returns the resulting storage key as a text/plain response func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) { + postRawCount.Inc(1) if r.uri.Path != "" { + postRawFail.Inc(1) s.BadRequest(w, r, "raw POST request cannot contain a path") return } if r.Header.Get("Content-Length") == "" { + postRawFail.Inc(1) s.BadRequest(w, r, "missing Content-Length header in request") return } key, err := s.api.Store(r.Body, r.ContentLength, nil) if err != nil { + postRawFail.Inc(1) s.Error(w, r, err) return } @@ -117,8 +145,10 @@ func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) { // existing manifest or to a new manifest under
and returns the // resulting manifest hash as a text/plain response func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) { + postFilesCount.Inc(1) contentType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) if err != nil { + postFilesFail.Inc(1) s.BadRequest(w, r, err.Error()) return } @@ -127,12 +157,14 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) { if r.uri.Addr != "" { key, err = s.api.Resolve(r.uri) if err != nil { + postFilesFail.Inc(1) s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) return } } else { key, err = s.api.NewManifest() if err != nil { + postFilesFail.Inc(1) s.Error(w, r, err) return } @@ -152,6 +184,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) { } }) if err != nil { + postFilesFail.Inc(1) s.Error(w, r, fmt.Errorf("error creating manifest: %s", err)) return } @@ -270,8 +303,10 @@ func (s *Server) handleDirectUpload(req *Request, mw *api.ManifestWriter) error //
from
and returns the resulting manifest hash as a // text/plain response func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) { + deleteCount.Inc(1) key, err := s.api.Resolve(r.uri) if err != nil { + deleteFail.Inc(1) s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) return } @@ -281,6 +316,7 @@ func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) { return mw.RemoveEntry(r.uri.Path) }) if err != nil { + deleteFail.Inc(1) s.Error(w, r, fmt.Errorf("error updating manifest: %s", err)) return } @@ -296,8 +332,10 @@ func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) { // - bzz-hash://
and responds with the hash of the content stored // at the given storage key as a text/plain response func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { + getCount.Inc(1) key, err := s.api.Resolve(r.uri) if err != nil { + getFail.Inc(1) s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) return } @@ -307,6 +345,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { if r.uri.Path != "" { walker, err := s.api.NewManifestWalker(key, nil) if err != nil { + getFail.Inc(1) s.BadRequest(w, r, fmt.Sprintf("%s is not a manifest", key)) return } @@ -335,6 +374,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { return api.SkipManifest }) if entry == nil { + getFail.Inc(1) s.NotFound(w, r, fmt.Errorf("Manifest entry could not be loaded")) return } @@ -344,6 +384,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { // check the root chunk exists by retrieving the file's size reader := s.api.Retrieve(key) if _, err := reader.Size(nil); err != nil { + getFail.Inc(1) s.NotFound(w, r, fmt.Errorf("Root chunk not found %s: %s", key, err)) return } @@ -370,19 +411,23 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) { // header of "application/x-tar" and returns a tar stream of all files // contained in the manifest func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) { + getFilesCount.Inc(1) if r.uri.Path != "" { + getFilesFail.Inc(1) s.BadRequest(w, r, "files request cannot contain a path") return } key, err := s.api.Resolve(r.uri) if err != nil { + getFilesFail.Inc(1) s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) return } walker, err := s.api.NewManifestWalker(key, nil) if err != nil { + getFilesFail.Inc(1) s.Error(w, r, err) return } @@ -430,6 +475,7 @@ func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) { return nil }) if err != nil { + getFilesFail.Inc(1) s.logError("error generating tar stream: %s", err) } } @@ -438,6 +484,7 @@ func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) { // a list of all files contained in
under
grouped into // common prefixes using "/" as a delimiter func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) { + getListCount.Inc(1) // ensure the root path has a trailing slash so that relative URLs work if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") { http.Redirect(w, &r.Request, r.URL.Path+"/", http.StatusMovedPermanently) @@ -446,6 +493,7 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) { key, err := s.api.Resolve(r.uri) if err != nil { + getListFail.Inc(1) s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) return } @@ -453,6 +501,7 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) { list, err := s.getManifestList(key, r.uri.Path) if err != nil { + getListFail.Inc(1) s.Error(w, r, err) return } @@ -470,6 +519,7 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) { List: &list, }) if err != nil { + getListFail.Inc(1) s.logError("error rendering list HTML: %s", err) } return @@ -538,6 +588,7 @@ func (s *Server) getManifestList(key storage.Key, prefix string) (list api.Manif // HandleGetFile handles a GET request to bzz://
/
and responds // with the content of the file at
from the given
func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { + getFileCount.Inc(1) // ensure the root path has a trailing slash so that relative URLs work if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") { http.Redirect(w, &r.Request, r.URL.Path+"/", http.StatusMovedPermanently) @@ -546,6 +597,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { key, err := s.api.Resolve(r.uri) if err != nil { + getFileFail.Inc(1) s.Error(w, r, fmt.Errorf("error resolving %s: %s", r.uri.Addr, err)) return } @@ -554,8 +606,10 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { if err != nil { switch status { case http.StatusNotFound: + getFileNotFound.Inc(1) s.NotFound(w, r, err) default: + getFileFail.Inc(1) s.Error(w, r, err) } return @@ -567,6 +621,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { list, err := s.getManifestList(key, r.uri.Path) if err != nil { + getFileFail.Inc(1) s.Error(w, r, err) return } @@ -579,6 +634,7 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { // check the root chunk exists by retrieving the file's size if _, err := reader.Size(nil); err != nil { + getFileNotFound.Inc(1) s.NotFound(w, r, fmt.Errorf("File not found %s: %s", r.uri, err)) return } @@ -589,6 +645,19 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { } func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if metrics.Enabled { + //The increment for request count and request timer themselves have a flag check + //for metrics.Enabled. Nevertheless, we introduce the if here because we + //are looking into the header just to see what request type it is (json/html). + //So let's take advantage and add all metrics related stuff here + requestCount.Inc(1) + defer requestTimer.UpdateSince(time.Now()) + if r.Header.Get("Accept") == "application/json" { + jsonRequestCount.Inc(1) + } else { + htmlRequestCount.Inc(1) + } + } s.logDebug("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, r.URL.Host, r.URL.Path, r.Referer(), r.Header.Get("Accept")) uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/")) -- cgit v1.2.3 From 22fc6928d720e3cc1f090ce2891f553db1e076ab Mon Sep 17 00:00:00 2001 From: cooganb
Date: Mon, 26 Feb 2018 02:56:40 -0600 Subject: swarm: creates Swarm landing page for browser 'localhost:xxxx/' GET request when running Swarm (#15926) * swarm: began work on GetHandleFile method re: issue #155 * swarm: now able to serve landing page template * swarm: added landing page template * swarm: landing page has working input * swarm: fixed CSS issue in template * swarm: deleted extra lines * swarm: deleted time header and made redirect a relative path * swarm: removed code mistakenly left --- swarm/api/http/server.go | 9 +++ swarm/api/http/templates.go | 143 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+) (limited to 'swarm/api') diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 6ebfc8059..df90fd04f 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -660,6 +660,15 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { } s.logDebug("HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, r.URL.Host, r.URL.Path, r.Referer(), r.Header.Get("Accept")) + if r.RequestURI == "/" && strings.Contains(r.Header.Get("Accept"), "text/html") { + + err := landingPageTemplate.Execute(w, nil) + if err != nil { + s.logError("error rendering landing page: %s", err) + } + return + } + uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/")) req := &Request{Request: *r, uri: uri} if err != nil { diff --git a/swarm/api/http/templates.go b/swarm/api/http/templates.go index 189a99912..cd9d21289 100644 --- a/swarm/api/http/templates.go +++ b/swarm/api/http/templates.go @@ -70,3 +70,146 @@ var htmlListTemplate = template.Must(template.New("html-list").Funcs(template.Fu
+
+
+