diff options
Diffstat (limited to 'swarm/api/client')
-rw-r--r-- | swarm/api/client/client.go | 829 | ||||
-rw-r--r-- | swarm/api/client/client_test.go | 608 |
2 files changed, 0 insertions, 1437 deletions
diff --git a/swarm/api/client/client.go b/swarm/api/client/client.go deleted file mode 100644 index 9ad0948f4..000000000 --- a/swarm/api/client/client.go +++ /dev/null @@ -1,829 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. - -package client - -import ( - "archive/tar" - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "io/ioutil" - "mime/multipart" - "net/http" - "net/http/httptrace" - "net/textproto" - "net/url" - "os" - "path/filepath" - "regexp" - "strconv" - "strings" - "time" - - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/metrics" - "github.com/ethereum/go-ethereum/swarm/api" - swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http" - "github.com/ethereum/go-ethereum/swarm/spancontext" - "github.com/ethereum/go-ethereum/swarm/storage/feed" - "github.com/pborman/uuid" -) - -var ( - ErrUnauthorized = errors.New("unauthorized") -) - -func NewClient(gateway string) *Client { - return &Client{ - Gateway: gateway, - } -} - -// Client wraps interaction with a swarm HTTP gateway. -type Client struct { - Gateway string -} - -// UploadRaw uploads raw data to swarm and returns the resulting hash. If toEncrypt is true it -// uploads encrypted data -func (c *Client) UploadRaw(r io.Reader, size int64, toEncrypt bool) (string, error) { - if size <= 0 { - return "", errors.New("data size must be greater than zero") - } - addr := "" - if toEncrypt { - addr = "encrypt" - } - req, err := http.NewRequest("POST", c.Gateway+"/bzz-raw:/"+addr, r) - if err != nil { - return "", err - } - req.ContentLength = size - req.Header.Set(swarmhttp.SwarmTagHeaderName, fmt.Sprintf("raw_upload_%d", time.Now().Unix())) - - res, err := http.DefaultClient.Do(req) - if err != nil { - return "", err - } - defer res.Body.Close() - if res.StatusCode != http.StatusOK { - return "", fmt.Errorf("unexpected HTTP status: %s", res.Status) - } - data, err := ioutil.ReadAll(res.Body) - if err != nil { - return "", err - } - return string(data), nil -} - -// DownloadRaw downloads raw data from swarm and it returns a ReadCloser and a bool whether the -// content was encrypted -func (c *Client) DownloadRaw(hash string) (io.ReadCloser, bool, error) { - uri := c.Gateway + "/bzz-raw:/" + hash - res, err := http.DefaultClient.Get(uri) - if err != nil { - return nil, false, err - } - if res.StatusCode != http.StatusOK { - res.Body.Close() - return nil, false, fmt.Errorf("unexpected HTTP status: %s", res.Status) - } - isEncrypted := (res.Header.Get("X-Decrypted") == "true") - return res.Body, isEncrypted, nil -} - -// File represents a file in a swarm manifest and is used for uploading and -// downloading content to and from swarm -type File struct { - io.ReadCloser - api.ManifestEntry - Tag string -} - -// Open opens a local file which can then be passed to client.Upload to upload -// it to swarm -func Open(path string) (*File, error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - stat, err := f.Stat() - if err != nil { - f.Close() - return nil, err - } - - contentType, err := api.DetectContentType(f.Name(), f) - if err != nil { - return nil, err - } - - return &File{ - ReadCloser: f, - ManifestEntry: api.ManifestEntry{ - ContentType: contentType, - Mode: int64(stat.Mode()), - Size: stat.Size(), - ModTime: stat.ModTime(), - }, - Tag: filepath.Base(path), - }, nil -} - -// Upload uploads a file to swarm and either adds it to an existing manifest -// (if the manifest argument is non-empty) or creates a new manifest containing -// the file, returning the resulting manifest hash (the file will then be -// available at bzz:/<hash>/<path>) -func (c *Client) Upload(file *File, manifest string, toEncrypt bool) (string, error) { - if file.Size <= 0 { - return "", errors.New("file size must be greater than zero") - } - return c.TarUpload(manifest, &FileUploader{file}, "", toEncrypt) -} - -// Download downloads a file with the given path from the swarm manifest with -// the given hash (i.e. it gets bzz:/<hash>/<path>) -func (c *Client) Download(hash, path string) (*File, error) { - uri := c.Gateway + "/bzz:/" + hash + "/" + path - res, err := http.DefaultClient.Get(uri) - if err != nil { - return nil, err - } - if res.StatusCode != http.StatusOK { - res.Body.Close() - return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status) - } - return &File{ - ReadCloser: res.Body, - ManifestEntry: api.ManifestEntry{ - ContentType: res.Header.Get("Content-Type"), - Size: res.ContentLength, - }, - }, nil -} - -// UploadDirectory uploads a directory tree to swarm and either adds the files -// to an existing manifest (if the manifest argument is non-empty) or creates a -// new manifest, returning the resulting manifest hash (files from the -// directory will then be available at bzz:/<hash>/path/to/file), with -// the file specified in defaultPath being uploaded to the root of the manifest -// (i.e. bzz:/<hash>/) -func (c *Client) UploadDirectory(dir, defaultPath, manifest string, toEncrypt bool) (string, error) { - stat, err := os.Stat(dir) - if err != nil { - return "", err - } else if !stat.IsDir() { - return "", fmt.Errorf("not a directory: %s", dir) - } - if defaultPath != "" { - if _, err := os.Stat(filepath.Join(dir, defaultPath)); err != nil { - if os.IsNotExist(err) { - return "", fmt.Errorf("the default path %q was not found in the upload directory %q", defaultPath, dir) - } - return "", fmt.Errorf("default path: %v", err) - } - } - return c.TarUpload(manifest, &DirectoryUploader{dir}, defaultPath, toEncrypt) -} - -// DownloadDirectory downloads the files contained in a swarm manifest under -// the given path into a local directory (existing files will be overwritten) -func (c *Client) DownloadDirectory(hash, path, destDir, credentials string) error { - stat, err := os.Stat(destDir) - if err != nil { - return err - } else if !stat.IsDir() { - return fmt.Errorf("not a directory: %s", destDir) - } - - uri := c.Gateway + "/bzz:/" + hash + "/" + path - req, err := http.NewRequest("GET", uri, nil) - if err != nil { - return err - } - if credentials != "" { - req.SetBasicAuth("", credentials) - } - req.Header.Set("Accept", "application/x-tar") - res, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - defer res.Body.Close() - switch res.StatusCode { - case http.StatusOK: - case http.StatusUnauthorized: - return ErrUnauthorized - default: - return fmt.Errorf("unexpected HTTP status: %s", res.Status) - } - tr := tar.NewReader(res.Body) - for { - hdr, err := tr.Next() - if err == io.EOF { - return nil - } else if err != nil { - return err - } - // ignore the default path file - if hdr.Name == "" { - continue - } - - dstPath := filepath.Join(destDir, filepath.Clean(strings.TrimPrefix(hdr.Name, path))) - if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil { - return err - } - var mode os.FileMode = 0644 - if hdr.Mode > 0 { - mode = os.FileMode(hdr.Mode) - } - dst, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode) - if err != nil { - return err - } - n, err := io.Copy(dst, tr) - dst.Close() - if err != nil { - return err - } else if n != hdr.Size { - return fmt.Errorf("expected %s to be %d bytes but got %d", hdr.Name, hdr.Size, n) - } - } -} - -// DownloadFile downloads a single file into the destination directory -// if the manifest entry does not specify a file name - it will fallback -// to the hash of the file as a filename -func (c *Client) DownloadFile(hash, path, dest, credentials string) error { - hasDestinationFilename := false - if stat, err := os.Stat(dest); err == nil { - hasDestinationFilename = !stat.IsDir() - } else { - if os.IsNotExist(err) { - // does not exist - should be created - hasDestinationFilename = true - } else { - return fmt.Errorf("could not stat path: %v", err) - } - } - - manifestList, err := c.List(hash, path, credentials) - if err != nil { - return err - } - - switch len(manifestList.Entries) { - case 0: - return fmt.Errorf("could not find path requested at manifest address. make sure the path you've specified is correct") - case 1: - //continue - default: - return fmt.Errorf("got too many matches for this path") - } - - uri := c.Gateway + "/bzz:/" + hash + "/" + path - req, err := http.NewRequest("GET", uri, nil) - if err != nil { - return err - } - if credentials != "" { - req.SetBasicAuth("", credentials) - } - res, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - defer res.Body.Close() - switch res.StatusCode { - case http.StatusOK: - case http.StatusUnauthorized: - return ErrUnauthorized - default: - return fmt.Errorf("unexpected HTTP status: expected 200 OK, got %d", res.StatusCode) - } - filename := "" - if hasDestinationFilename { - filename = dest - } else { - // try to assert - re := regexp.MustCompile("[^/]+$") //everything after last slash - - if results := re.FindAllString(path, -1); len(results) > 0 { - filename = results[len(results)-1] - } else { - if entry := manifestList.Entries[0]; entry.Path != "" && entry.Path != "/" { - filename = entry.Path - } else { - // assume hash as name if there's nothing from the command line - filename = hash - } - } - filename = filepath.Join(dest, filename) - } - filePath, err := filepath.Abs(filename) - if err != nil { - return err - } - - if err := os.MkdirAll(filepath.Dir(filePath), 0777); err != nil { - return err - } - - dst, err := os.Create(filename) - if err != nil { - return err - } - defer dst.Close() - - _, err = io.Copy(dst, res.Body) - return err -} - -// UploadManifest uploads the given manifest to swarm -func (c *Client) UploadManifest(m *api.Manifest, toEncrypt bool) (string, error) { - data, err := json.Marshal(m) - if err != nil { - return "", err - } - return c.UploadRaw(bytes.NewReader(data), int64(len(data)), toEncrypt) -} - -// DownloadManifest downloads a swarm manifest -func (c *Client) DownloadManifest(hash string) (*api.Manifest, bool, error) { - res, isEncrypted, err := c.DownloadRaw(hash) - if err != nil { - return nil, isEncrypted, err - } - defer res.Close() - var manifest api.Manifest - if err := json.NewDecoder(res).Decode(&manifest); err != nil { - return nil, isEncrypted, err - } - return &manifest, isEncrypted, nil -} - -// List list files in a swarm manifest which have the given prefix, grouping -// common prefixes using "/" as a delimiter. -// -// For example, if the manifest represents the following directory structure: -// -// file1.txt -// file2.txt -// dir1/file3.txt -// dir1/dir2/file4.txt -// -// Then: -// -// - a prefix of "" would return [dir1/, file1.txt, file2.txt] -// - a prefix of "file" would return [file1.txt, file2.txt] -// - a prefix of "dir1/" would return [dir1/dir2/, dir1/file3.txt] -// -// where entries ending with "/" are common prefixes. -func (c *Client) List(hash, prefix, credentials string) (*api.ManifestList, error) { - req, err := http.NewRequest(http.MethodGet, c.Gateway+"/bzz-list:/"+hash+"/"+prefix, nil) - if err != nil { - return nil, err - } - if credentials != "" { - req.SetBasicAuth("", credentials) - } - res, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - switch res.StatusCode { - case http.StatusOK: - case http.StatusUnauthorized: - return nil, ErrUnauthorized - default: - return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status) - } - var list api.ManifestList - if err := json.NewDecoder(res.Body).Decode(&list); err != nil { - return nil, err - } - return &list, nil -} - -// Uploader uploads files to swarm using a provided UploadFn -type Uploader interface { - Upload(UploadFn) error - Tag() string -} - -type UploaderFunc func(UploadFn) error - -func (u UploaderFunc) Upload(upload UploadFn) error { - return u(upload) -} - -func (u UploaderFunc) Tag() string { - return fmt.Sprintf("multipart_upload_%d", time.Now().Unix()) -} - -// DirectoryUploader implements Uploader -var _ Uploader = &DirectoryUploader{} - -// DirectoryUploader uploads all files in a directory, optionally uploading -// a file to the default path -type DirectoryUploader struct { - Dir string -} - -func (d *DirectoryUploader) Tag() string { - return filepath.Base(d.Dir) -} - -// Upload performs the upload of the directory and default path -func (d *DirectoryUploader) Upload(upload UploadFn) error { - return filepath.Walk(d.Dir, func(path string, f os.FileInfo, err error) error { - if err != nil { - return err - } - if f.IsDir() { - return nil - } - file, err := Open(path) - if err != nil { - return err - } - relPath, err := filepath.Rel(d.Dir, path) - if err != nil { - return err - } - file.Path = filepath.ToSlash(relPath) - return upload(file) - }) -} - -var _ Uploader = &FileUploader{} - -// FileUploader uploads a single file -type FileUploader struct { - File *File -} - -func (f *FileUploader) Tag() string { - return f.File.Tag -} - -// Upload performs the upload of the file -func (f *FileUploader) Upload(upload UploadFn) error { - return upload(f.File) -} - -// UploadFn is the type of function passed to an Uploader to perform the upload -// of a single file (for example, a directory uploader would call a provided -// UploadFn for each file in the directory tree) -type UploadFn func(file *File) error - -// TarUpload uses the given Uploader to upload files to swarm as a tar stream, -// returning the resulting manifest hash -func (c *Client) TarUpload(hash string, uploader Uploader, defaultPath string, toEncrypt bool) (string, error) { - ctx, sp := spancontext.StartSpan(context.Background(), "api.client.tarupload") - defer sp.Finish() - - var tn time.Time - - reqR, reqW := io.Pipe() - defer reqR.Close() - addr := hash - - // If there is a hash already (a manifest), then that manifest will determine if the upload has - // to be encrypted or not. If there is no manifest then the toEncrypt parameter decides if - // there is encryption or not. - if hash == "" && toEncrypt { - // This is the built-in address for the encrypted upload endpoint - addr = "encrypt" - } - req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+addr, reqR) - if err != nil { - return "", err - } - - trace := GetClientTrace("swarm api client - upload tar", "api.client.uploadtar", uuid.New()[:8], &tn) - - req = req.WithContext(httptrace.WithClientTrace(ctx, trace)) - transport := http.DefaultTransport - - req.Header.Set("Content-Type", "application/x-tar") - if defaultPath != "" { - q := req.URL.Query() - q.Set("defaultpath", defaultPath) - req.URL.RawQuery = q.Encode() - } - - tag := uploader.Tag() - if tag == "" { - tag = "unnamed_tag_" + fmt.Sprintf("%d", time.Now().Unix()) - } - log.Trace("setting upload tag", "tag", tag) - - req.Header.Set(swarmhttp.SwarmTagHeaderName, tag) - - // use 'Expect: 100-continue' so we don't send the request body if - // the server refuses the request - req.Header.Set("Expect", "100-continue") - - tw := tar.NewWriter(reqW) - - // define an UploadFn which adds files to the tar stream - uploadFn := func(file *File) error { - hdr := &tar.Header{ - Name: file.Path, - Mode: file.Mode, - Size: file.Size, - ModTime: file.ModTime, - Xattrs: map[string]string{ - "user.swarm.content-type": file.ContentType, - }, - } - if err := tw.WriteHeader(hdr); err != nil { - return err - } - _, err = io.Copy(tw, file) - return err - } - - // run the upload in a goroutine so we can send the request headers and - // wait for a '100 Continue' response before sending the tar stream - go func() { - err := uploader.Upload(uploadFn) - if err == nil { - err = tw.Close() - } - reqW.CloseWithError(err) - }() - tn = time.Now() - res, err := transport.RoundTrip(req) - if err != nil { - return "", err - } - defer res.Body.Close() - if res.StatusCode != http.StatusOK { - return "", fmt.Errorf("unexpected HTTP status: %s", res.Status) - } - data, err := ioutil.ReadAll(res.Body) - if err != nil { - return "", err - } - return string(data), nil -} - -// MultipartUpload uses the given Uploader to upload files to swarm as a -// multipart form, returning the resulting manifest hash -func (c *Client) MultipartUpload(hash string, uploader Uploader) (string, error) { - reqR, reqW := io.Pipe() - defer reqR.Close() - req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+hash, reqR) - if err != nil { - return "", err - } - - // use 'Expect: 100-continue' so we don't send the request body if - // the server refuses the request - req.Header.Set("Expect", "100-continue") - - mw := multipart.NewWriter(reqW) - req.Header.Set("Content-Type", fmt.Sprintf("multipart/form-data; boundary=%q", mw.Boundary())) - req.Header.Set(swarmhttp.SwarmTagHeaderName, fmt.Sprintf("multipart_upload_%d", time.Now().Unix())) - - // define an UploadFn which adds files to the multipart form - uploadFn := func(file *File) error { - hdr := make(textproto.MIMEHeader) - hdr.Set("Content-Disposition", fmt.Sprintf("form-data; name=%q", file.Path)) - hdr.Set("Content-Type", file.ContentType) - hdr.Set("Content-Length", strconv.FormatInt(file.Size, 10)) - w, err := mw.CreatePart(hdr) - if err != nil { - return err - } - _, err = io.Copy(w, file) - return err - } - - // run the upload in a goroutine so we can send the request headers and - // wait for a '100 Continue' response before sending the multipart form - go func() { - err := uploader.Upload(uploadFn) - if err == nil { - err = mw.Close() - } - reqW.CloseWithError(err) - }() - - res, err := http.DefaultClient.Do(req) - if err != nil { - return "", err - } - defer res.Body.Close() - if res.StatusCode != http.StatusOK { - return "", fmt.Errorf("unexpected HTTP status: %s", res.Status) - } - data, err := ioutil.ReadAll(res.Body) - if err != nil { - return "", err - } - return string(data), nil -} - -// ErrNoFeedUpdatesFound is returned when Swarm cannot find updates of the given feed -var ErrNoFeedUpdatesFound = errors.New("No updates found for this feed") - -// CreateFeedWithManifest creates a feed manifest, initializing it with the provided -// data -// Returns the resulting feed manifest address that you can use to include in an ENS Resolver (setContent) -// or reference future updates (Client.UpdateFeed) -func (c *Client) CreateFeedWithManifest(request *feed.Request) (string, error) { - responseStream, err := c.updateFeed(request, true) - if err != nil { - return "", err - } - defer responseStream.Close() - - body, err := ioutil.ReadAll(responseStream) - if err != nil { - return "", err - } - - var manifestAddress string - if err = json.Unmarshal(body, &manifestAddress); err != nil { - return "", err - } - return manifestAddress, nil -} - -// UpdateFeed allows you to set a new version of your content -func (c *Client) UpdateFeed(request *feed.Request) error { - _, err := c.updateFeed(request, false) - return err -} - -func (c *Client) updateFeed(request *feed.Request, createManifest bool) (io.ReadCloser, error) { - URL, err := url.Parse(c.Gateway) - if err != nil { - return nil, err - } - URL.Path = "/bzz-feed:/" - values := URL.Query() - body := request.AppendValues(values) - if createManifest { - values.Set("manifest", "1") - } - URL.RawQuery = values.Encode() - - req, err := http.NewRequest("POST", URL.String(), bytes.NewBuffer(body)) - if err != nil { - return nil, err - } - - res, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - - return res.Body, nil -} - -// QueryFeed returns a byte stream with the raw content of the feed update -// manifestAddressOrDomain is the address you obtained in CreateFeedWithManifest or an ENS domain whose Resolver -// points to that address -func (c *Client) QueryFeed(query *feed.Query, manifestAddressOrDomain string) (io.ReadCloser, error) { - return c.queryFeed(query, manifestAddressOrDomain, false) -} - -// queryFeed returns a byte stream with the raw content of the feed update -// manifestAddressOrDomain is the address you obtained in CreateFeedWithManifest or an ENS domain whose Resolver -// points to that address -// meta set to true will instruct the node return feed metainformation instead -func (c *Client) queryFeed(query *feed.Query, manifestAddressOrDomain string, meta bool) (io.ReadCloser, error) { - URL, err := url.Parse(c.Gateway) - if err != nil { - return nil, err - } - URL.Path = "/bzz-feed:/" + manifestAddressOrDomain - values := URL.Query() - if query != nil { - query.AppendValues(values) //adds query parameters - } - if meta { - values.Set("meta", "1") - } - URL.RawQuery = values.Encode() - res, err := http.Get(URL.String()) - if err != nil { - return nil, err - } - - if res.StatusCode != http.StatusOK { - if res.StatusCode == http.StatusNotFound { - return nil, ErrNoFeedUpdatesFound - } - errorMessageBytes, err := ioutil.ReadAll(res.Body) - var errorMessage string - if err != nil { - errorMessage = "cannot retrieve error message: " + err.Error() - } else { - errorMessage = string(errorMessageBytes) - } - return nil, fmt.Errorf("Error retrieving feed updates: %s", errorMessage) - } - - return res.Body, nil -} - -// GetFeedRequest returns a structure that describes the referenced feed status -// manifestAddressOrDomain is the address you obtained in CreateFeedWithManifest or an ENS domain whose Resolver -// points to that address -func (c *Client) GetFeedRequest(query *feed.Query, manifestAddressOrDomain string) (*feed.Request, error) { - - responseStream, err := c.queryFeed(query, manifestAddressOrDomain, true) - if err != nil { - return nil, err - } - defer responseStream.Close() - - body, err := ioutil.ReadAll(responseStream) - if err != nil { - return nil, err - } - - var metadata feed.Request - if err := metadata.UnmarshalJSON(body); err != nil { - return nil, err - } - return &metadata, nil -} - -func GetClientTrace(traceMsg, metricPrefix, ruid string, tn *time.Time) *httptrace.ClientTrace { - trace := &httptrace.ClientTrace{ - GetConn: func(_ string) { - log.Trace(traceMsg+" - http get", "event", "GetConn", "ruid", ruid) - metrics.GetOrRegisterResettingTimer(metricPrefix+".getconn", nil).Update(time.Since(*tn)) - }, - GotConn: func(_ httptrace.GotConnInfo) { - log.Trace(traceMsg+" - http get", "event", "GotConn", "ruid", ruid) - metrics.GetOrRegisterResettingTimer(metricPrefix+".gotconn", nil).Update(time.Since(*tn)) - }, - PutIdleConn: func(err error) { - log.Trace(traceMsg+" - http get", "event", "PutIdleConn", "ruid", ruid, "err", err) - metrics.GetOrRegisterResettingTimer(metricPrefix+".putidle", nil).Update(time.Since(*tn)) - }, - GotFirstResponseByte: func() { - log.Trace(traceMsg+" - http get", "event", "GotFirstResponseByte", "ruid", ruid) - metrics.GetOrRegisterResettingTimer(metricPrefix+".firstbyte", nil).Update(time.Since(*tn)) - }, - Got100Continue: func() { - log.Trace(traceMsg, "event", "Got100Continue", "ruid", ruid) - metrics.GetOrRegisterResettingTimer(metricPrefix+".got100continue", nil).Update(time.Since(*tn)) - }, - DNSStart: func(_ httptrace.DNSStartInfo) { - log.Trace(traceMsg, "event", "DNSStart", "ruid", ruid) - metrics.GetOrRegisterResettingTimer(metricPrefix+".dnsstart", nil).Update(time.Since(*tn)) - }, - DNSDone: func(_ httptrace.DNSDoneInfo) { - log.Trace(traceMsg, "event", "DNSDone", "ruid", ruid) - metrics.GetOrRegisterResettingTimer(metricPrefix+".dnsdone", nil).Update(time.Since(*tn)) - }, - ConnectStart: func(network, addr string) { - log.Trace(traceMsg, "event", "ConnectStart", "ruid", ruid, "network", network, "addr", addr) - metrics.GetOrRegisterResettingTimer(metricPrefix+".connectstart", nil).Update(time.Since(*tn)) - }, - ConnectDone: func(network, addr string, err error) { - log.Trace(traceMsg, "event", "ConnectDone", "ruid", ruid, "network", network, "addr", addr, "err", err) - metrics.GetOrRegisterResettingTimer(metricPrefix+".connectdone", nil).Update(time.Since(*tn)) - }, - WroteHeaders: func() { - log.Trace(traceMsg, "event", "WroteHeaders(request)", "ruid", ruid) - metrics.GetOrRegisterResettingTimer(metricPrefix+".wroteheaders", nil).Update(time.Since(*tn)) - }, - Wait100Continue: func() { - log.Trace(traceMsg, "event", "Wait100Continue", "ruid", ruid) - metrics.GetOrRegisterResettingTimer(metricPrefix+".wait100continue", nil).Update(time.Since(*tn)) - }, - WroteRequest: func(_ httptrace.WroteRequestInfo) { - log.Trace(traceMsg, "event", "WroteRequest", "ruid", ruid) - metrics.GetOrRegisterResettingTimer(metricPrefix+".wroterequest", nil).Update(time.Since(*tn)) - }, - } - return trace -} diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go deleted file mode 100644 index 92489849c..000000000 --- a/swarm/api/client/client_test.go +++ /dev/null @@ -1,608 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>. - -package client - -import ( - "bytes" - "io/ioutil" - "os" - "path/filepath" - "reflect" - "sort" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/swarm/api" - swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http" - "github.com/ethereum/go-ethereum/swarm/storage" - "github.com/ethereum/go-ethereum/swarm/storage/feed" - "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" - "github.com/ethereum/go-ethereum/swarm/testutil" -) - -func serverFunc(api *api.API) swarmhttp.TestServer { - return swarmhttp.NewServer(api, "") -} - -// TestClientUploadDownloadRaw test uploading and downloading raw data to swarm -func TestClientUploadDownloadRaw(t *testing.T) { - testClientUploadDownloadRaw(false, t) -} - -func TestClientUploadDownloadRawEncrypted(t *testing.T) { - if testutil.RaceEnabled { - t.Skip("flaky with -race on Travis") - // See: https://github.com/ethersphere/go-ethereum/issues/1254 - } - - testClientUploadDownloadRaw(true, t) -} - -func testClientUploadDownloadRaw(toEncrypt bool, t *testing.T) { - srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) - defer srv.Close() - - client := NewClient(srv.URL) - - // upload some raw data - data := []byte("foo123") - hash, err := client.UploadRaw(bytes.NewReader(data), int64(len(data)), toEncrypt) - if err != nil { - t.Fatal(err) - } - - // check the tag was created successfully - tag := srv.Tags.All()[0] - testutil.CheckTag(t, tag, 1, 1, 0, 1) - - // check we can download the same data - res, isEncrypted, err := client.DownloadRaw(hash) - if err != nil { - t.Fatal(err) - } - if isEncrypted != toEncrypt { - t.Fatalf("Expected encyption status %v got %v", toEncrypt, isEncrypted) - } - defer res.Close() - gotData, err := ioutil.ReadAll(res) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(gotData, data) { - t.Fatalf("expected downloaded data to be %q, got %q", data, gotData) - } -} - -// TestClientUploadDownloadFiles test uploading and downloading files to swarm -// manifests -func TestClientUploadDownloadFiles(t *testing.T) { - testClientUploadDownloadFiles(false, t) -} - -func TestClientUploadDownloadFilesEncrypted(t *testing.T) { - testClientUploadDownloadFiles(true, t) -} - -func testClientUploadDownloadFiles(toEncrypt bool, t *testing.T) { - srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) - defer srv.Close() - - client := NewClient(srv.URL) - upload := func(manifest, path string, data []byte) string { - file := &File{ - ReadCloser: ioutil.NopCloser(bytes.NewReader(data)), - ManifestEntry: api.ManifestEntry{ - Path: path, - ContentType: "text/plain", - Size: int64(len(data)), - }, - } - hash, err := client.Upload(file, manifest, toEncrypt) - if err != nil { - t.Fatal(err) - } - return hash - } - checkDownload := func(manifest, path string, expected []byte) { - file, err := client.Download(manifest, path) - if err != nil { - t.Fatal(err) - } - defer file.Close() - if file.Size != int64(len(expected)) { - t.Fatalf("expected downloaded file to be %d bytes, got %d", len(expected), file.Size) - } - if file.ContentType != "text/plain" { - t.Fatalf("expected downloaded file to have type %q, got %q", "text/plain", file.ContentType) - } - data, err := ioutil.ReadAll(file) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(data, expected) { - t.Fatalf("expected downloaded data to be %q, got %q", expected, data) - } - } - - // upload a file to the root of a manifest - rootData := []byte("some-data") - rootHash := upload("", "", rootData) - - // check we can download the root file - checkDownload(rootHash, "", rootData) - - // upload another file to the same manifest - otherData := []byte("some-other-data") - newHash := upload(rootHash, "some/other/path", otherData) - - // check we can download both files from the new manifest - checkDownload(newHash, "", rootData) - checkDownload(newHash, "some/other/path", otherData) - - // replace the root file with different data - newHash = upload(newHash, "", otherData) - - // check both files have the other data - checkDownload(newHash, "", otherData) - checkDownload(newHash, "some/other/path", otherData) -} - -var testDirFiles = []string{ - "file1.txt", - "file2.txt", - "dir1/file3.txt", - "dir1/file4.txt", - "dir2/file5.txt", - "dir2/dir3/file6.txt", - "dir2/dir4/file7.txt", - "dir2/dir4/file8.txt", -} - -func newTestDirectory(t *testing.T) string { - dir, err := ioutil.TempDir("", "swarm-client-test") - if err != nil { - t.Fatal(err) - } - - for _, file := range testDirFiles { - path := filepath.Join(dir, file) - if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { - os.RemoveAll(dir) - t.Fatalf("error creating dir for %s: %s", path, err) - } - if err := ioutil.WriteFile(path, []byte(file), 0644); err != nil { - os.RemoveAll(dir) - t.Fatalf("error writing file %s: %s", path, err) - } - } - - return dir -} - -// TestClientUploadDownloadDirectory tests uploading and downloading a -// directory of files to a swarm manifest -func TestClientUploadDownloadDirectory(t *testing.T) { - srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) - defer srv.Close() - - dir := newTestDirectory(t) - defer os.RemoveAll(dir) - - // upload the directory - client := NewClient(srv.URL) - defaultPath := testDirFiles[0] - hash, err := client.UploadDirectory(dir, defaultPath, "", false) - if err != nil { - t.Fatalf("error uploading directory: %s", err) - } - - // check the tag was created successfully - tag := srv.Tags.All()[0] - testutil.CheckTag(t, tag, 9, 9, 0, 9) - - // check we can download the individual files - checkDownloadFile := func(path string, expected []byte) { - file, err := client.Download(hash, path) - if err != nil { - t.Fatal(err) - } - defer file.Close() - data, err := ioutil.ReadAll(file) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(data, expected) { - t.Fatalf("expected data to be %q, got %q", expected, data) - } - } - for _, file := range testDirFiles { - checkDownloadFile(file, []byte(file)) - } - - // check we can download the default path - checkDownloadFile("", []byte(testDirFiles[0])) - - // check we can download the directory - tmp, err := ioutil.TempDir("", "swarm-client-test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmp) - if err := client.DownloadDirectory(hash, "", tmp, ""); err != nil { - t.Fatal(err) - } - for _, file := range testDirFiles { - data, err := ioutil.ReadFile(filepath.Join(tmp, file)) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(data, []byte(file)) { - t.Fatalf("expected data to be %q, got %q", file, data) - } - } -} - -// TestClientFileList tests listing files in a swarm manifest -func TestClientFileList(t *testing.T) { - testClientFileList(false, t) -} - -func TestClientFileListEncrypted(t *testing.T) { - testClientFileList(true, t) -} - -func testClientFileList(toEncrypt bool, t *testing.T) { - srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) - defer srv.Close() - - dir := newTestDirectory(t) - defer os.RemoveAll(dir) - - client := NewClient(srv.URL) - hash, err := client.UploadDirectory(dir, "", "", toEncrypt) - if err != nil { - t.Fatalf("error uploading directory: %s", err) - } - - ls := func(prefix string) []string { - list, err := client.List(hash, prefix, "") - if err != nil { - t.Fatal(err) - } - paths := make([]string, 0, len(list.CommonPrefixes)+len(list.Entries)) - paths = append(paths, list.CommonPrefixes...) - for _, entry := range list.Entries { - paths = append(paths, entry.Path) - } - sort.Strings(paths) - return paths - } - - tests := map[string][]string{ - "": {"dir1/", "dir2/", "file1.txt", "file2.txt"}, - "file": {"file1.txt", "file2.txt"}, - "file1": {"file1.txt"}, - "file2.txt": {"file2.txt"}, - "file12": {}, - "dir": {"dir1/", "dir2/"}, - "dir1": {"dir1/"}, - "dir1/": {"dir1/file3.txt", "dir1/file4.txt"}, - "dir1/file": {"dir1/file3.txt", "dir1/file4.txt"}, - "dir1/file3.txt": {"dir1/file3.txt"}, - "dir1/file34": {}, - "dir2/": {"dir2/dir3/", "dir2/dir4/", "dir2/file5.txt"}, - "dir2/file": {"dir2/file5.txt"}, - "dir2/dir": {"dir2/dir3/", "dir2/dir4/"}, - "dir2/dir3/": {"dir2/dir3/file6.txt"}, - "dir2/dir4/": {"dir2/dir4/file7.txt", "dir2/dir4/file8.txt"}, - "dir2/dir4/file": {"dir2/dir4/file7.txt", "dir2/dir4/file8.txt"}, - "dir2/dir4/file7.txt": {"dir2/dir4/file7.txt"}, - "dir2/dir4/file78": {}, - } - for prefix, expected := range tests { - actual := ls(prefix) - if !reflect.DeepEqual(actual, expected) { - t.Fatalf("expected prefix %q to return %v, got %v", prefix, expected, actual) - } - } -} - -// TestClientMultipartUpload tests uploading files to swarm using a multipart -// upload -func TestClientMultipartUpload(t *testing.T) { - srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) - defer srv.Close() - - // define an uploader which uploads testDirFiles with some data - // note: this test should result in SEEN chunks. assert accordingly - data := []byte("some-data") - uploader := UploaderFunc(func(upload UploadFn) error { - for _, name := range testDirFiles { - file := &File{ - ReadCloser: ioutil.NopCloser(bytes.NewReader(data)), - ManifestEntry: api.ManifestEntry{ - Path: name, - ContentType: "text/plain", - Size: int64(len(data)), - }, - } - if err := upload(file); err != nil { - return err - } - } - return nil - }) - - // upload the files as a multipart upload - client := NewClient(srv.URL) - hash, err := client.MultipartUpload("", uploader) - if err != nil { - t.Fatal(err) - } - - // check the tag was created successfully - tag := srv.Tags.All()[0] - testutil.CheckTag(t, tag, 9, 9, 7, 9) - - // check we can download the individual files - checkDownloadFile := func(path string) { - file, err := client.Download(hash, path) - if err != nil { - t.Fatal(err) - } - defer file.Close() - gotData, err := ioutil.ReadAll(file) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(gotData, data) { - t.Fatalf("expected data to be %q, got %q", data, gotData) - } - } - for _, file := range testDirFiles { - checkDownloadFile(file) - } -} - -func newTestSigner() (*feed.GenericSigner, error) { - privKey, err := crypto.HexToECDSA("deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef") - if err != nil { - return nil, err - } - return feed.NewGenericSigner(privKey), nil -} - -// Test the transparent resolving of feed updates with bzz:// scheme -// -// First upload data to bzz:, and store the Swarm hash to the resulting manifest in a feed update. -// This effectively uses a feed to store a pointer to content rather than the content itself -// Retrieving the update with the Swarm hash should return the manifest pointing directly to the data -// and raw retrieve of that hash should return the data -func TestClientBzzWithFeed(t *testing.T) { - - signer, _ := newTestSigner() - - // Initialize a Swarm test server - srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) - swarmClient := NewClient(srv.URL) - defer srv.Close() - - // put together some data for our test: - dataBytes := []byte(` - // - // Create some data our manifest will point to. Data that could be very big and wouldn't fit in a feed update. - // So what we are going to do is upload it to Swarm bzz:// and obtain a **manifest hash** pointing to it: - // - // MANIFEST HASH --> DATA - // - // Then, we store that **manifest hash** into a Swarm Feed update. Once we have done this, - // we can use the **feed manifest hash** in bzz:// instead, this way: bzz://feed-manifest-hash. - // - // FEED MANIFEST HASH --> MANIFEST HASH --> DATA - // - // Given that we can update the feed at any time with a new **manifest hash** but the **feed manifest hash** - // stays constant, we have effectively created a fixed address to changing content. (Applause) - // - // FEED MANIFEST HASH (the same) --> MANIFEST HASH(2) --> DATA(2) - // - `) - - // Create a virtual File out of memory containing the above data - f := &File{ - ReadCloser: ioutil.NopCloser(bytes.NewReader(dataBytes)), - ManifestEntry: api.ManifestEntry{ - ContentType: "text/plain", - Mode: 0660, - Size: int64(len(dataBytes)), - }, - } - - // upload data to bzz:// and retrieve the content-addressed manifest hash, hex-encoded. - manifestAddressHex, err := swarmClient.Upload(f, "", false) - if err != nil { - t.Fatalf("Error creating manifest: %s", err) - } - - // convert the hex-encoded manifest hash to a 32-byte slice - manifestAddress := common.FromHex(manifestAddressHex) - - if len(manifestAddress) != storage.AddressLength { - t.Fatalf("Something went wrong. Got a hash of an unexpected length. Expected %d bytes. Got %d", storage.AddressLength, len(manifestAddress)) - } - - // Now create a **feed manifest**. For that, we need a topic: - topic, _ := feed.NewTopic("interesting topic indeed", nil) - - // Build a feed request to update data - request := feed.NewFirstRequest(topic) - - // Put the 32-byte address of the manifest into the feed update - request.SetData(manifestAddress) - - // Sign the update - if err := request.Sign(signer); err != nil { - t.Fatalf("Error signing update: %s", err) - } - - // Publish the update and at the same time request a **feed manifest** to be created - feedManifestAddressHex, err := swarmClient.CreateFeedWithManifest(request) - if err != nil { - t.Fatalf("Error creating feed manifest: %s", err) - } - - // Check we have received the exact **feed manifest** to be expected - // given the topic and user signing the updates: - correctFeedManifestAddrHex := "747c402e5b9dc715a25a4393147512167bab018a007fad7cdcd9adc7fce1ced2" - if feedManifestAddressHex != correctFeedManifestAddrHex { - t.Fatalf("Response feed manifest mismatch, expected '%s', got '%s'", correctFeedManifestAddrHex, feedManifestAddressHex) - } - - // Check we get a not found error when trying to get feed updates with a made-up manifest - _, err = swarmClient.QueryFeed(nil, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb") - if err != ErrNoFeedUpdatesFound { - t.Fatalf("Expected to receive ErrNoFeedUpdatesFound error. Got: %s", err) - } - - // If we query the feed directly we should get **manifest hash** back: - reader, err := swarmClient.QueryFeed(nil, correctFeedManifestAddrHex) - if err != nil { - t.Fatalf("Error retrieving feed updates: %s", err) - } - defer reader.Close() - gotData, err := ioutil.ReadAll(reader) - if err != nil { - t.Fatal(err) - } - - //Check that indeed the **manifest hash** is retrieved - if !bytes.Equal(manifestAddress, gotData) { - t.Fatalf("Expected: %v, got %v", manifestAddress, gotData) - } - - // Now the final test we were looking for: Use bzz://<feed-manifest> and that should resolve all manifests - // and return the original data directly: - f, err = swarmClient.Download(feedManifestAddressHex, "") - if err != nil { - t.Fatal(err) - } - gotData, err = ioutil.ReadAll(f) - if err != nil { - t.Fatal(err) - } - - // Check that we get back the original data: - if !bytes.Equal(dataBytes, gotData) { - t.Fatalf("Expected: %v, got %v", manifestAddress, gotData) - } -} - -// TestClientCreateUpdateFeed will check that feeds can be created and updated via the HTTP client. -func TestClientCreateUpdateFeed(t *testing.T) { - - signer, _ := newTestSigner() - - srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil) - client := NewClient(srv.URL) - defer srv.Close() - - // set raw data for the feed update - databytes := []byte("En un lugar de La Mancha, de cuyo nombre no quiero acordarme...") - - // our feed topic name - topic, _ := feed.NewTopic("El Quijote", nil) - createRequest := feed.NewFirstRequest(topic) - - createRequest.SetData(databytes) - if err := createRequest.Sign(signer); err != nil { - t.Fatalf("Error signing update: %s", err) - } - - feedManifestHash, err := client.CreateFeedWithManifest(createRequest) - if err != nil { - t.Fatal(err) - } - - correctManifestAddrHex := "0e9b645ebc3da167b1d56399adc3276f7a08229301b72a03336be0e7d4b71882" - if feedManifestHash != correctManifestAddrHex { - t.Fatalf("Response feed manifest mismatch, expected '%s', got '%s'", correctManifestAddrHex, feedManifestHash) - } - - reader, err := client.QueryFeed(nil, correctManifestAddrHex) - if err != nil { - t.Fatalf("Error retrieving feed updates: %s", err) - } - defer reader.Close() - gotData, err := ioutil.ReadAll(reader) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(databytes, gotData) { - t.Fatalf("Expected: %v, got %v", databytes, gotData) - } - - // define different data - databytes = []byte("... no ha mucho tiempo que vivĂa un hidalgo de los de lanza en astillero ...") - - updateRequest, err := client.GetFeedRequest(nil, correctManifestAddrHex) - if err != nil { - t.Fatalf("Error retrieving update request template: %s", err) - } - - updateRequest.SetData(databytes) - if err := updateRequest.Sign(signer); err != nil { - t.Fatalf("Error signing update: %s", err) - } - - if err = client.UpdateFeed(updateRequest); err != nil { - t.Fatalf("Error updating feed: %s", err) - } - - reader, err = client.QueryFeed(nil, correctManifestAddrHex) - if err != nil { - t.Fatalf("Error retrieving feed updates: %s", err) - } - defer reader.Close() - gotData, err = ioutil.ReadAll(reader) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(databytes, gotData) { - t.Fatalf("Expected: %v, got %v", databytes, gotData) - } - - // now try retrieving feed updates without a manifest - - fd := &feed.Feed{ - Topic: topic, - User: signer.Address(), - } - - lookupParams := feed.NewQueryLatest(fd, lookup.NoClue) - reader, err = client.QueryFeed(lookupParams, "") - if err != nil { - t.Fatalf("Error retrieving feed updates: %s", err) - } - defer reader.Close() - gotData, err = ioutil.ReadAll(reader) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(databytes, gotData) { - t.Fatalf("Expected: %v, got %v", databytes, gotData) - } -} |