swarm/api: refactor and improve HTTP API (#3773)

This PR deprecates the file related RPC calls in favour of an improved HTTP API.

The main aim is to expose a simple to use API which can be consumed by thin
clients (e.g. curl and HTML forms) without the need for complex logic (e.g.
manipulating prefix trie manifests).

2 files changed, 588 insertions, 223 deletions

diff --git a/swarm/api/client/client.go b/swarm/api/client/client.go
index ef5335be3..f9c3e51e8 100644
--- a/swarm/api/client/client.go
+++ b/swarm/api/client/client.go
@@ -17,18 +17,23 @@
 package client
 
 import (
+	"archive/tar"
 	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"mime"
+	"mime/multipart"
 	"net/http"
+	"net/textproto"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 
-	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/swarm/api"
 )
 
 var (
@@ -36,18 +41,6 @@ var (
 	DefaultClient  = NewClient(DefaultGateway)
 )
 
-// Manifest represents a swarm manifest.
-type Manifest struct {
-	Entries []ManifestEntry `json:"entries,omitempty"`
-}
-
-// ManifestEntry represents an entry in a swarm manifest.
-type ManifestEntry struct {
-	Hash        string `json:"hash,omitempty"`
-	ContentType string `json:"contentType,omitempty"`
-	Path        string `json:"path,omitempty"`
-}
-
 func NewClient(gateway string) *Client {
 	return &Client{
 		Gateway: gateway,
@@ -59,160 +52,207 @@ type Client struct {
 	Gateway string
 }
 
-func (c *Client) UploadDirectory(dir string, defaultPath string) (string, error) {
-	mhash, err := c.postRaw("application/json", 2, ioutil.NopCloser(bytes.NewReader([]byte("{}"))))
-	if err != nil {
-		return "", fmt.Errorf("failed to upload empty manifest")
+// UploadRaw uploads raw data to swarm and returns the resulting hash
+func (c *Client) UploadRaw(r io.Reader, size int64) (string, error) {
+	if size <= 0 {
+		return "", errors.New("data size must be greater than zero")
 	}
-	if len(defaultPath) > 0 {
-		fi, err := os.Stat(defaultPath)
-		if err != nil {
-			return "", err
-		}
-		mhash, err = c.uploadToManifest(mhash, "", defaultPath, fi)
-		if err != nil {
-			return "", err
-		}
+	req, err := http.NewRequest("POST", c.Gateway+"/bzzr:/", r)
+	if err != nil {
+		return "", err
 	}
-	prefix := filepath.ToSlash(filepath.Clean(dir)) + "/"
-	err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
-		if err != nil || fi.IsDir() {
-			return err
-		}
-		if !strings.HasPrefix(path, dir) {
-			return fmt.Errorf("path %s outside directory %s", path, dir)
-		}
-		uripath := strings.TrimPrefix(filepath.ToSlash(filepath.Clean(path)), prefix)
-		mhash, err = c.uploadToManifest(mhash, uripath, path, fi)
-		return err
-	})
-	return mhash, err
-}
-
-func (c *Client) UploadFile(file string, fi os.FileInfo, mimetype_hint string) (ManifestEntry, error) {
-	var mimetype string
-	hash, err := c.uploadFileContent(file, fi)
-	if mimetype_hint != "" {
-		mimetype = mimetype_hint
-		log.Info("Mime type set by override", "mime", mimetype)
-	} else {
-		ext := filepath.Ext(file)
-		log.Info("Ext", "ext", ext, "file", file)
-		if ext != "" {
-			mimetype = mime.TypeByExtension(filepath.Ext(fi.Name()))
-			log.Info("Mime type set by fileextension", "mime", mimetype, "ext", filepath.Ext(file))
-		} else {
-			f, err := os.Open(file)
-			if err == nil {
-				first512 := make([]byte, 512)
-				fread, _ := f.ReadAt(first512, 0)
-				if fread > 0 {
-					mimetype = http.DetectContentType(first512[:fread])
-					log.Info("Mime type set by autodetection", "mime", mimetype)
-				}
-			}
-			f.Close()
-		}
-
+	req.ContentLength = size
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
 	}
-	m := ManifestEntry{
-		Hash:        hash,
-		ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())),
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("unexpected HTTP status: %s", res.Status)
 	}
-	return m, err
-}
-
-func (c *Client) uploadFileContent(file string, fi os.FileInfo) (string, error) {
-	fd, err := os.Open(file)
+	data, err := ioutil.ReadAll(res.Body)
 	if err != nil {
 		return "", err
 	}
-	defer fd.Close()
-	log.Info("Uploading swarm content", "file", file, "bytes", fi.Size())
-	return c.postRaw("application/octet-stream", fi.Size(), fd)
+	return string(data), nil
 }
 
-func (c *Client) UploadManifest(m Manifest) (string, error) {
-	jsm, err := json.Marshal(m)
+// DownloadRaw downloads raw data from swarm
+func (c *Client) DownloadRaw(hash string) (io.ReadCloser, error) {
+	uri := c.Gateway + "/bzzr:/" + hash
+	res, err := http.DefaultClient.Get(uri)
 	if err != nil {
-		panic(err)
+		return nil, err
+	}
+	if res.StatusCode != http.StatusOK {
+		res.Body.Close()
+		return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status)
 	}
-	log.Info("Uploading swarm manifest")
-	return c.postRaw("application/json", int64(len(jsm)), ioutil.NopCloser(bytes.NewReader(jsm)))
+	return res.Body, nil
 }
 
-func (c *Client) uploadToManifest(mhash string, path string, fpath string, fi os.FileInfo) (string, error) {
-	fd, err := os.Open(fpath)
-	if err != nil {
-		return "", err
-	}
-	defer fd.Close()
-	log.Info("Uploading swarm content and path", "file", fpath, "bytes", fi.Size(), "path", path)
-	req, err := http.NewRequest("PUT", c.Gateway+"/bzz:/"+mhash+"/"+path, fd)
+// File represents a file in a swarm manifest and is used for uploading and
+// downloading content to and from swarm
+type File struct {
+	io.ReadCloser
+	api.ManifestEntry
+}
+
+// Open opens a local file which can then be passed to client.Upload to upload
+// it to swarm
+func Open(path string) (*File, error) {
+	f, err := os.Open(path)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
-	req.Header.Set("content-type", mime.TypeByExtension(filepath.Ext(fi.Name())))
-	req.ContentLength = fi.Size()
-	resp, err := http.DefaultClient.Do(req)
+	stat, err := f.Stat()
 	if err != nil {
-		return "", err
+		f.Close()
+		return nil, err
 	}
-	defer resp.Body.Close()
-	if resp.StatusCode >= 400 {
-		return "", fmt.Errorf("bad status: %s", resp.Status)
+	return &File{
+		ReadCloser: f,
+		ManifestEntry: api.ManifestEntry{
+			ContentType: mime.TypeByExtension(filepath.Ext(path)),
+			Mode:        int64(stat.Mode()),
+			Size:        stat.Size(),
+			ModTime:     stat.ModTime(),
+		},
+	}, nil
+}
+
+// Upload uploads a file to swarm and either adds it to an existing manifest
+// (if the manifest argument is non-empty) or creates a new manifest containing
+// the file, returning the resulting manifest hash (the file will then be
+// available at bzz:/<hash>/<path>)
+func (c *Client) Upload(file *File, manifest string) (string, error) {
+	if file.Size <= 0 {
+		return "", errors.New("file size must be greater than zero")
 	}
-	content, err := ioutil.ReadAll(resp.Body)
-	return string(content), err
+	return c.TarUpload(manifest, &FileUploader{file})
 }
 
-func (c *Client) postRaw(mimetype string, size int64, body io.ReadCloser) (string, error) {
-	req, err := http.NewRequest("POST", c.Gateway+"/bzzr:/", body)
+// Download downloads a file with the given path from the swarm manifest with
+// the given hash (i.e. it gets bzz:/<hash>/<path>)
+func (c *Client) Download(hash, path string) (*File, error) {
+	uri := c.Gateway + "/bzz:/" + hash + "/" + path
+	res, err := http.DefaultClient.Get(uri)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
-	req.Header.Set("content-type", mimetype)
-	req.ContentLength = size
-	resp, err := http.DefaultClient.Do(req)
+	if res.StatusCode != http.StatusOK {
+		res.Body.Close()
+		return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status)
+	}
+	return &File{
+		ReadCloser: res.Body,
+		ManifestEntry: api.ManifestEntry{
+			ContentType: res.Header.Get("Content-Type"),
+			Size:        res.ContentLength,
+		},
+	}, nil
+}
+
+// UploadDirectory uploads a directory tree to swarm and either adds the files
+// to an existing manifest (if the manifest argument is non-empty) or creates a
+// new manifest, returning the resulting manifest hash (files from the
+// directory will then be available at bzz:/<hash>/path/to/file), with
+// the file specified in defaultPath being uploaded to the root of the manifest
+// (i.e. bzz:/<hash>/)
+func (c *Client) UploadDirectory(dir, defaultPath, manifest string) (string, error) {
+	stat, err := os.Stat(dir)
 	if err != nil {
 		return "", err
+	} else if !stat.IsDir() {
+		return "", fmt.Errorf("not a directory: %s", dir)
 	}
-	defer resp.Body.Close()
-	if resp.StatusCode >= 400 {
-		return "", fmt.Errorf("bad status: %s", resp.Status)
-	}
-	content, err := ioutil.ReadAll(resp.Body)
-	return string(content), err
+	return c.TarUpload(manifest, &DirectoryUploader{dir, defaultPath})
 }
 
-func (c *Client) DownloadManifest(mhash string) (Manifest, error) {
+// DownloadDirectory downloads the files contained in a swarm manifest under
+// the given path into a local directory (existing files will be overwritten)
+func (c *Client) DownloadDirectory(hash, path, destDir string) error {
+	stat, err := os.Stat(destDir)
+	if err != nil {
+		return err
+	} else if !stat.IsDir() {
+		return fmt.Errorf("not a directory: %s", destDir)
+	}
 
-	mroot := Manifest{}
-	req, err := http.NewRequest("GET", c.Gateway+"/bzzr:/"+mhash, nil)
+	uri := c.Gateway + "/bzz:/" + hash + "/" + path
+	req, err := http.NewRequest("GET", uri, nil)
 	if err != nil {
-		return mroot, err
+		return err
 	}
-	resp, err := http.DefaultClient.Do(req)
+	req.Header.Set("Accept", "application/x-tar")
+	res, err := http.DefaultClient.Do(req)
 	if err != nil {
-		return mroot, err
+		return err
 	}
-	defer resp.Body.Close()
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		return fmt.Errorf("unexpected HTTP status: %s", res.Status)
+	}
+	tr := tar.NewReader(res.Body)
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			return nil
+		} else if err != nil {
+			return err
+		}
+		// ignore the default path file
+		if hdr.Name == "" {
+			continue
+		}
 
-	if resp.StatusCode >= 400 {
-		return mroot, fmt.Errorf("bad status: %s", resp.Status)
+		dstPath := filepath.Join(destDir, filepath.Clean(strings.TrimPrefix(hdr.Name, path)))
+		if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil {
+			return err
+		}
+		var mode os.FileMode = 0644
+		if hdr.Mode > 0 {
+			mode = os.FileMode(hdr.Mode)
+		}
+		dst, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode)
+		if err != nil {
+			return err
+		}
+		n, err := io.Copy(dst, tr)
+		dst.Close()
+		if err != nil {
+			return err
+		} else if n != hdr.Size {
+			return fmt.Errorf("expected %s to be %d bytes but got %d", hdr.Name, hdr.Size, n)
+		}
+	}
+}
 
+// UploadManifest uploads the given manifest to swarm
+func (c *Client) UploadManifest(m *api.Manifest) (string, error) {
+	data, err := json.Marshal(m)
+	if err != nil {
+		return "", err
 	}
-	content, err := ioutil.ReadAll(resp.Body)
+	return c.UploadRaw(bytes.NewReader(data), int64(len(data)))
+}
 
-	err = json.Unmarshal(content, &mroot)
+// DownloadManifest downloads a swarm manifest
+func (c *Client) DownloadManifest(hash string) (*api.Manifest, error) {
+	res, err := c.DownloadRaw(hash)
 	if err != nil {
-		return mroot, fmt.Errorf("Manifest %v is malformed: %v", mhash, err)
+		return nil, err
+	}
+	defer res.Close()
+	var manifest api.Manifest
+	if err := json.NewDecoder(res).Decode(&manifest); err != nil {
+		return nil, err
 	}
-	return mroot, err
+	return &manifest, nil
 }
 
-// ManifestFileList downloads the manifest with the given hash and generates a
-// list of files and directory prefixes which have the specified prefix.
+// List list files in a swarm manifest which have the given prefix, grouping
+// common prefixes using "/" as a delimiter.
 //
 // For example, if the manifest represents the following directory structure:
 //
@@ -226,97 +266,200 @@ func (c *Client) DownloadManifest(mhash string) (Manifest, error) {
 // - a prefix of ""      would return [dir1/, file1.txt, file2.txt]
 // - a prefix of "file"  would return [file1.txt, file2.txt]
 // - a prefix of "dir1/" would return [dir1/dir2/, dir1/file3.txt]
-func (c *Client) ManifestFileList(hash, prefix string) (entries []ManifestEntry, err error) {
-	manifest, err := c.DownloadManifest(hash)
+//
+// where entries ending with "/" are common prefixes.
+func (c *Client) List(hash, prefix string) (*api.ManifestList, error) {
+	res, err := http.DefaultClient.Get(c.Gateway + "/bzz:/" + hash + "/" + prefix + "?list=true")
 	if err != nil {
 		return nil, err
 	}
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status)
+	}
+	var list api.ManifestList
+	if err := json.NewDecoder(res.Body).Decode(&list); err != nil {
+		return nil, err
+	}
+	return &list, nil
+}
+
+// Uploader uploads files to swarm using a provided UploadFn
+type Uploader interface {
+	Upload(UploadFn) error
+}
+
+type UploaderFunc func(UploadFn) error
+
+func (u UploaderFunc) Upload(upload UploadFn) error {
+	return u(upload)
+}
+
+// DirectoryUploader uploads all files in a directory, optionally uploading
+// a file to the default path
+type DirectoryUploader struct {
+	Dir         string
+	DefaultPath string
+}
 
-	// handleFile handles a manifest entry which is a direct reference to a
-	// file (i.e. it is not a swarm manifest)
-	handleFile := func(entry ManifestEntry) {
-		// ignore the file if it doesn't have the specified prefix
-		if !strings.HasPrefix(entry.Path, prefix) {
-			return
+// Upload performs the upload of the directory and default path
+func (d *DirectoryUploader) Upload(upload UploadFn) error {
+	if d.DefaultPath != "" {
+		file, err := Open(d.DefaultPath)
+		if err != nil {
+			return err
 		}
-		// if the path after the prefix contains a directory separator,
-		// add a directory prefix to the entries, otherwise add the
-		// file
-		suffix := strings.TrimPrefix(entry.Path, prefix)
-		if sepIndex := strings.Index(suffix, "/"); sepIndex > -1 {
-			entries = append(entries, ManifestEntry{
-				Path:        prefix + suffix[:sepIndex+1],
-				ContentType: "DIR",
-			})
-		} else {
-			if entry.Path == "" {
-				entry.Path = "/"
-			}
-			entries = append(entries, entry)
+		if err := upload(file); err != nil {
+			return err
 		}
 	}
-
-	// handleManifest handles a manifest entry which is a reference to
-	// another swarm manifest.
-	handleManifest := func(entry ManifestEntry) error {
-		// if the manifest's path is a prefix of the specified prefix
-		// then just recurse into the manifest by stripping its path
-		// from the prefix
-		if strings.HasPrefix(prefix, entry.Path) {
-			subPrefix := strings.TrimPrefix(prefix, entry.Path)
-			subEntries, err := c.ManifestFileList(entry.Hash, subPrefix)
-			if err != nil {
-				return err
-			}
-			// prefix the manifest's path to the sub entries and
-			// add them to the returned entries
-			for i, subEntry := range subEntries {
-				subEntry.Path = entry.Path + subEntry.Path
-				subEntries[i] = subEntry
-			}
-			entries = append(entries, subEntries...)
+	return filepath.Walk(d.Dir, func(path string, f os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if f.IsDir() {
 			return nil
 		}
+		file, err := Open(path)
+		if err != nil {
+			return err
+		}
+		relPath, err := filepath.Rel(d.Dir, path)
+		if err != nil {
+			return err
+		}
+		file.Path = filepath.ToSlash(relPath)
+		return upload(file)
+	})
+}
 
-		// if the manifest's path has the specified prefix, then if the
-		// path after the prefix contains a directory separator, add a
-		// directory prefix to the entries, otherwise recurse into the
-		// manifest
-		if strings.HasPrefix(entry.Path, prefix) {
-			suffix := strings.TrimPrefix(entry.Path, prefix)
-			sepIndex := strings.Index(suffix, "/")
-			if sepIndex > -1 {
-				entries = append(entries, ManifestEntry{
-					Path:        prefix + suffix[:sepIndex+1],
-					ContentType: "DIR",
-				})
-				return nil
-			}
-			subEntries, err := c.ManifestFileList(entry.Hash, "")
-			if err != nil {
-				return err
-			}
-			// prefix the manifest's path to the sub entries and
-			// add them to the returned entries
-			for i, subEntry := range subEntries {
-				subEntry.Path = entry.Path + subEntry.Path
-				subEntries[i] = subEntry
-			}
-			entries = append(entries, subEntries...)
-			return nil
+// FileUploader uploads a single file
+type FileUploader struct {
+	File *File
+}
+
+// Upload performs the upload of the file
+func (f *FileUploader) Upload(upload UploadFn) error {
+	return upload(f.File)
+}
+
+// UploadFn is the type of function passed to an Uploader to perform the upload
+// of a single file (for example, a directory uploader would call a provided
+// UploadFn for each file in the directory tree)
+type UploadFn func(file *File) error
+
+// TarUpload uses the given Uploader to upload files to swarm as a tar stream,
+// returning the resulting manifest hash
+func (c *Client) TarUpload(hash string, uploader Uploader) (string, error) {
+	reqR, reqW := io.Pipe()
+	defer reqR.Close()
+	req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+hash, reqR)
+	if err != nil {
+		return "", err
+	}
+	req.Header.Set("Content-Type", "application/x-tar")
+
+	// use 'Expect: 100-continue' so we don't send the request body if
+	// the server refuses the request
+	req.Header.Set("Expect", "100-continue")
+
+	tw := tar.NewWriter(reqW)
+
+	// define an UploadFn which adds files to the tar stream
+	uploadFn := func(file *File) error {
+		hdr := &tar.Header{
+			Name:    file.Path,
+			Mode:    file.Mode,
+			Size:    file.Size,
+			ModTime: file.ModTime,
+			Xattrs: map[string]string{
+				"user.swarm.content-type": file.ContentType,
+			},
+		}
+		if err := tw.WriteHeader(hdr); err != nil {
+			return err
 		}
-		return nil
+		_, err = io.Copy(tw, file)
+		return err
 	}
 
-	for _, entry := range manifest.Entries {
-		if entry.ContentType == "application/bzz-manifest+json" {
-			if err := handleManifest(entry); err != nil {
-				return nil, err
-			}
-		} else {
-			handleFile(entry)
+	// run the upload in a goroutine so we can send the request headers and
+	// wait for a '100 Continue' response before sending the tar stream
+	go func() {
+		err := uploader.Upload(uploadFn)
+		if err == nil {
+			err = tw.Close()
 		}
+		reqW.CloseWithError(err)
+	}()
+
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("unexpected HTTP status: %s", res.Status)
+	}
+	data, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return "", err
+	}
+	return string(data), nil
+}
+
+// MultipartUpload uses the given Uploader to upload files to swarm as a
+// multipart form, returning the resulting manifest hash
+func (c *Client) MultipartUpload(hash string, uploader Uploader) (string, error) {
+	reqR, reqW := io.Pipe()
+	defer reqR.Close()
+	req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+hash, reqR)
+	if err != nil {
+		return "", err
 	}
 
-	return
+	// use 'Expect: 100-continue' so we don't send the request body if
+	// the server refuses the request
+	req.Header.Set("Expect", "100-continue")
+
+	mw := multipart.NewWriter(reqW)
+	req.Header.Set("Content-Type", fmt.Sprintf("multipart/form-data; boundary=%q", mw.Boundary()))
+
+	// define an UploadFn which adds files to the multipart form
+	uploadFn := func(file *File) error {
+		hdr := make(textproto.MIMEHeader)
+		hdr.Set("Content-Disposition", fmt.Sprintf("form-data; name=%q", file.Path))
+		hdr.Set("Content-Type", file.ContentType)
+		hdr.Set("Content-Length", strconv.FormatInt(file.Size, 10))
+		w, err := mw.CreatePart(hdr)
+		if err != nil {
+			return err
+		}
+		_, err = io.Copy(w, file)
+		return err
+	}
+
+	// run the upload in a goroutine so we can send the request headers and
+	// wait for a '100 Continue' response before sending the multipart form
+	go func() {
+		err := uploader.Upload(uploadFn)
+		if err == nil {
+			err = mw.Close()
+		}
+		reqW.CloseWithError(err)
+	}()
+
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("unexpected HTTP status: %s", res.Status)
+	}
+	data, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return "", err
+	}
+	return string(data), nil
 }
diff --git a/swarm/api/client/client_test.go b/swarm/api/client/client_test.go
index 135474475..4d02ceaf4 100644
--- a/swarm/api/client/client_test.go
+++ b/swarm/api/client/client_test.go
@@ -17,6 +17,7 @@
 package client
 
 import (
+	"bytes"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -24,52 +25,221 @@ import (
 	"sort"
 	"testing"
 
+	"github.com/ethereum/go-ethereum/swarm/api"
 	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
-func TestClientManifestFileList(t *testing.T) {
+// TestClientUploadDownloadRaw test uploading and downloading raw data to swarm
+func TestClientUploadDownloadRaw(t *testing.T) {
 	srv := testutil.NewTestSwarmServer(t)
 	defer srv.Close()
 
+	client := NewClient(srv.URL)
+
+	// upload some raw data
+	data := []byte("foo123")
+	hash, err := client.UploadRaw(bytes.NewReader(data), int64(len(data)))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// check we can download the same data
+	res, err := client.DownloadRaw(hash)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer res.Close()
+	gotData, err := ioutil.ReadAll(res)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(gotData, data) {
+		t.Fatalf("expected downloaded data to be %q, got %q", data, gotData)
+	}
+}
+
+// TestClientUploadDownloadFiles test uploading and downloading files to swarm
+// manifests
+func TestClientUploadDownloadFiles(t *testing.T) {
+	srv := testutil.NewTestSwarmServer(t)
+	defer srv.Close()
+
+	client := NewClient(srv.URL)
+	upload := func(manifest, path string, data []byte) string {
+		file := &File{
+			ReadCloser: ioutil.NopCloser(bytes.NewReader(data)),
+			ManifestEntry: api.ManifestEntry{
+				Path:        path,
+				ContentType: "text/plain",
+				Size:        int64(len(data)),
+			},
+		}
+		hash, err := client.Upload(file, manifest)
+		if err != nil {
+			t.Fatal(err)
+		}
+		return hash
+	}
+	checkDownload := func(manifest, path string, expected []byte) {
+		file, err := client.Download(manifest, path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer file.Close()
+		if file.Size != int64(len(expected)) {
+			t.Fatalf("expected downloaded file to be %d bytes, got %d", len(expected), file.Size)
+		}
+		if file.ContentType != file.ContentType {
+			t.Fatalf("expected downloaded file to have type %q, got %q", file.ContentType, file.ContentType)
+		}
+		data, err := ioutil.ReadAll(file)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !bytes.Equal(data, expected) {
+			t.Fatalf("expected downloaded data to be %q, got %q", expected, data)
+		}
+	}
+
+	// upload a file to the root of a manifest
+	rootData := []byte("some-data")
+	rootHash := upload("", "", rootData)
+
+	// check we can download the root file
+	checkDownload(rootHash, "", rootData)
+
+	// upload another file to the same manifest
+	otherData := []byte("some-other-data")
+	newHash := upload(rootHash, "some/other/path", otherData)
+
+	// check we can download both files from the new manifest
+	checkDownload(newHash, "", rootData)
+	checkDownload(newHash, "some/other/path", otherData)
+
+	// replace the root file with different data
+	newHash = upload(newHash, "", otherData)
+
+	// check both files have the other data
+	checkDownload(newHash, "", otherData)
+	checkDownload(newHash, "some/other/path", otherData)
+}
+
+var testDirFiles = []string{
+	"file1.txt",
+	"file2.txt",
+	"dir1/file3.txt",
+	"dir1/file4.txt",
+	"dir2/file5.txt",
+	"dir2/dir3/file6.txt",
+	"dir2/dir4/file7.txt",
+	"dir2/dir4/file8.txt",
+}
+
+func newTestDirectory(t *testing.T) string {
 	dir, err := ioutil.TempDir("", "swarm-client-test")
 	if err != nil {
 		t.Fatal(err)
 	}
-	files := []string{
-		"file1.txt",
-		"file2.txt",
-		"dir1/file3.txt",
-		"dir1/file4.txt",
-		"dir2/file5.txt",
-		"dir2/dir3/file6.txt",
-		"dir2/dir4/file7.txt",
-		"dir2/dir4/file8.txt",
-	}
-	for _, file := range files {
+
+	for _, file := range testDirFiles {
 		path := filepath.Join(dir, file)
 		if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+			os.RemoveAll(dir)
 			t.Fatalf("error creating dir for %s: %s", path, err)
 		}
-		if err := ioutil.WriteFile(path, []byte("data"), 0644); err != nil {
+		if err := ioutil.WriteFile(path, []byte(file), 0644); err != nil {
+			os.RemoveAll(dir)
 			t.Fatalf("error writing file %s: %s", path, err)
 		}
 	}
 
+	return dir
+}
+
+// TestClientUploadDownloadDirectory tests uploading and downloading a
+// directory of files to a swarm manifest
+func TestClientUploadDownloadDirectory(t *testing.T) {
+	srv := testutil.NewTestSwarmServer(t)
+	defer srv.Close()
+
+	dir := newTestDirectory(t)
+	defer os.RemoveAll(dir)
+
+	// upload the directory
 	client := NewClient(srv.URL)
+	defaultPath := filepath.Join(dir, testDirFiles[0])
+	hash, err := client.UploadDirectory(dir, defaultPath, "")
+	if err != nil {
+		t.Fatalf("error uploading directory: %s", err)
+	}
+
+	// check we can download the individual files
+	checkDownloadFile := func(path string, expected []byte) {
+		file, err := client.Download(hash, path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer file.Close()
+		data, err := ioutil.ReadAll(file)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !bytes.Equal(data, expected) {
+			t.Fatalf("expected data to be %q, got %q", expected, data)
+		}
+	}
+	for _, file := range testDirFiles {
+		checkDownloadFile(file, []byte(file))
+	}
+
+	// check we can download the default path
+	checkDownloadFile("", []byte(testDirFiles[0]))
+
+	// check we can download the directory
+	tmp, err := ioutil.TempDir("", "swarm-client-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmp)
+	if err := client.DownloadDirectory(hash, "", tmp); err != nil {
+		t.Fatal(err)
+	}
+	for _, file := range testDirFiles {
+		data, err := ioutil.ReadFile(filepath.Join(tmp, file))
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !bytes.Equal(data, []byte(file)) {
+			t.Fatalf("expected data to be %q, got %q", file, data)
+		}
+	}
+}
+
+// TestClientFileList tests listing files in a swarm manifest
+func TestClientFileList(t *testing.T) {
+	srv := testutil.NewTestSwarmServer(t)
+	defer srv.Close()
 
-	hash, err := client.UploadDirectory(dir, "")
+	dir := newTestDirectory(t)
+	defer os.RemoveAll(dir)
+
+	client := NewClient(srv.URL)
+	hash, err := client.UploadDirectory(dir, "", "")
 	if err != nil {
 		t.Fatalf("error uploading directory: %s", err)
 	}
 
 	ls := func(prefix string) []string {
-		entries, err := client.ManifestFileList(hash, prefix)
+		list, err := client.List(hash, prefix)
 		if err != nil {
 			t.Fatal(err)
 		}
-		paths := make([]string, len(entries))
-		for i, entry := range entries {
-			paths[i] = entry.Path
+		paths := make([]string, 0, len(list.CommonPrefixes)+len(list.Entries))
+		for _, prefix := range list.CommonPrefixes {
+			paths = append(paths, prefix)
+		}
+		for _, entry := range list.Entries {
+			paths = append(paths, entry.Path)
 		}
 		sort.Strings(paths)
 		return paths
@@ -99,7 +269,59 @@ func TestClientManifestFileList(t *testing.T) {
 	for prefix, expected := range tests {
 		actual := ls(prefix)
 		if !reflect.DeepEqual(actual, expected) {
-			t.Fatalf("expected prefix %q to return paths %v, got %v", prefix, expected, actual)
+			t.Fatalf("expected prefix %q to return %v, got %v", prefix, expected, actual)
+		}
+	}
+}
+
+// TestClientMultipartUpload tests uploading files to swarm using a multipart
+// upload
+func TestClientMultipartUpload(t *testing.T) {
+	srv := testutil.NewTestSwarmServer(t)
+	defer srv.Close()
+
+	// define an uploader which uploads testDirFiles with some data
+	data := []byte("some-data")
+	uploader := UploaderFunc(func(upload UploadFn) error {
+		for _, name := range testDirFiles {
+			file := &File{
+				ReadCloser: ioutil.NopCloser(bytes.NewReader(data)),
+				ManifestEntry: api.ManifestEntry{
+					Path:        name,
+					ContentType: "text/plain",
+					Size:        int64(len(data)),
+				},
+			}
+			if err := upload(file); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+
+	// upload the files as a multipart upload
+	client := NewClient(srv.URL)
+	hash, err := client.MultipartUpload("", uploader)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// check we can download the individual files
+	checkDownloadFile := func(path string) {
+		file, err := client.Download(hash, path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer file.Close()
+		gotData, err := ioutil.ReadAll(file)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !bytes.Equal(gotData, data) {
+			t.Fatalf("expected data to be %q, got %q", data, gotData)
 		}
 	}
+	for _, file := range testDirFiles {
+		checkDownloadFile(file)
+	}
 }