cmd/swarm, swarm/api/client: add HTTP API client and 'swarm ls' command (#3742)

This adds a swarm ls command which lists files and directories stored in a
manifest. Rather than listing all files, it uses "directory prefixes" in case there are a
lot of files in a manifest but you just want to traverse it.

This also includes some refactoring to the tests and the introduction of a
swarm/api/client package to make things easier to test.

4 files changed, 99 insertions, 182 deletions

diff --git a/cmd/swarm/list.go b/cmd/swarm/list.go
new file mode 100644
index 000000000..3a68fef03
--- /dev/null
+++ b/cmd/swarm/list.go
@@ -0,0 +1,58 @@
+// Copyright 2016 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"text/tabwriter"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+	"gopkg.in/urfave/cli.v1"
+)
+
+func list(ctx *cli.Context) {
+	args := ctx.Args()
+
+	if len(args) < 1 {
+		utils.Fatalf("Please supply a manifest reference as the first argument")
+	} else if len(args) > 2 {
+		utils.Fatalf("Too many arguments - usage 'swarm ls manifest [prefix]'")
+	}
+	manifest := args[0]
+
+	var prefix string
+	if len(args) == 2 {
+		prefix = args[1]
+	}
+
+	bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+	client := swarm.NewClient(bzzapi)
+	entries, err := client.ManifestFileList(manifest, prefix)
+	if err != nil {
+		utils.Fatalf("Failed to generate file and directory list: %s", err)
+	}
+
+	w := tabwriter.NewWriter(os.Stdout, 1, 2, 2, ' ', 0)
+	defer w.Flush()
+	fmt.Fprintln(w, "HASH\tCONTENT TYPE\tPATH")
+	for _, entry := range entries {
+		fmt.Fprintf(w, "%s\t%s\t%s\n", entry.Hash, entry.ContentType, entry.Path)
+	}
+}
diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go
index 3c82497e5..731c300f8 100644
--- a/cmd/swarm/main.go
+++ b/cmd/swarm/main.go
@@ -148,6 +148,15 @@ The output of this command is supposed to be machine-readable.
 `,
 		},
 		{
+			Action:    list,
+			Name:      "ls",
+			Usage:     "list files and directories contained in a manifest",
+			ArgsUsage: " <manifest> [<prefix>]",
+			Description: `
+Lists files and directories contained in a manifest.
+`,
+		},
+		{
 			Action:    hash,
 			Name:      "hash",
 			Usage:     "print the swarm hash of a file or directory",
diff --git a/cmd/swarm/manifest.go b/cmd/swarm/manifest.go
index 2b6b02313..698b8ddb8 100644
--- a/cmd/swarm/manifest.go
+++ b/cmd/swarm/manifest.go
@@ -25,6 +25,7 @@ import (
 	"strings"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
+	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -41,7 +42,7 @@ func add(ctx *cli.Context) {
 
 		ctype        string
 		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
-		mroot        manifest
+		mroot        swarm.Manifest
 	)
 
 	if len(args) > 3 {
@@ -75,7 +76,7 @@ func update(ctx *cli.Context) {
 
 		ctype        string
 		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
-		mroot        manifest
+		mroot        swarm.Manifest
 	)
 	if len(args) > 3 {
 		ctype = args[3]
@@ -105,7 +106,7 @@ func remove(ctx *cli.Context) {
 		path  = args[1]
 
 		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
-		mroot        manifest
+		mroot        swarm.Manifest
 	)
 
 	newManifest := removeEntryFromManifest(ctx, mhash, path)
@@ -123,21 +124,21 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 
 	var (
 		bzzapi           = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client           = &client{api: bzzapi}
-		longestPathEntry = manifestEntry{
+		client           = swarm.NewClient(bzzapi)
+		longestPathEntry = swarm.ManifestEntry{
 			Path:        "",
 			Hash:        "",
 			ContentType: "",
 		}
 	)
 
-	mroot, err := client.downloadManifest(mhash)
+	mroot, err := client.DownloadManifest(mhash)
 	if err != nil {
 		utils.Fatalf("Manifest download failed: %v", err)
 	}
 
 	//TODO: check if the "hash" to add is valid and present in swarm
-	_, err = client.downloadManifest(hash)
+	_, err = client.DownloadManifest(hash)
 	if err != nil {
 		utils.Fatalf("Hash to add is not present: %v", err)
 	}
@@ -162,7 +163,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 		newHash := addEntryToManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
 
 		// Replace the hash for parent Manifests
-		newMRoot := manifest{}
+		newMRoot := swarm.Manifest{}
 		for _, entry := range mroot.Entries {
 			if longestPathEntry.Path == entry.Path {
 				entry.Hash = newHash
@@ -172,7 +173,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 		mroot = newMRoot
 	} else {
 		// Add the entry in the leaf Manifest
-		newEntry := manifestEntry{
+		newEntry := swarm.ManifestEntry{
 			Path:        path,
 			Hash:        hash,
 			ContentType: ctype,
@@ -180,7 +181,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 		mroot.Entries = append(mroot.Entries, newEntry)
 	}
 
-	newManifestHash, err := client.uploadManifest(mroot)
+	newManifestHash, err := client.UploadManifest(mroot)
 	if err != nil {
 		utils.Fatalf("Manifest upload failed: %v", err)
 	}
@@ -192,20 +193,20 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
 
 	var (
 		bzzapi   = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client   = &client{api: bzzapi}
-		newEntry = manifestEntry{
+		client   = swarm.NewClient(bzzapi)
+		newEntry = swarm.ManifestEntry{
 			Path:        "",
 			Hash:        "",
 			ContentType: "",
 		}
-		longestPathEntry = manifestEntry{
+		longestPathEntry = swarm.ManifestEntry{
 			Path:        "",
 			Hash:        "",
 			ContentType: "",
 		}
 	)
 
-	mroot, err := client.downloadManifest(mhash)
+	mroot, err := client.DownloadManifest(mhash)
 	if err != nil {
 		utils.Fatalf("Manifest download failed: %v", err)
 	}
@@ -236,7 +237,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
 		newHash := updateEntryInManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
 
 		// Replace the hash for parent Manifests
-		newMRoot := manifest{}
+		newMRoot := swarm.Manifest{}
 		for _, entry := range mroot.Entries {
 			if longestPathEntry.Path == entry.Path {
 				entry.Hash = newHash
@@ -249,10 +250,10 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
 
 	if newEntry.Path != "" {
 		// Replace the hash for leaf Manifest
-		newMRoot := manifest{}
+		newMRoot := swarm.Manifest{}
 		for _, entry := range mroot.Entries {
 			if newEntry.Path == entry.Path {
-				myEntry := manifestEntry{
+				myEntry := swarm.ManifestEntry{
 					Path:        entry.Path,
 					Hash:        hash,
 					ContentType: ctype,
@@ -265,7 +266,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
 		mroot = newMRoot
 	}
 
-	newManifestHash, err := client.uploadManifest(mroot)
+	newManifestHash, err := client.UploadManifest(mroot)
 	if err != nil {
 		utils.Fatalf("Manifest upload failed: %v", err)
 	}
@@ -276,20 +277,20 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 
 	var (
 		bzzapi        = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client        = &client{api: bzzapi}
-		entryToRemove = manifestEntry{
+		client        = swarm.NewClient(bzzapi)
+		entryToRemove = swarm.ManifestEntry{
 			Path:        "",
 			Hash:        "",
 			ContentType: "",
 		}
-		longestPathEntry = manifestEntry{
+		longestPathEntry = swarm.ManifestEntry{
 			Path:        "",
 			Hash:        "",
 			ContentType: "",
 		}
 	)
 
-	mroot, err := client.downloadManifest(mhash)
+	mroot, err := client.DownloadManifest(mhash)
 	if err != nil {
 		utils.Fatalf("Manifest download failed: %v", err)
 	}
@@ -318,7 +319,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 		newHash := removeEntryFromManifest(ctx, longestPathEntry.Hash, newPath)
 
 		// Replace the hash for parent Manifests
-		newMRoot := manifest{}
+		newMRoot := swarm.Manifest{}
 		for _, entry := range mroot.Entries {
 			if longestPathEntry.Path == entry.Path {
 				entry.Hash = newHash
@@ -330,7 +331,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 
 	if entryToRemove.Path != "" {
 		// remove the entry in this Manifest
-		newMRoot := manifest{}
+		newMRoot := swarm.Manifest{}
 		for _, entry := range mroot.Entries {
 			if entryToRemove.Path != entry.Path {
 				newMRoot.Entries = append(newMRoot.Entries, entry)
@@ -339,7 +340,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 		mroot = newMRoot
 	}
 
-	newManifestHash, err := client.uploadManifest(mroot)
+	newManifestHash, err := client.UploadManifest(mroot)
 	if err != nil {
 		utils.Fatalf("Manifest upload failed: %v", err)
 	}
diff --git a/cmd/swarm/upload.go b/cmd/swarm/upload.go
index 7b4961778..696b907d2 100644
--- a/cmd/swarm/upload.go
+++ b/cmd/swarm/upload.go
@@ -18,21 +18,15 @@
 package main
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
-	"io"
-	"io/ioutil"
-	"mime"
-	"net/http"
 	"os"
 	"os/user"
 	"path"
-	"path/filepath"
 	"strings"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
-	"github.com/ethereum/go-ethereum/log"
+	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -50,7 +44,7 @@ func upload(ctx *cli.Context) {
 
 	var (
 		file   = args[0]
-		client = &client{api: bzzapi}
+		client = swarm.NewClient(bzzapi)
 	)
 	fi, err := os.Stat(expandPath(file))
 	if err != nil {
@@ -63,25 +57,25 @@ func upload(ctx *cli.Context) {
 		if !wantManifest {
 			utils.Fatalf("Manifest is required for directory uploads")
 		}
-		mhash, err := client.uploadDirectory(file, defaultPath)
+		mhash, err := client.UploadDirectory(file, defaultPath)
 		if err != nil {
 			utils.Fatalf("Failed to upload directory: %v", err)
 		}
 		fmt.Println(mhash)
 		return
 	}
-	entry, err := client.uploadFile(file, fi)
+	entry, err := client.UploadFile(file, fi)
 	if err != nil {
 		utils.Fatalf("Upload failed: %v", err)
 	}
-	mroot := manifest{[]manifestEntry{entry}}
+	mroot := swarm.Manifest{Entries: []swarm.ManifestEntry{entry}}
 	if !wantManifest {
 		// Print the manifest. This is the only output to stdout.
 		mrootJSON, _ := json.MarshalIndent(mroot, "", "  ")
 		fmt.Println(string(mrootJSON))
 		return
 	}
-	hash, err := client.uploadManifest(mroot)
+	hash, err := client.UploadManifest(mroot)
 	if err != nil {
 		utils.Fatalf("Manifest upload failed: %v", err)
 	}
@@ -111,148 +105,3 @@ func homeDir() string {
 	}
 	return ""
 }
-
-// client wraps interaction with the swarm HTTP gateway.
-type client struct {
-	api string
-}
-
-// manifest is the JSON representation of a swarm manifest.
-type manifestEntry struct {
-	Hash        string `json:"hash,omitempty"`
-	ContentType string `json:"contentType,omitempty"`
-	Path        string `json:"path,omitempty"`
-}
-
-// manifest is the JSON representation of a swarm manifest.
-type manifest struct {
-	Entries []manifestEntry `json:"entries,omitempty"`
-}
-
-func (c *client) uploadDirectory(dir string, defaultPath string) (string, error) {
-	mhash, err := c.postRaw("application/json", 2, ioutil.NopCloser(bytes.NewReader([]byte("{}"))))
-	if err != nil {
-		return "", fmt.Errorf("failed to upload empty manifest")
-	}
-	if len(defaultPath) > 0 {
-		fi, err := os.Stat(defaultPath)
-		if err != nil {
-			return "", err
-		}
-		mhash, err = c.uploadToManifest(mhash, "", defaultPath, fi)
-		if err != nil {
-			return "", err
-		}
-	}
-	prefix := filepath.ToSlash(filepath.Clean(dir)) + "/"
-	err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
-		if err != nil || fi.IsDir() {
-			return err
-		}
-		if !strings.HasPrefix(path, dir) {
-			return fmt.Errorf("path %s outside directory %s", path, dir)
-		}
-		uripath := strings.TrimPrefix(filepath.ToSlash(filepath.Clean(path)), prefix)
-		mhash, err = c.uploadToManifest(mhash, uripath, path, fi)
-		return err
-	})
-	return mhash, err
-}
-
-func (c *client) uploadFile(file string, fi os.FileInfo) (manifestEntry, error) {
-	hash, err := c.uploadFileContent(file, fi)
-	m := manifestEntry{
-		Hash:        hash,
-		ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())),
-	}
-	return m, err
-}
-
-func (c *client) uploadFileContent(file string, fi os.FileInfo) (string, error) {
-	fd, err := os.Open(file)
-	if err != nil {
-		return "", err
-	}
-	defer fd.Close()
-	log.Info("Uploading swarm content", "file", file, "bytes", fi.Size())
-	return c.postRaw("application/octet-stream", fi.Size(), fd)
-}
-
-func (c *client) uploadManifest(m manifest) (string, error) {
-	jsm, err := json.Marshal(m)
-	if err != nil {
-		panic(err)
-	}
-	log.Info("Uploading swarm manifest")
-	return c.postRaw("application/json", int64(len(jsm)), ioutil.NopCloser(bytes.NewReader(jsm)))
-}
-
-func (c *client) uploadToManifest(mhash string, path string, fpath string, fi os.FileInfo) (string, error) {
-	fd, err := os.Open(fpath)
-	if err != nil {
-		return "", err
-	}
-	defer fd.Close()
-	log.Info("Uploading swarm content and path", "file", fpath, "bytes", fi.Size(), "path", path)
-	req, err := http.NewRequest("PUT", c.api+"/bzz:/"+mhash+"/"+path, fd)
-	if err != nil {
-		return "", err
-	}
-	req.Header.Set("content-type", mime.TypeByExtension(filepath.Ext(fi.Name())))
-	req.ContentLength = fi.Size()
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode >= 400 {
-		return "", fmt.Errorf("bad status: %s", resp.Status)
-	}
-	content, err := ioutil.ReadAll(resp.Body)
-	return string(content), err
-}
-
-func (c *client) postRaw(mimetype string, size int64, body io.ReadCloser) (string, error) {
-	req, err := http.NewRequest("POST", c.api+"/bzzr:/", body)
-	if err != nil {
-		return "", err
-	}
-	req.Header.Set("content-type", mimetype)
-	req.ContentLength = size
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode >= 400 {
-		return "", fmt.Errorf("bad status: %s", resp.Status)
-	}
-	content, err := ioutil.ReadAll(resp.Body)
-	return string(content), err
-}
-
-func (c *client) downloadManifest(mhash string) (manifest, error) {
-
-	mroot := manifest{}
-	req, err := http.NewRequest("GET", c.api+"/bzzr:/"+mhash, nil)
-	if err != nil {
-		return mroot, err
-	}
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return mroot, err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode >= 400 {
-		return mroot, fmt.Errorf("bad status: %s", resp.Status)
-
-	}
-	content, err := ioutil.ReadAll(resp.Body)
-
-	err = json.Unmarshal(content, &mroot)
-	if err != nil {
-		return mroot, fmt.Errorf("Manifest %v is malformed: %v", mhash, err)
-	}
-	return mroot, err
-}