swarm/api: refactor and improve HTTP API (#3773)

This PR deprecates the file related RPC calls in favour of an improved HTTP API.

The main aim is to expose a simple to use API which can be consumed by thin
clients (e.g. curl and HTML forms) without the need for complex logic (e.g.
manipulating prefix trie manifests).

3 files changed, 77 insertions, 71 deletions

diff --git a/cmd/swarm/list.go b/cmd/swarm/list.go
index 3a68fef03..06d3883cf 100644
--- a/cmd/swarm/list.go
+++ b/cmd/swarm/list.go
@@ -44,7 +44,7 @@ func list(ctx *cli.Context) {
 
 	bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
 	client := swarm.NewClient(bzzapi)
-	entries, err := client.ManifestFileList(manifest, prefix)
+	list, err := client.List(manifest, prefix)
 	if err != nil {
 		utils.Fatalf("Failed to generate file and directory list: %s", err)
 	}
@@ -52,7 +52,10 @@ func list(ctx *cli.Context) {
 	w := tabwriter.NewWriter(os.Stdout, 1, 2, 2, ' ', 0)
 	defer w.Flush()
 	fmt.Fprintln(w, "HASH\tCONTENT TYPE\tPATH")
-	for _, entry := range entries {
+	for _, prefix := range list.CommonPrefixes {
+		fmt.Fprintf(w, "%s\t%s\t%s\n", "", "DIR", prefix)
+	}
+	for _, entry := range list.Entries {
 		fmt.Fprintf(w, "%s\t%s\t%s\n", entry.Hash, entry.ContentType, entry.Path)
 	}
 }
diff --git a/cmd/swarm/manifest.go b/cmd/swarm/manifest.go
index 698b8ddb8..9729022c0 100644
--- a/cmd/swarm/manifest.go
+++ b/cmd/swarm/manifest.go
@@ -25,6 +25,7 @@ import (
 	"strings"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/swarm/api"
 	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
 	"gopkg.in/urfave/cli.v1"
 )
@@ -42,7 +43,7 @@ func add(ctx *cli.Context) {
 
 		ctype        string
 		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
-		mroot        swarm.Manifest
+		mroot        api.Manifest
 	)
 
 	if len(args) > 3 {
@@ -76,7 +77,7 @@ func update(ctx *cli.Context) {
 
 		ctype        string
 		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
-		mroot        swarm.Manifest
+		mroot        api.Manifest
 	)
 	if len(args) > 3 {
 		ctype = args[3]
@@ -106,7 +107,7 @@ func remove(ctx *cli.Context) {
 		path  = args[1]
 
 		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
-		mroot        swarm.Manifest
+		mroot        api.Manifest
 	)
 
 	newManifest := removeEntryFromManifest(ctx, mhash, path)
@@ -125,11 +126,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 	var (
 		bzzapi           = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
 		client           = swarm.NewClient(bzzapi)
-		longestPathEntry = swarm.ManifestEntry{
-			Path:        "",
-			Hash:        "",
-			ContentType: "",
-		}
+		longestPathEntry = api.ManifestEntry{}
 	)
 
 	mroot, err := client.DownloadManifest(mhash)
@@ -163,7 +160,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 		newHash := addEntryToManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
 
 		// Replace the hash for parent Manifests
-		newMRoot := swarm.Manifest{}
+		newMRoot := &api.Manifest{}
 		for _, entry := range mroot.Entries {
 			if longestPathEntry.Path == entry.Path {
 				entry.Hash = newHash
@@ -173,9 +170,9 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 		mroot = newMRoot
 	} else {
 		// Add the entry in the leaf Manifest
-		newEntry := swarm.ManifestEntry{
-			Path:        path,
+		newEntry := api.ManifestEntry{
 			Hash:        hash,
+			Path:        path,
 			ContentType: ctype,
 		}
 		mroot.Entries = append(mroot.Entries, newEntry)
@@ -192,18 +189,10 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) string {
 
 	var (
-		bzzapi   = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client   = swarm.NewClient(bzzapi)
-		newEntry = swarm.ManifestEntry{
-			Path:        "",
-			Hash:        "",
-			ContentType: "",
-		}
-		longestPathEntry = swarm.ManifestEntry{
-			Path:        "",
-			Hash:        "",
-			ContentType: "",
-		}
+		bzzapi           = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		client           = swarm.NewClient(bzzapi)
+		newEntry         = api.ManifestEntry{}
+		longestPathEntry = api.ManifestEntry{}
 	)
 
 	mroot, err := client.DownloadManifest(mhash)
@@ -237,7 +226,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
 		newHash := updateEntryInManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
 
 		// Replace the hash for parent Manifests
-		newMRoot := swarm.Manifest{}
+		newMRoot := &api.Manifest{}
 		for _, entry := range mroot.Entries {
 			if longestPathEntry.Path == entry.Path {
 				entry.Hash = newHash
@@ -250,12 +239,12 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
 
 	if newEntry.Path != "" {
 		// Replace the hash for leaf Manifest
-		newMRoot := swarm.Manifest{}
+		newMRoot := &api.Manifest{}
 		for _, entry := range mroot.Entries {
 			if newEntry.Path == entry.Path {
-				myEntry := swarm.ManifestEntry{
-					Path:        entry.Path,
+				myEntry := api.ManifestEntry{
 					Hash:        hash,
+					Path:        entry.Path,
 					ContentType: ctype,
 				}
 				newMRoot.Entries = append(newMRoot.Entries, myEntry)
@@ -276,18 +265,10 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
 func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 
 	var (
-		bzzapi        = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client        = swarm.NewClient(bzzapi)
-		entryToRemove = swarm.ManifestEntry{
-			Path:        "",
-			Hash:        "",
-			ContentType: "",
-		}
-		longestPathEntry = swarm.ManifestEntry{
-			Path:        "",
-			Hash:        "",
-			ContentType: "",
-		}
+		bzzapi           = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+		client           = swarm.NewClient(bzzapi)
+		entryToRemove    = api.ManifestEntry{}
+		longestPathEntry = api.ManifestEntry{}
 	)
 
 	mroot, err := client.DownloadManifest(mhash)
@@ -319,7 +300,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 		newHash := removeEntryFromManifest(ctx, longestPathEntry.Hash, newPath)
 
 		// Replace the hash for parent Manifests
-		newMRoot := swarm.Manifest{}
+		newMRoot := &api.Manifest{}
 		for _, entry := range mroot.Entries {
 			if longestPathEntry.Path == entry.Path {
 				entry.Hash = newHash
@@ -331,7 +312,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 
 	if entryToRemove.Path != "" {
 		// remove the entry in this Manifest
-		newMRoot := swarm.Manifest{}
+		newMRoot := &api.Manifest{}
 		for _, entry := range mroot.Entries {
 			if entryToRemove.Path != entry.Path {
 				newMRoot.Entries = append(newMRoot.Entries, entry)
diff --git a/cmd/swarm/upload.go b/cmd/swarm/upload.go
index 46f10c4be..42673ae21 100644
--- a/cmd/swarm/upload.go
+++ b/cmd/swarm/upload.go
@@ -18,13 +18,15 @@
 package main
 
 import (
-	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
+	"mime"
+	"net/http"
 	"os"
 	"os/user"
 	"path"
+	"path/filepath"
 	"strings"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
@@ -42,12 +44,10 @@ func upload(ctx *cli.Context) {
 		defaultPath  = ctx.GlobalString(SwarmUploadDefaultPath.Name)
 		fromStdin    = ctx.GlobalBool(SwarmUpFromStdinFlag.Name)
 		mimeType     = ctx.GlobalString(SwarmUploadMimeType.Name)
+		client       = swarm.NewClient(bzzapi)
+		file         string
 	)
 
-	var client = swarm.NewClient(bzzapi)
-	var entry swarm.ManifestEntry
-	var file string
-
 	if len(args) != 1 {
 		if fromStdin {
 			tmp, err := ioutil.TempFile("", "swarm-stdin")
@@ -66,41 +66,47 @@ func upload(ctx *cli.Context) {
 			utils.Fatalf("Need filename as the first and only argument")
 		}
 	} else {
-		file = args[0]
+		file = expandPath(args[0])
+	}
+
+	if !wantManifest {
+		f, err := swarm.Open(file)
+		if err != nil {
+			utils.Fatalf("Error opening file: %s", err)
+		}
+		defer f.Close()
+		hash, err := client.UploadRaw(f, f.Size)
+		if err != nil {
+			utils.Fatalf("Upload failed: %s", err)
+		}
+		fmt.Println(hash)
+		return
 	}
 
-	fi, err := os.Stat(expandPath(file))
+	stat, err := os.Stat(file)
 	if err != nil {
-		utils.Fatalf("Failed to stat file: %v", err)
+		utils.Fatalf("Error opening file: %s", err)
 	}
-	if fi.IsDir() {
+	var hash string
+	if stat.IsDir() {
 		if !recursive {
 			utils.Fatalf("Argument is a directory and recursive upload is disabled")
 		}
-		if !wantManifest {
-			utils.Fatalf("Manifest is required for directory uploads")
+		hash, err = client.UploadDirectory(file, defaultPath, "")
+	} else {
+		if mimeType == "" {
+			mimeType = detectMimeType(file)
 		}
-		mhash, err := client.UploadDirectory(file, defaultPath)
+		f, err := swarm.Open(file)
 		if err != nil {
-			utils.Fatalf("Failed to upload directory: %v", err)
+			utils.Fatalf("Error opening file: %s", err)
 		}
-		fmt.Println(mhash)
-		return
+		defer f.Close()
+		f.ContentType = mimeType
+		hash, err = client.Upload(f, "")
 	}
-	entry, err = client.UploadFile(file, fi, mimeType)
 	if err != nil {
-		utils.Fatalf("Upload failed: %v", err)
-	}
-	mroot := swarm.Manifest{Entries: []swarm.ManifestEntry{entry}}
-	if !wantManifest {
-		// Print the manifest. This is the only output to stdout.
-		mrootJSON, _ := json.MarshalIndent(mroot, "", "  ")
-		fmt.Println(string(mrootJSON))
-		return
-	}
-	hash, err := client.UploadManifest(mroot)
-	if err != nil {
-		utils.Fatalf("Manifest upload failed: %v", err)
+		utils.Fatalf("Upload failed: %s", err)
 	}
 	fmt.Println(hash)
 }
@@ -128,3 +134,19 @@ func homeDir() string {
 	}
 	return ""
 }
+
+func detectMimeType(file string) string {
+	if ext := filepath.Ext(file); ext != "" {
+		return mime.TypeByExtension(ext)
+	}
+	f, err := os.Open(file)
+	if err != nil {
+		return ""
+	}
+	defer f.Close()
+	buf := make([]byte, 512)
+	if n, _ := f.Read(buf); n > 0 {
+		return http.DetectContentType(buf)
+	}
+	return ""
+}