From 6d1e292eefa70b5cb76cd03ff61fc6c4550d7c36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Fri, 10 Aug 2018 16:12:55 +0200 Subject: Manifest cli fix and upload defaultpath only once (#17375) * cmd/swarm: fix manifest subcommands and add tests * cmd/swarm: manifest update: update default entry for non-encrypted uploads * swarm/api: upload defaultpath file only once * swarm/api/client: improve UploadDirectory default path handling * cmd/swarm: support absolute and relative default path values * cmd/swarm: fix a typo in test * cmd/swarm: check encrypted uploads in manifest update tests --- cmd/swarm/main.go | 10 +- cmd/swarm/manifest.go | 234 +++++++++--------- cmd/swarm/manifest_test.go | 579 +++++++++++++++++++++++++++++++++++++++++++++ cmd/swarm/upload.go | 11 + cmd/swarm/upload_test.go | 81 +++++++ 5 files changed, 787 insertions(+), 128 deletions(-) create mode 100644 cmd/swarm/manifest_test.go (limited to 'cmd') diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 258f24d32..ac09ae998 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -322,23 +322,23 @@ Downloads a swarm bzz uri to the given dir. When no dir is provided, working dir Description: "Updates a MANIFEST by adding/removing/updating the hash of a path.\nCOMMAND could be: add, update, remove", Subcommands: []cli.Command{ { - Action: add, + Action: manifestAdd, CustomHelpTemplate: helpTemplate, Name: "add", Usage: "add a new path to the manifest", - ArgsUsage: " []", + ArgsUsage: " ", Description: "Adds a new path to the manifest", }, { - Action: update, + Action: manifestUpdate, CustomHelpTemplate: helpTemplate, Name: "update", Usage: "update the hash for an already existing path in the manifest", - ArgsUsage: " []", + ArgsUsage: " ", Description: "Update the hash for an already existing path in the manifest", }, { - Action: remove, + Action: manifestRemove, CustomHelpTemplate: helpTemplate, Name: "remove", Usage: "removes a path from the manifest", diff --git a/cmd/swarm/manifest.go b/cmd/swarm/manifest.go index 82166edf6..0216ffc1d 100644 --- a/cmd/swarm/manifest.go +++ b/cmd/swarm/manifest.go @@ -18,10 +18,8 @@ package main import ( - "encoding/json" "fmt" - "mime" - "path/filepath" + "os" "strings" "github.com/ethereum/go-ethereum/cmd/utils" @@ -30,127 +28,118 @@ import ( "gopkg.in/urfave/cli.v1" ) -const bzzManifestJSON = "application/bzz-manifest+json" - -func add(ctx *cli.Context) { +// manifestAdd adds a new entry to the manifest at the given path. +// New entry hash, the last argument, must be the hash of a manifest +// with only one entry, which meta-data will be added to the original manifest. +// On success, this function will print new (updated) manifest's hash. +func manifestAdd(ctx *cli.Context) { args := ctx.Args() - if len(args) < 3 { - utils.Fatalf("Need at least three arguments []") + if len(args) != 3 { + utils.Fatalf("Need exactly three arguments ") } var ( mhash = args[0] path = args[1] hash = args[2] - - ctype string - wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name) - mroot api.Manifest ) - if len(args) > 3 { - ctype = args[3] - } else { - ctype = mime.TypeByExtension(filepath.Ext(path)) + bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") + client := swarm.NewClient(bzzapi) + + m, _, err := client.DownloadManifest(hash) + if err != nil { + utils.Fatalf("Error downloading manifest to add: %v", err) + } + l := len(m.Entries) + if l == 0 { + utils.Fatalf("No entries in manifest %s", hash) + } else if l > 1 { + utils.Fatalf("Too many entries in manifest %s", hash) } - newManifest := addEntryToManifest(ctx, mhash, path, hash, ctype) + newManifest := addEntryToManifest(client, mhash, path, m.Entries[0]) fmt.Println(newManifest) - - if !wantManifest { - // Print the manifest. This is the only output to stdout. - mrootJSON, _ := json.MarshalIndent(mroot, "", " ") - fmt.Println(string(mrootJSON)) - return - } } -func update(ctx *cli.Context) { - +// manifestUpdate replaces an existing entry of the manifest at the given path. +// New entry hash, the last argument, must be the hash of a manifest +// with only one entry, which meta-data will be added to the original manifest. +// On success, this function will print hash of the updated manifest. +func manifestUpdate(ctx *cli.Context) { args := ctx.Args() - if len(args) < 3 { - utils.Fatalf("Need at least three arguments ") + if len(args) != 3 { + utils.Fatalf("Need exactly three arguments ") } var ( mhash = args[0] path = args[1] hash = args[2] - - ctype string - wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name) - mroot api.Manifest ) - if len(args) > 3 { - ctype = args[3] - } else { - ctype = mime.TypeByExtension(filepath.Ext(path)) - } - newManifest := updateEntryInManifest(ctx, mhash, path, hash, ctype) - fmt.Println(newManifest) + bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") + client := swarm.NewClient(bzzapi) - if !wantManifest { - // Print the manifest. This is the only output to stdout. - mrootJSON, _ := json.MarshalIndent(mroot, "", " ") - fmt.Println(string(mrootJSON)) - return + m, _, err := client.DownloadManifest(hash) + if err != nil { + utils.Fatalf("Error downloading manifest to update: %v", err) + } + l := len(m.Entries) + if l == 0 { + utils.Fatalf("No entries in manifest %s", hash) + } else if l > 1 { + utils.Fatalf("Too many entries in manifest %s", hash) } + + newManifest, _, defaultEntryUpdated := updateEntryInManifest(client, mhash, path, m.Entries[0], true) + if defaultEntryUpdated { + // Print informational message to stderr + // allowing the user to get the new manifest hash from stdout + // without the need to parse the complete output. + fmt.Fprintln(os.Stderr, "Manifest default entry is updated, too") + } + fmt.Println(newManifest) } -func remove(ctx *cli.Context) { +// manifestRemove removes an existing entry of the manifest at the given path. +// On success, this function will print hash of the manifest which does not +// contain the path. +func manifestRemove(ctx *cli.Context) { args := ctx.Args() - if len(args) < 2 { - utils.Fatalf("Need at least two arguments ") + if len(args) != 2 { + utils.Fatalf("Need exactly two arguments ") } var ( mhash = args[0] path = args[1] - - wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name) - mroot api.Manifest ) - newManifest := removeEntryFromManifest(ctx, mhash, path) - fmt.Println(newManifest) + bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") + client := swarm.NewClient(bzzapi) - if !wantManifest { - // Print the manifest. This is the only output to stdout. - mrootJSON, _ := json.MarshalIndent(mroot, "", " ") - fmt.Println(string(mrootJSON)) - return - } + newManifest := removeEntryFromManifest(client, mhash, path) + fmt.Println(newManifest) } -func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) string { - - var ( - bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") - client = swarm.NewClient(bzzapi) - longestPathEntry = api.ManifestEntry{} - ) +func addEntryToManifest(client *swarm.Client, mhash, path string, entry api.ManifestEntry) string { + var longestPathEntry = api.ManifestEntry{} mroot, isEncrypted, err := client.DownloadManifest(mhash) if err != nil { utils.Fatalf("Manifest download failed: %v", err) } - //TODO: check if the "hash" to add is valid and present in swarm - _, _, err = client.DownloadManifest(hash) - if err != nil { - utils.Fatalf("Hash to add is not present: %v", err) - } - // See if we path is in this Manifest or do we have to dig deeper - for _, entry := range mroot.Entries { - if path == entry.Path { + for _, e := range mroot.Entries { + if path == e.Path { utils.Fatalf("Path %s already present, not adding anything", path) } else { - if entry.ContentType == bzzManifestJSON { - prfxlen := strings.HasPrefix(path, entry.Path) + if e.ContentType == api.ManifestType { + prfxlen := strings.HasPrefix(path, e.Path) if prfxlen && len(path) > len(longestPathEntry.Path) { - longestPathEntry = entry + longestPathEntry = e } } } @@ -159,25 +148,21 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin if longestPathEntry.Path != "" { // Load the child Manifest add the entry there newPath := path[len(longestPathEntry.Path):] - newHash := addEntryToManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype) + newHash := addEntryToManifest(client, longestPathEntry.Hash, newPath, entry) // Replace the hash for parent Manifests newMRoot := &api.Manifest{} - for _, entry := range mroot.Entries { - if longestPathEntry.Path == entry.Path { - entry.Hash = newHash + for _, e := range mroot.Entries { + if longestPathEntry.Path == e.Path { + e.Hash = newHash } - newMRoot.Entries = append(newMRoot.Entries, entry) + newMRoot.Entries = append(newMRoot.Entries, e) } mroot = newMRoot } else { // Add the entry in the leaf Manifest - newEntry := api.ManifestEntry{ - Hash: hash, - Path: path, - ContentType: ctype, - } - mroot.Entries = append(mroot.Entries, newEntry) + entry.Path = path + mroot.Entries = append(mroot.Entries, entry) } newManifestHash, err := client.UploadManifest(mroot, isEncrypted) @@ -185,14 +170,16 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin utils.Fatalf("Manifest upload failed: %v", err) } return newManifestHash - } -func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) string { - +// updateEntryInManifest updates an existing entry o path with a new one in the manifest with provided mhash +// finding the path recursively through all nested manifests. Argument isRoot is used for default +// entry update detection. If the updated entry has the same hash as the default entry, then the +// default entry in root manifest will be updated too. +// Returned values are the new manifest hash, hash of the entry that was replaced by the new entry and +// a a bool that is true if default entry is updated. +func updateEntryInManifest(client *swarm.Client, mhash, path string, entry api.ManifestEntry, isRoot bool) (newManifestHash, oldHash string, defaultEntryUpdated bool) { var ( - bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") - client = swarm.NewClient(bzzapi) newEntry = api.ManifestEntry{} longestPathEntry = api.ManifestEntry{} ) @@ -202,17 +189,18 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st utils.Fatalf("Manifest download failed: %v", err) } - //TODO: check if the "hash" with which to update is valid and present in swarm - // See if we path is in this Manifest or do we have to dig deeper - for _, entry := range mroot.Entries { - if path == entry.Path { - newEntry = entry + for _, e := range mroot.Entries { + if path == e.Path { + newEntry = e + // keep the reference of the hash of the entry that should be replaced + // for default entry detection + oldHash = e.Hash } else { - if entry.ContentType == bzzManifestJSON { - prfxlen := strings.HasPrefix(path, entry.Path) + if e.ContentType == api.ManifestType { + prfxlen := strings.HasPrefix(path, e.Path) if prfxlen && len(path) > len(longestPathEntry.Path) { - longestPathEntry = entry + longestPathEntry = e } } } @@ -225,50 +213,50 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st if longestPathEntry.Path != "" { // Load the child Manifest add the entry there newPath := path[len(longestPathEntry.Path):] - newHash := updateEntryInManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype) + var newHash string + newHash, oldHash, _ = updateEntryInManifest(client, longestPathEntry.Hash, newPath, entry, false) // Replace the hash for parent Manifests newMRoot := &api.Manifest{} - for _, entry := range mroot.Entries { - if longestPathEntry.Path == entry.Path { - entry.Hash = newHash + for _, e := range mroot.Entries { + if longestPathEntry.Path == e.Path { + e.Hash = newHash } - newMRoot.Entries = append(newMRoot.Entries, entry) + newMRoot.Entries = append(newMRoot.Entries, e) } mroot = newMRoot } - if newEntry.Path != "" { + // update the manifest if the new entry is found and + // check if default entry should be updated + if newEntry.Path != "" || isRoot { // Replace the hash for leaf Manifest newMRoot := &api.Manifest{} - for _, entry := range mroot.Entries { - if newEntry.Path == entry.Path { - myEntry := api.ManifestEntry{ - Hash: hash, - Path: entry.Path, - ContentType: ctype, - } - newMRoot.Entries = append(newMRoot.Entries, myEntry) - } else { + for _, e := range mroot.Entries { + if newEntry.Path == e.Path { + entry.Path = e.Path newMRoot.Entries = append(newMRoot.Entries, entry) + } else if isRoot && e.Path == "" && e.Hash == oldHash { + entry.Path = e.Path + newMRoot.Entries = append(newMRoot.Entries, entry) + defaultEntryUpdated = true + } else { + newMRoot.Entries = append(newMRoot.Entries, e) } } mroot = newMRoot } - newManifestHash, err := client.UploadManifest(mroot, isEncrypted) + newManifestHash, err = client.UploadManifest(mroot, isEncrypted) if err != nil { utils.Fatalf("Manifest upload failed: %v", err) } - return newManifestHash + return newManifestHash, oldHash, defaultEntryUpdated } -func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string { - +func removeEntryFromManifest(client *swarm.Client, mhash, path string) string { var ( - bzzapi = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/") - client = swarm.NewClient(bzzapi) entryToRemove = api.ManifestEntry{} longestPathEntry = api.ManifestEntry{} ) @@ -283,7 +271,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string { if path == entry.Path { entryToRemove = entry } else { - if entry.ContentType == bzzManifestJSON { + if entry.ContentType == api.ManifestType { prfxlen := strings.HasPrefix(path, entry.Path) if prfxlen && len(path) > len(longestPathEntry.Path) { longestPathEntry = entry @@ -299,7 +287,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string { if longestPathEntry.Path != "" { // Load the child Manifest remove the entry there newPath := path[len(longestPathEntry.Path):] - newHash := removeEntryFromManifest(ctx, longestPathEntry.Hash, newPath) + newHash := removeEntryFromManifest(client, longestPathEntry.Hash, newPath) // Replace the hash for parent Manifests newMRoot := &api.Manifest{} diff --git a/cmd/swarm/manifest_test.go b/cmd/swarm/manifest_test.go new file mode 100644 index 000000000..08fe0b2eb --- /dev/null +++ b/cmd/swarm/manifest_test.go @@ -0,0 +1,579 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "bytes" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/ethereum/go-ethereum/swarm/api" + swarm "github.com/ethereum/go-ethereum/swarm/api/client" +) + +// TestManifestChange tests manifest add, update and remove +// cli commands without encryption. +func TestManifestChange(t *testing.T) { + testManifestChange(t, false) +} + +// TestManifestChange tests manifest add, update and remove +// cli commands with encryption enabled. +func TestManifestChangeEncrypted(t *testing.T) { + testManifestChange(t, true) +} + +// testManifestChange performs cli commands: +// - manifest add +// - manifest update +// - manifest remove +// on a manifest, testing the functionality of this +// comands on paths that are in root manifest or a nested one. +// Argument encrypt controls whether to use encryption or not. +func testManifestChange(t *testing.T, encrypt bool) { + t.Parallel() + cluster := newTestCluster(t, 1) + defer cluster.Shutdown() + + tmp, err := ioutil.TempDir("", "swarm-manifest-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmp) + + origDir := filepath.Join(tmp, "orig") + if err := os.Mkdir(origDir, 0777); err != nil { + t.Fatal(err) + } + + indexDataFilename := filepath.Join(origDir, "index.html") + err = ioutil.WriteFile(indexDataFilename, []byte("

Test

"), 0666) + if err != nil { + t.Fatal(err) + } + // Files paths robots.txt and robots.html share the same prefix "robots." + // which will result a manifest with a nested manifest under path "robots.". + // This will allow testing manifest changes on both root and nested manifest. + err = ioutil.WriteFile(filepath.Join(origDir, "robots.txt"), []byte("Disallow: /"), 0666) + if err != nil { + t.Fatal(err) + } + err = ioutil.WriteFile(filepath.Join(origDir, "robots.html"), []byte("No Robots Allowed"), 0666) + if err != nil { + t.Fatal(err) + } + err = ioutil.WriteFile(filepath.Join(origDir, "mutants.txt"), []byte("Frank\nMarcus"), 0666) + if err != nil { + t.Fatal(err) + } + + args := []string{ + "--bzzapi", + cluster.Nodes[0].URL, + "--recursive", + "--defaultpath", + indexDataFilename, + "up", + origDir, + } + if encrypt { + args = append(args, "--encrypt") + } + + origManifestHash := runSwarmExpectHash(t, args...) + + checkHashLength(t, origManifestHash, encrypt) + + client := swarm.NewClient(cluster.Nodes[0].URL) + + // upload a new file and use its manifest to add it the original manifest. + t.Run("add", func(t *testing.T) { + humansData := []byte("Ann\nBob") + humansDataFilename := filepath.Join(tmp, "humans.txt") + err = ioutil.WriteFile(humansDataFilename, humansData, 0666) + if err != nil { + t.Fatal(err) + } + + humansManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "up", + humansDataFilename, + ) + + newManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "manifest", + "add", + origManifestHash, + "humans.txt", + humansManifestHash, + ) + + checkHashLength(t, newManifestHash, encrypt) + + newManifest := downloadManifest(t, client, newManifestHash, encrypt) + + var found bool + for _, e := range newManifest.Entries { + if e.Path == "humans.txt" { + found = true + if e.Size != int64(len(humansData)) { + t.Errorf("expected humans.txt size %v, got %v", len(humansData), e.Size) + } + if e.ModTime.IsZero() { + t.Errorf("got zero mod time for humans.txt") + } + ct := "text/plain; charset=utf-8" + if e.ContentType != ct { + t.Errorf("expected content type %q, got %q", ct, e.ContentType) + } + break + } + } + if !found { + t.Fatal("no humans.txt in new manifest") + } + + checkFile(t, client, newManifestHash, "humans.txt", humansData) + }) + + // upload a new file and use its manifest to add it the original manifest, + // but ensure that the file will be in the nested manifest of the original one. + t.Run("add nested", func(t *testing.T) { + robotsData := []byte(`{"disallow": "/"}`) + robotsDataFilename := filepath.Join(tmp, "robots.json") + err = ioutil.WriteFile(robotsDataFilename, robotsData, 0666) + if err != nil { + t.Fatal(err) + } + + robotsManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "up", + robotsDataFilename, + ) + + newManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "manifest", + "add", + origManifestHash, + "robots.json", + robotsManifestHash, + ) + + checkHashLength(t, newManifestHash, encrypt) + + newManifest := downloadManifest(t, client, newManifestHash, encrypt) + + var found bool + loop: + for _, e := range newManifest.Entries { + if e.Path == "robots." { + nestedManifest := downloadManifest(t, client, e.Hash, encrypt) + for _, e := range nestedManifest.Entries { + if e.Path == "json" { + found = true + if e.Size != int64(len(robotsData)) { + t.Errorf("expected robots.json size %v, got %v", len(robotsData), e.Size) + } + if e.ModTime.IsZero() { + t.Errorf("got zero mod time for robots.json") + } + ct := "application/json" + if e.ContentType != ct { + t.Errorf("expected content type %q, got %q", ct, e.ContentType) + } + break loop + } + } + } + } + if !found { + t.Fatal("no robots.json in new manifest") + } + + checkFile(t, client, newManifestHash, "robots.json", robotsData) + }) + + // upload a new file and use its manifest to change the file it the original manifest. + t.Run("update", func(t *testing.T) { + indexData := []byte("

Ethereum Swarm

") + indexDataFilename := filepath.Join(tmp, "index.html") + err = ioutil.WriteFile(indexDataFilename, indexData, 0666) + if err != nil { + t.Fatal(err) + } + + indexManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "up", + indexDataFilename, + ) + + newManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "manifest", + "update", + origManifestHash, + "index.html", + indexManifestHash, + ) + + checkHashLength(t, newManifestHash, encrypt) + + newManifest := downloadManifest(t, client, newManifestHash, encrypt) + + var found bool + for _, e := range newManifest.Entries { + if e.Path == "index.html" { + found = true + if e.Size != int64(len(indexData)) { + t.Errorf("expected index.html size %v, got %v", len(indexData), e.Size) + } + if e.ModTime.IsZero() { + t.Errorf("got zero mod time for index.html") + } + ct := "text/html; charset=utf-8" + if e.ContentType != ct { + t.Errorf("expected content type %q, got %q", ct, e.ContentType) + } + break + } + } + if !found { + t.Fatal("no index.html in new manifest") + } + + checkFile(t, client, newManifestHash, "index.html", indexData) + + // check default entry change + checkFile(t, client, newManifestHash, "", indexData) + }) + + // upload a new file and use its manifest to change the file it the original manifest, + // but ensure that the file is in the nested manifest of the original one. + t.Run("update nested", func(t *testing.T) { + robotsData := []byte(`Only humans allowed!!!`) + robotsDataFilename := filepath.Join(tmp, "robots.html") + err = ioutil.WriteFile(robotsDataFilename, robotsData, 0666) + if err != nil { + t.Fatal(err) + } + + humansManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "up", + robotsDataFilename, + ) + + newManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "manifest", + "update", + origManifestHash, + "robots.html", + humansManifestHash, + ) + + checkHashLength(t, newManifestHash, encrypt) + + newManifest := downloadManifest(t, client, newManifestHash, encrypt) + + var found bool + loop: + for _, e := range newManifest.Entries { + if e.Path == "robots." { + nestedManifest := downloadManifest(t, client, e.Hash, encrypt) + for _, e := range nestedManifest.Entries { + if e.Path == "html" { + found = true + if e.Size != int64(len(robotsData)) { + t.Errorf("expected robots.html size %v, got %v", len(robotsData), e.Size) + } + if e.ModTime.IsZero() { + t.Errorf("got zero mod time for robots.html") + } + ct := "text/html; charset=utf-8" + if e.ContentType != ct { + t.Errorf("expected content type %q, got %q", ct, e.ContentType) + } + break loop + } + } + } + } + if !found { + t.Fatal("no robots.html in new manifest") + } + + checkFile(t, client, newManifestHash, "robots.html", robotsData) + }) + + // remove a file from the manifest. + t.Run("remove", func(t *testing.T) { + newManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "manifest", + "remove", + origManifestHash, + "mutants.txt", + ) + + checkHashLength(t, newManifestHash, encrypt) + + newManifest := downloadManifest(t, client, newManifestHash, encrypt) + + var found bool + for _, e := range newManifest.Entries { + if e.Path == "mutants.txt" { + found = true + break + } + } + if found { + t.Fatal("mutants.txt is not removed") + } + }) + + // remove a file from the manifest, but ensure that the file is in + // the nested manifest of the original one. + t.Run("remove nested", func(t *testing.T) { + newManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "manifest", + "remove", + origManifestHash, + "robots.html", + ) + + checkHashLength(t, newManifestHash, encrypt) + + newManifest := downloadManifest(t, client, newManifestHash, encrypt) + + var found bool + loop: + for _, e := range newManifest.Entries { + if e.Path == "robots." { + nestedManifest := downloadManifest(t, client, e.Hash, encrypt) + for _, e := range nestedManifest.Entries { + if e.Path == "html" { + found = true + break loop + } + } + } + } + if found { + t.Fatal("robots.html in not removed") + } + }) +} + +// TestNestedDefaultEntryUpdate tests if the default entry is updated +// if the file in nested manifest used for it is also updated. +func TestNestedDefaultEntryUpdate(t *testing.T) { + testNestedDefaultEntryUpdate(t, false) +} + +// TestNestedDefaultEntryUpdateEncrypted tests if the default entry +// of encrypted upload is updated if the file in nested manifest +// used for it is also updated. +func TestNestedDefaultEntryUpdateEncrypted(t *testing.T) { + testNestedDefaultEntryUpdate(t, true) +} + +func testNestedDefaultEntryUpdate(t *testing.T, encrypt bool) { + t.Parallel() + cluster := newTestCluster(t, 1) + defer cluster.Shutdown() + + tmp, err := ioutil.TempDir("", "swarm-manifest-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmp) + + origDir := filepath.Join(tmp, "orig") + if err := os.Mkdir(origDir, 0777); err != nil { + t.Fatal(err) + } + + indexData := []byte("

Test

") + indexDataFilename := filepath.Join(origDir, "index.html") + err = ioutil.WriteFile(indexDataFilename, indexData, 0666) + if err != nil { + t.Fatal(err) + } + // Add another file with common prefix as the default entry to test updates of + // default entry with nested manifests. + err = ioutil.WriteFile(filepath.Join(origDir, "index.txt"), []byte("Test"), 0666) + if err != nil { + t.Fatal(err) + } + + args := []string{ + "--bzzapi", + cluster.Nodes[0].URL, + "--recursive", + "--defaultpath", + indexDataFilename, + "up", + origDir, + } + if encrypt { + args = append(args, "--encrypt") + } + + origManifestHash := runSwarmExpectHash(t, args...) + + checkHashLength(t, origManifestHash, encrypt) + + client := swarm.NewClient(cluster.Nodes[0].URL) + + newIndexData := []byte("

Ethereum Swarm

") + newIndexDataFilename := filepath.Join(tmp, "index.html") + err = ioutil.WriteFile(newIndexDataFilename, newIndexData, 0666) + if err != nil { + t.Fatal(err) + } + + newIndexManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "up", + newIndexDataFilename, + ) + + newManifestHash := runSwarmExpectHash(t, + "--bzzapi", + cluster.Nodes[0].URL, + "manifest", + "update", + origManifestHash, + "index.html", + newIndexManifestHash, + ) + + checkHashLength(t, newManifestHash, encrypt) + + newManifest := downloadManifest(t, client, newManifestHash, encrypt) + + var found bool + for _, e := range newManifest.Entries { + if e.Path == "index." { + found = true + newManifest = downloadManifest(t, client, e.Hash, encrypt) + break + } + } + if !found { + t.Fatal("no index. path in new manifest") + } + + found = false + for _, e := range newManifest.Entries { + if e.Path == "html" { + found = true + if e.Size != int64(len(newIndexData)) { + t.Errorf("expected index.html size %v, got %v", len(newIndexData), e.Size) + } + if e.ModTime.IsZero() { + t.Errorf("got zero mod time for index.html") + } + ct := "text/html; charset=utf-8" + if e.ContentType != ct { + t.Errorf("expected content type %q, got %q", ct, e.ContentType) + } + break + } + } + if !found { + t.Fatal("no html in new manifest") + } + + checkFile(t, client, newManifestHash, "index.html", newIndexData) + + // check default entry change + checkFile(t, client, newManifestHash, "", newIndexData) +} + +func runSwarmExpectHash(t *testing.T, args ...string) (hash string) { + t.Helper() + hashRegexp := `[a-f\d]{64,128}` + up := runSwarm(t, args...) + _, matches := up.ExpectRegexp(hashRegexp) + up.ExpectExit() + + if len(matches) < 1 { + t.Fatal("no matches found") + } + return matches[0] +} + +func checkHashLength(t *testing.T, hash string, encrypted bool) { + t.Helper() + l := len(hash) + if encrypted && l != 128 { + t.Errorf("expected hash length 128, got %v", l) + } + if !encrypted && l != 64 { + t.Errorf("expected hash length 64, got %v", l) + } +} + +func downloadManifest(t *testing.T, client *swarm.Client, hash string, encrypted bool) (manifest *api.Manifest) { + t.Helper() + m, isEncrypted, err := client.DownloadManifest(hash) + if err != nil { + t.Fatal(err) + } + + if encrypted != isEncrypted { + t.Error("new manifest encryption flag is not correct") + } + return m +} + +func checkFile(t *testing.T, client *swarm.Client, hash, path string, expected []byte) { + t.Helper() + f, err := client.Download(hash, path) + if err != nil { + t.Fatal(err) + } + + got, err := ioutil.ReadAll(f) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, expected) { + t.Errorf("expected file content %q, got %q", expected, got) + } +} diff --git a/cmd/swarm/upload.go b/cmd/swarm/upload.go index 8ba0e7c5f..9eae2a3f8 100644 --- a/cmd/swarm/upload.go +++ b/cmd/swarm/upload.go @@ -98,6 +98,17 @@ func upload(ctx *cli.Context) { if !recursive { return "", errors.New("Argument is a directory and recursive upload is disabled") } + if defaultPath != "" { + // construct absolute default path + absDefaultPath, _ := filepath.Abs(defaultPath) + absFile, _ := filepath.Abs(file) + // make sure absolute directory ends with only one "/" + // to trim it from absolute default path and get relative default path + absFile = strings.TrimRight(absFile, "/") + "/" + if absDefaultPath != "" && absFile != "" && strings.HasPrefix(absDefaultPath, absFile) { + defaultPath = strings.TrimPrefix(absDefaultPath, absFile) + } + } return client.UploadDirectory(file, defaultPath, "", toEncrypt) } } else { diff --git a/cmd/swarm/upload_test.go b/cmd/swarm/upload_test.go index 2afc9b3a1..c3199dadc 100644 --- a/cmd/swarm/upload_test.go +++ b/cmd/swarm/upload_test.go @@ -273,3 +273,84 @@ func testCLISwarmUpRecursive(toEncrypt bool, t *testing.T) { } } } + +// TestCLISwarmUpDefaultPath tests swarm recursive upload with relative and absolute +// default paths and with encryption. +func TestCLISwarmUpDefaultPath(t *testing.T) { + testCLISwarmUpDefaultPath(false, false, t) + testCLISwarmUpDefaultPath(false, true, t) + testCLISwarmUpDefaultPath(true, false, t) + testCLISwarmUpDefaultPath(true, true, t) +} + +func testCLISwarmUpDefaultPath(toEncrypt bool, absDefaultPath bool, t *testing.T) { + cluster := newTestCluster(t, 1) + defer cluster.Shutdown() + + tmp, err := ioutil.TempDir("", "swarm-defaultpath-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmp) + + err = ioutil.WriteFile(filepath.Join(tmp, "index.html"), []byte("

Test

"), 0666) + if err != nil { + t.Fatal(err) + } + err = ioutil.WriteFile(filepath.Join(tmp, "robots.txt"), []byte("Disallow: /"), 0666) + if err != nil { + t.Fatal(err) + } + + defaultPath := "index.html" + if absDefaultPath { + defaultPath = filepath.Join(tmp, defaultPath) + } + + args := []string{ + "--bzzapi", + cluster.Nodes[0].URL, + "--recursive", + "--defaultpath", + defaultPath, + "up", + tmp, + } + if toEncrypt { + args = append(args, "--encrypt") + } + + up := runSwarm(t, args...) + hashRegexp := `[a-f\d]{64,128}` + _, matches := up.ExpectRegexp(hashRegexp) + up.ExpectExit() + hash := matches[0] + + client := swarm.NewClient(cluster.Nodes[0].URL) + + m, isEncrypted, err := client.DownloadManifest(hash) + if err != nil { + t.Fatal(err) + } + + if toEncrypt != isEncrypted { + t.Error("downloaded manifest is not encrypted") + } + + var found bool + var entriesCount int + for _, e := range m.Entries { + entriesCount++ + if e.Path == "" { + found = true + } + } + + if !found { + t.Error("manifest default entry was not found") + } + + if entriesCount != 3 { + t.Errorf("manifest contains %v entries, expected %v", entriesCount, 3) + } +} -- cgit v1.2.3