From 3d46f69a10a60324ee6af0c96d064cc73ed5c37d Mon Sep 17 00:00:00 2001 From: bojie Date: Fri, 28 Dec 2018 13:15:06 +0800 Subject: app: use gcp storage instead of azure for builds (#81) * vendor: add dependencies for using GCP storage * app: use gcp storage instead of azure for builds --- build/ci.go | 154 +++++++++++++++++++++++++++++------------------------------- 1 file changed, 75 insertions(+), 79 deletions(-) (limited to 'build/ci.go') diff --git a/build/ci.go b/build/ci.go index c29992f50..f49071295 100644 --- a/build/ci.go +++ b/build/ci.go @@ -1,18 +1,18 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. +// Copyright 2016 The go-dexon Authors +// This file is part of the go-dexon library. // -// The go-ethereum library is free software: you can redistribute it and/or modify +// The go-dexon library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // -// The go-ethereum library is distributed in the hope that it will be useful, +// The go-dexon library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . +// along with the go-dexon library. If not, see . // +build none @@ -64,19 +64,19 @@ import ( ) var ( - // Files that end up in the geth*.zip archive. - gethArchiveFiles = []string{ + // Files that end up in the gdex*.zip archive. + gdexArchiveFiles = []string{ "COPYING", - executablePath("geth"), + executablePath("gdex"), } - // Files that end up in the geth-alltools*.zip archive. + // Files that end up in the gdex-alltools*.zip archive. allToolsArchiveFiles = []string{ "COPYING", executablePath("abigen"), executablePath("bootnode"), executablePath("evm"), - executablePath("geth"), + executablePath("gdex"), executablePath("puppeth"), executablePath("rlpdump"), executablePath("wnode"), @@ -92,23 +92,23 @@ var ( debExecutables = []debExecutable{ { BinaryName: "abigen", - Description: "Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages.", + Description: "Source code generator to convert Dexon contract definitions into easy to use, compile-time type-safe Go packages.", }, { BinaryName: "bootnode", - Description: "Ethereum bootnode.", + Description: "Dexon bootnode.", }, { BinaryName: "evm", - Description: "Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode.", + Description: "Developer utility version of the EVM (Dexon Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode.", }, { - BinaryName: "geth", - Description: "Ethereum CLI client.", + BinaryName: "gdex", + Description: "Dexon CLI client.", }, { BinaryName: "puppeth", - Description: "Ethereum private network manager.", + Description: "Dexon private network manager.", }, { BinaryName: "rlpdump", @@ -116,7 +116,7 @@ var ( }, { BinaryName: "wnode", - Description: "Ethereum Whisper diagnostic tool", + Description: "Dexon Whisper diagnostic tool", }, } @@ -124,19 +124,19 @@ var ( debSwarmExecutables = []debExecutable{ { BinaryName: "swarm", - PackageName: "ethereum-swarm", - Description: "Ethereum Swarm daemon and tools", + PackageName: "dexon-swarm", + Description: "Dexon Swarm daemon and tools", }, } - debEthereum = debPackage{ - Name: "ethereum", + debDexon = debPackage{ + Name: "dexon", Version: params.Version, Executables: debExecutables, } debSwarm = debPackage{ - Name: "ethereum-swarm", + Name: "dexon-swarm", Version: sv.Version, Executables: debSwarmExecutables, } @@ -144,7 +144,7 @@ var ( // Debian meta packages to build and push to Ubuntu PPA debPackages = []debPackage{ debSwarm, - debEthereum, + debDexon, } // Packages to be cross-compiled by the xgo command @@ -222,7 +222,7 @@ func doInstall(cmdline []string) { if minor < 9 { log.Println("You have Go version", runtime.Version()) - log.Println("go-ethereum requires at least Go version 1.9 and cannot") + log.Println("go-dexon requires at least Go version 1.9 and cannot") log.Println("be compiled with an earlier version. Please upgrade your Go installation.") os.Exit(1) } @@ -384,7 +384,7 @@ func doArchive(cmdline []string) { arch = flag.String("arch", runtime.GOARCH, "Architecture cross packaging") atype = flag.String("type", "zip", "Type of archive to write (zip|tar)") signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. LINUX_SIGNING_KEY)`) - upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) + upload = flag.String("upload", "", `Destination to upload the archives (usually "dexon-builds")`) ext string ) flag.CommandLine.Parse(cmdline) @@ -400,15 +400,15 @@ func doArchive(cmdline []string) { var ( env = build.Env() - basegeth = archiveBasename(*arch, params.ArchiveVersion(env.Commit)) - geth = "geth-" + basegeth + ext - alltools = "geth-alltools-" + basegeth + ext + basegdex = archiveBasename(*arch, params.ArchiveVersion(env.Commit)) + gdex = "gdex-" + basegdex + ext + alltools = "gdex-alltools-" + basegdex + ext baseswarm = archiveBasename(*arch, sv.ArchiveVersion(env.Commit)) swarm = "swarm-" + baseswarm + ext ) maybeSkipArchive(env) - if err := build.WriteArchive(geth, gethArchiveFiles); err != nil { + if err := build.WriteArchive(gdex, gdexArchiveFiles); err != nil { log.Fatal(err) } if err := build.WriteArchive(alltools, allToolsArchiveFiles); err != nil { @@ -417,7 +417,7 @@ func doArchive(cmdline []string) { if err := build.WriteArchive(swarm, swarmArchiveFiles); err != nil { log.Fatal(err) } - for _, archive := range []string{geth, alltools, swarm} { + for _, archive := range []string{gdex, alltools, swarm} { if err := archiveUpload(archive, *upload, *signer); err != nil { log.Fatal(err) } @@ -448,16 +448,14 @@ func archiveUpload(archive string, blobstore string, signer string) error { } // If uploading to Azure was requested, push the archive possibly with its signature if blobstore != "" { - auth := build.AzureBlobstoreConfig{ - Account: strings.Split(blobstore, "/")[0], - Token: os.Getenv("AZURE_BLOBSTORE_TOKEN"), - Container: strings.SplitN(blobstore, "/", 2)[1], + auth := build.GCPOption{ + CredentialPath: os.Getenv("GCP_CREDENTIAL_PATH"), } - if err := build.AzureBlobstoreUpload(archive, filepath.Base(archive), auth); err != nil { + if err := build.GCPFileUpload(archive, blobstore, filepath.Base(archive), auth); err != nil { return err } if signer != "" { - if err := build.AzureBlobstoreUpload(archive+".asc", filepath.Base(archive+".asc"), auth); err != nil { + if err := build.GCPFileUpload(archive+".asc", blobstore, filepath.Base(archive+".asc"), auth); err != nil { return err } } @@ -475,7 +473,7 @@ func maybeSkipArchive(env build.Environment) { log.Printf("skipping because this is a cron job") os.Exit(0) } - if env.Branch != "master" && !strings.HasPrefix(env.Tag, "v1.") { + if env.Branch != "dev" && !strings.HasPrefix(env.Tag, "v1.") { log.Printf("skipping because branch %q, tag %q is not on the whitelist", env.Branch, env.Tag) os.Exit(0) } @@ -485,7 +483,7 @@ func maybeSkipArchive(env build.Environment) { func doDebianSource(cmdline []string) { var ( signer = flag.String("signer", "", `Signing key name, also used as package author`) - upload = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`) + upload = flag.String("upload", "", `Where to upload the source package (usually "ppa:dexon/dexon")`) sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`) workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) now = time.Now() @@ -564,7 +562,7 @@ func makeWorkdir(wdflag string) string { if wdflag != "" { err = os.MkdirAll(wdflag, 0744) } else { - wdflag, err = ioutil.TempDir("", "geth-build-") + wdflag, err = ioutil.TempDir("", "gdex-build-") } if err != nil { log.Fatal(err) @@ -580,7 +578,7 @@ func isUnstableBuild(env build.Environment) bool { } type debPackage struct { - Name string // the name of the Debian package to produce, e.g. "ethereum", or "ethereum-swarm" + Name string // the name of the Debian package to produce, e.g. "dexon", or "dexon-swarm" Version string // the clean version of the debPackage, e.g. 1.8.12 or 0.3.0, without any metadata Executables []debExecutable // executables to be included in the package } @@ -590,7 +588,7 @@ type debMetadata struct { PackageName string - // go-ethereum version being built. Note that this + // go-dexon version being built. Note that this // is not the debian package version. The package version // is constructed by VersionString. Version string @@ -618,7 +616,7 @@ func (d debExecutable) Package() string { func newDebMetadata(distro, author string, env build.Environment, t time.Time, name string, version string, exes []debExecutable) debMetadata { if author == "" { // No signing key, use default author. - author = "Ethereum Builds " + author = "Dexon Builds " } return debMetadata{ PackageName: name, @@ -681,7 +679,7 @@ func (meta debMetadata) ExeConflicts(exe debExecutable) string { // be preferred and the conflicting files should be handled via // alternates. We might do this eventually but using a conflict is // easier now. - return "ethereum, " + exe.Package() + return "dexon, " + exe.Package() } return "" } @@ -720,7 +718,7 @@ func doWindowsInstaller(cmdline []string) { var ( arch = flag.String("arch", runtime.GOARCH, "Architecture for cross build packaging") signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. WINDOWS_SIGNING_KEY)`) - upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) + upload = flag.String("upload", "", `Destination to upload the archives (usually "dexon.build")`) workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) ) flag.CommandLine.Parse(cmdline) @@ -732,28 +730,28 @@ func doWindowsInstaller(cmdline []string) { var ( devTools []string allTools []string - gethTool string + gdexTool string ) for _, file := range allToolsArchiveFiles { if file == "COPYING" { // license, copied later continue } allTools = append(allTools, filepath.Base(file)) - if filepath.Base(file) == "geth.exe" { - gethTool = file + if filepath.Base(file) == "gdex.exe" { + gdexTool = file } else { devTools = append(devTools, file) } } // Render NSIS scripts: Installer NSIS contains two installer sections, - // first section contains the geth binary, second section holds the dev tools. + // first section contains the gdex binary, second section holds the dev tools. templateData := map[string]interface{}{ "License": "COPYING", - "Geth": gethTool, + "Gdex": gdexTool, "DevTools": devTools, } - build.Render("build/nsis.geth.nsi", filepath.Join(*workdir, "geth.nsi"), 0644, nil) + build.Render("build/nsis.gdex.nsi", filepath.Join(*workdir, "gdex.nsi"), 0644, nil) build.Render("build/nsis.install.nsh", filepath.Join(*workdir, "install.nsh"), 0644, templateData) build.Render("build/nsis.uninstall.nsh", filepath.Join(*workdir, "uninstall.nsh"), 0644, allTools) build.Render("build/nsis.pathupdate.nsh", filepath.Join(*workdir, "PathUpdate.nsh"), 0644, nil) @@ -768,14 +766,14 @@ func doWindowsInstaller(cmdline []string) { if env.Commit != "" { version[2] += "-" + env.Commit[:8] } - installer, _ := filepath.Abs("geth-" + archiveBasename(*arch, params.ArchiveVersion(env.Commit)) + ".exe") + installer, _ := filepath.Abs("gdex-" + archiveBasename(*arch, params.ArchiveVersion(env.Commit)) + ".exe") build.MustRunCommand("makensis.exe", "/DOUTPUTFILE="+installer, "/DMAJORVERSION="+version[0], "/DMINORVERSION="+version[1], "/DBUILDVERSION="+version[2], "/DARCH="+*arch, - filepath.Join(*workdir, "geth.nsi"), + filepath.Join(*workdir, "gdex.nsi"), ) // Sign and publish installer. @@ -791,7 +789,7 @@ func doAndroidArchive(cmdline []string) { local = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`) signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. ANDROID_SIGNING_KEY)`) deploy = flag.String("deploy", "", `Destination to deploy the archive (usually "https://oss.sonatype.org")`) - upload = flag.String("upload", "", `Destination to upload the archive (usually "gethstore/builds")`) + upload = flag.String("upload", "", `Destination to upload the archive (usually "dexon-builds")`) ) flag.CommandLine.Parse(cmdline) env := build.Env() @@ -806,11 +804,11 @@ func doAndroidArchive(cmdline []string) { // Build the Android archive and Maven resources build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind")) build.MustRun(gomobileTool("init", "--ndk", os.Getenv("ANDROID_NDK"))) - build.MustRun(gomobileTool("bind", "-ldflags", "-s -w", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/dexon-foundation/dexon/mobile")) + build.MustRun(gomobileTool("bind", "-ldflags", "-s -w", "--target", "android", "--javapkg", "org.dexon", "-v", "github.com/dexon-foundation/dexon/mobile")) if *local { // If we're building locally, copy bundle to build dir and skip Maven - os.Rename("geth.aar", filepath.Join(GOBIN, "geth.aar")) + os.Rename("gdex.aar", filepath.Join(GOBIN, "gdex.aar")) return } meta := newMavenMetadata(env) @@ -820,8 +818,8 @@ func doAndroidArchive(cmdline []string) { maybeSkipArchive(env) // Sign and upload the archive to Azure - archive := "geth-" + archiveBasename("android", params.ArchiveVersion(env.Commit)) + ".aar" - os.Rename("geth.aar", archive) + archive := "gdex-" + archiveBasename("android", params.ArchiveVersion(env.Commit)) + ".aar" + os.Rename("gdex.aar", archive) if err := archiveUpload(archive, *upload, *signer); err != nil { log.Fatal(err) @@ -906,7 +904,7 @@ func newMavenMetadata(env build.Environment) mavenMetadata { } return mavenMetadata{ Version: version, - Package: "geth-" + version, + Package: "gdex-" + version, Develop: isUnstableBuild(env), Contributors: contribs, } @@ -919,7 +917,7 @@ func doXCodeFramework(cmdline []string) { local = flag.Bool("local", false, `Flag whether we're only doing a local build (skip Maven artifacts)`) signer = flag.String("signer", "", `Environment variable holding the signing key (e.g. IOS_SIGNING_KEY)`) deploy = flag.String("deploy", "", `Destination to deploy the archive (usually "trunk")`) - upload = flag.String("upload", "", `Destination to upload the archives (usually "gethstore/builds")`) + upload = flag.String("upload", "", `Destination to upload the archives (usually "dexon-builds")`) ) flag.CommandLine.Parse(cmdline) env := build.Env() @@ -935,7 +933,7 @@ func doXCodeFramework(cmdline []string) { build.MustRun(bind) return } - archive := "geth-" + archiveBasename("ios", params.ArchiveVersion(env.Commit)) + archive := "gdex-" + archiveBasename("ios", params.ArchiveVersion(env.Commit)) if err := os.Mkdir(archive, os.ModePerm); err != nil { log.Fatal(err) } @@ -953,8 +951,8 @@ func doXCodeFramework(cmdline []string) { // Prepare and upload a PodSpec to CocoaPods if *deploy != "" { meta := newPodMetadata(env, archive) - build.Render("build/pod.podspec", "Geth.podspec", 0755, meta) - build.MustRunCommand("pod", *deploy, "push", "Geth.podspec", "--allow-warnings", "--verbose") + build.Render("build/pod.podspec", "Gdex.podspec", 0755, meta) + build.MustRunCommand("pod", *deploy, "push", "Gdex.podspec", "--allow-warnings", "--verbose") } } @@ -1059,7 +1057,7 @@ func xgoTool(args []string) *exec.Cmd { func doPurge(cmdline []string) { var ( - store = flag.String("store", "", `Destination from where to purge archives (usually "gethstore/builds")`) + store = flag.String("store", "", `Destination from where to purge archives (usually "dexon-builds")`) limit = flag.Int("days", 30, `Age threshold above which to delete unstable archives`) ) flag.CommandLine.Parse(cmdline) @@ -1069,38 +1067,36 @@ func doPurge(cmdline []string) { os.Exit(0) } // Create the azure authentication and list the current archives - auth := build.AzureBlobstoreConfig{ - Account: strings.Split(*store, "/")[0], - Token: os.Getenv("AZURE_BLOBSTORE_TOKEN"), - Container: strings.SplitN(*store, "/", 2)[1], + auth := build.GCPOption{ + CredentialPath: os.Getenv("GCP_CREDENTIAL_PATH"), } - blobs, err := build.AzureBlobstoreList(auth) + objects, err := build.GCPFileList(*store, auth) if err != nil { log.Fatal(err) } // Iterate over the blobs, collect and sort all unstable builds - for i := 0; i < len(blobs); i++ { - if !strings.Contains(blobs[i].Name, "unstable") { - blobs = append(blobs[:i], blobs[i+1:]...) + for i := 0; i < len(objects); i++ { + if !strings.Contains(objects[i].Name, "unstable") { + objects = append(objects[:i], objects[i+1:]...) i-- } } - for i := 0; i < len(blobs); i++ { - for j := i + 1; j < len(blobs); j++ { - if blobs[i].Properties.LastModified.After(blobs[j].Properties.LastModified) { - blobs[i], blobs[j] = blobs[j], blobs[i] + for i := 0; i < len(objects); i++ { + for j := i + 1; j < len(objects); j++ { + if objects[i].Updated.After(objects[j].Updated) { + objects[i], objects[j] = objects[j], objects[i] } } } // Filter out all archives more recent that the given threshold - for i, blob := range blobs { - if time.Since(blob.Properties.LastModified) < time.Duration(*limit)*24*time.Hour { - blobs = blobs[:i] + for i, blob := range objects { + if time.Since(blob.Updated) < time.Duration(*limit)*24*time.Hour { + objects = objects[:i] break } } // Delete all marked as such and return - if err := build.AzureBlobstoreDelete(auth, blobs); err != nil { + if err := build.GCPFileDelete(*store, objects, auth); err != nil { log.Fatal(err) } } -- cgit v1.2.3