From 9f5fb15097198ea20aaae31983a7101ac0679eaa Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 12 Feb 2019 10:55:25 +0100 Subject: build: use SFTP for launchpad uploads (#19037) * build: use sftp for launchpad uploads * .travis.yml: configure sftp export * build: update CI docs (cherry picked from commit 3de19c8b31ab975eed1f7f276d31761f7f8b9af9) --- build/ci-notes.md | 13 +++++++++--- build/ci.go | 56 ++++++++++++++++++++++++++++++++++--------------- build/dput-launchpad.cf | 8 +++++++ 3 files changed, 57 insertions(+), 20 deletions(-) create mode 100644 build/dput-launchpad.cf (limited to 'build') diff --git a/build/ci-notes.md b/build/ci-notes.md index f5b0e869d..ba27cdb1d 100644 --- a/build/ci-notes.md +++ b/build/ci-notes.md @@ -7,11 +7,18 @@ Canonical. Packages of develop branch commits have suffix -unstable and cannot be installed alongside the stable version. Switching between release streams requires user intervention. +## Launchpad + The packages are built and served by launchpad.net. We generate a Debian source package for each distribution and upload it. Their builder picks up the source package, builds it and installs the new version into the PPA repository. Launchpad requires a valid signature -by a team member for source package uploads. The signing key is stored in an environment -variable which Travis CI makes available to certain builds. +by a team member for source package uploads. + +The signing key is stored in an environment variable which Travis CI makes available to +certain builds. Since Travis CI doesn't support FTP, SFTP is used to transfer the +packages. To set this up yourself, you need to create a Launchpad user and add a GPG key +and SSH key to it. Then encode both keys as base64 and configure 'secret' environment +variables `PPA_SIGNING_KEY` and `PPA_SSH_KEY` on Travis. We want to build go-ethereum with the most recent version of Go, irrespective of the Go version that is available in the main Ubuntu repository. In order to make this possible, @@ -27,7 +34,7 @@ Add the gophers PPA and install Go 1.10 and Debian packaging tools: $ sudo apt-add-repository ppa:gophers/ubuntu/archive $ sudo apt-get update - $ sudo apt-get install build-essential golang-1.10 devscripts debhelper + $ sudo apt-get install build-essential golang-1.10 devscripts debhelper python-bzrlib python-paramiko Create the source packages: diff --git a/build/ci.go b/build/ci.go index d2f4ce4fe..14d97135b 100644 --- a/build/ci.go +++ b/build/ci.go @@ -441,11 +441,8 @@ func archiveBasename(arch string, archiveVersion string) string { func archiveUpload(archive string, blobstore string, signer string) error { // If signing was requested, generate the signature files if signer != "" { - pgpkey, err := base64.StdEncoding.DecodeString(os.Getenv(signer)) - if err != nil { - return fmt.Errorf("invalid base64 %s", signer) - } - if err := build.PGPSignFile(archive, archive+".asc", string(pgpkey)); err != nil { + key := getenvBase64(signer) + if err := build.PGPSignFile(archive, archive+".asc", string(key)); err != nil { return err } } @@ -489,6 +486,7 @@ func doDebianSource(cmdline []string) { var ( signer = flag.String("signer", "", `Signing key name, also used as package author`) upload = flag.String("upload", "", `Where to upload the source package (usually "ppa:ethereum/ethereum")`) + sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`) workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) now = time.Now() ) @@ -498,11 +496,7 @@ func doDebianSource(cmdline []string) { maybeSkipArchive(env) // Import the signing key. - if b64key := os.Getenv("PPA_SIGNING_KEY"); b64key != "" { - key, err := base64.StdEncoding.DecodeString(b64key) - if err != nil { - log.Fatal("invalid base64 PPA_SIGNING_KEY") - } + if key := getenvBase64("PPA_SIGNING_KEY"); len(key) > 0 { gpg := exec.Command("gpg", "--import") gpg.Stdin = bytes.NewReader(key) build.MustRun(gpg) @@ -523,12 +517,45 @@ func doDebianSource(cmdline []string) { build.MustRunCommand("debsign", changes) } if *upload != "" { - build.MustRunCommand("dput", "--passive", "--no-upload-log", *upload, changes) + uploadDebianSource(*workdir, *upload, *sshUser, changes) } } } } +func uploadDebianSource(workdir, ppa, sshUser, changes string) { + // Create the dput config file. + dputConfig := filepath.Join(workdir, "dput.cf") + p := strings.Split(ppa, "/") + if len(p) != 2 { + log.Fatal("-upload PPA name must contain single /") + } + templateData := map[string]string{ + "LaunchpadUser": p[0], + "LaunchpadPPA": p[1], + "LaunchpadSSH": sshUser, + } + if sshkey := getenvBase64("PPA_SSH_KEY"); len(sshkey) > 0 { + idfile := filepath.Join(workdir, "sshkey") + ioutil.WriteFile(idfile, sshkey, 0600) + templateData["IdentityFile"] = idfile + } + build.Render("build/dput-launchpad.cf", dputConfig, 0644, templateData) + + // Run dput to do the upload. + dput := exec.Command("dput", "-c", dputConfig, "--no-upload-log", ppa, changes) + dput.Stdin = strings.NewReader("Yes\n") // accept SSH host key + build.MustRun(dput) +} + +func getenvBase64(variable string) []byte { + dec, err := base64.StdEncoding.DecodeString(os.Getenv(variable)) + if err != nil { + log.Fatal("invalid base64 " + variable) + } + return []byte(dec) +} + func makeWorkdir(wdflag string) string { var err error if wdflag != "" { @@ -800,15 +827,10 @@ func doAndroidArchive(cmdline []string) { os.Rename(archive, meta.Package+".aar") if *signer != "" && *deploy != "" { // Import the signing key into the local GPG instance - b64key := os.Getenv(*signer) - key, err := base64.StdEncoding.DecodeString(b64key) - if err != nil { - log.Fatalf("invalid base64 %s", *signer) - } + key := getenvBase64(*signer) gpg := exec.Command("gpg", "--import") gpg.Stdin = bytes.NewReader(key) build.MustRun(gpg) - keyID, err := build.PGPKeyID(string(key)) if err != nil { log.Fatal(err) diff --git a/build/dput-launchpad.cf b/build/dput-launchpad.cf new file mode 100644 index 000000000..3063c3c07 --- /dev/null +++ b/build/dput-launchpad.cf @@ -0,0 +1,8 @@ +[{{.LaunchpadUser}}/{{.LaunchpadPPA}}] +fqdn = ppa.launchpad.net +method = sftp +incoming = ~{{.LaunchpadUser}}/ubuntu/{{.LaunchpadPPA}}/ +login = {{.LaunchpadSSH}} +{{ if .IdentityFile }} + ssh_options = IdentityFile {{.IdentityFile}} +{{ end }} -- cgit v1.2.3 From 276f824707783ecbe092e521b5a7460515ee3e99 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 12 Feb 2019 13:02:40 +0100 Subject: .travis.yml: fix upload destination (#19043) (cherry picked from commit edf976ee8e7e1561cf11cbdc5a0c5edb497dda34) --- build/ci.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'build') diff --git a/build/ci.go b/build/ci.go index 14d97135b..6ca08d9ef 100644 --- a/build/ci.go +++ b/build/ci.go @@ -485,7 +485,7 @@ func maybeSkipArchive(env build.Environment) { func doDebianSource(cmdline []string) { var ( signer = flag.String("signer", "", `Signing key name, also used as package author`) - upload = flag.String("upload", "", `Where to upload the source package (usually "ppa:ethereum/ethereum")`) + upload = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`) sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`) workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) now = time.Now() -- cgit v1.2.3 From b247052a64a4954de83e50970c1551efe9f8a08b Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 14 Feb 2019 16:10:09 +0100 Subject: build: avoid dput and upload with sftp directly (#19067) (cherry picked from commit a8ddf7ad8393cff80848b193c698ce5e6440e061) --- build/ci.go | 41 ++++++++++++++++++++++------------------- build/dput-launchpad.cf | 8 -------- 2 files changed, 22 insertions(+), 27 deletions(-) delete mode 100644 build/dput-launchpad.cf (limited to 'build') diff --git a/build/ci.go b/build/ci.go index 6ca08d9ef..11fa759da 100644 --- a/build/ci.go +++ b/build/ci.go @@ -511,41 +511,44 @@ func doDebianSource(cmdline []string) { debuild.Dir = pkgdir build.MustRun(debuild) - changes := fmt.Sprintf("%s_%s_source.changes", meta.Name(), meta.VersionString()) - changes = filepath.Join(*workdir, changes) + var ( + basename = fmt.Sprintf("%s_%s", meta.Name(), meta.VersionString()) + source = filepath.Join(*workdir, basename+".tar.xz") + dsc = filepath.Join(*workdir, basename+".dsc") + changes = filepath.Join(*workdir, basename+"_source.changes") + ) if *signer != "" { build.MustRunCommand("debsign", changes) } if *upload != "" { - uploadDebianSource(*workdir, *upload, *sshUser, changes) + ppaUpload(*workdir, *upload, *sshUser, []string{source, dsc, changes}) } } } } -func uploadDebianSource(workdir, ppa, sshUser, changes string) { - // Create the dput config file. - dputConfig := filepath.Join(workdir, "dput.cf") +func ppaUpload(workdir, ppa, sshUser string, files []string) { p := strings.Split(ppa, "/") if len(p) != 2 { log.Fatal("-upload PPA name must contain single /") } - templateData := map[string]string{ - "LaunchpadUser": p[0], - "LaunchpadPPA": p[1], - "LaunchpadSSH": sshUser, + if sshUser == "" { + sshUser = p[0] } + incomingDir := fmt.Sprintf("~%s/ubuntu/%s", p[0], p[1]) + // Create the SSH identity file if it doesn't exist. + var idfile string if sshkey := getenvBase64("PPA_SSH_KEY"); len(sshkey) > 0 { - idfile := filepath.Join(workdir, "sshkey") - ioutil.WriteFile(idfile, sshkey, 0600) - templateData["IdentityFile"] = idfile + idfile = filepath.Join(workdir, "sshkey") + if _, err := os.Stat(idfile); os.IsNotExist(err) { + ioutil.WriteFile(idfile, sshkey, 0600) + } + } + // Upload + dest := sshUser + "@ppa.launchpad.net" + if err := build.UploadSFTP(idfile, dest, incomingDir, files); err != nil { + log.Fatal(err) } - build.Render("build/dput-launchpad.cf", dputConfig, 0644, templateData) - - // Run dput to do the upload. - dput := exec.Command("dput", "-c", dputConfig, "--no-upload-log", ppa, changes) - dput.Stdin = strings.NewReader("Yes\n") // accept SSH host key - build.MustRun(dput) } func getenvBase64(variable string) []byte { diff --git a/build/dput-launchpad.cf b/build/dput-launchpad.cf deleted file mode 100644 index 3063c3c07..000000000 --- a/build/dput-launchpad.cf +++ /dev/null @@ -1,8 +0,0 @@ -[{{.LaunchpadUser}}/{{.LaunchpadPPA}}] -fqdn = ppa.launchpad.net -method = sftp -incoming = ~{{.LaunchpadUser}}/ubuntu/{{.LaunchpadPPA}}/ -login = {{.LaunchpadSSH}} -{{ if .IdentityFile }} - ssh_options = IdentityFile {{.IdentityFile}} -{{ end }} -- cgit v1.2.3 From df355eceb41f4beec611faea0df617ea59980bef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 18 Feb 2019 12:12:26 +0200 Subject: build: explicitly force .xz compression (old debuild picks gzip) (#19118) (cherry picked from commit c0b9c763bb1572c202a60b82e7dcdc48dc3c280a) --- build/ci.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'build') diff --git a/build/ci.go b/build/ci.go index 11fa759da..4ee76ced5 100644 --- a/build/ci.go +++ b/build/ci.go @@ -507,7 +507,7 @@ func doDebianSource(cmdline []string) { for _, distro := range debDistros { meta := newDebMetadata(distro, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables) pkgdir := stageDebianSource(*workdir, meta) - debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d") + debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d", "-Zxz") debuild.Dir = pkgdir build.MustRun(debuild) -- cgit v1.2.3