From 3b75d0ce2373c6e6da9af5f341d2d0aae0b4ed96 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 30 Aug 2016 12:20:34 +0200 Subject: Godeps: update golang.org/x/... --- .../src/golang.org/x/tools/imports/fix.go | 856 +++++++++++++++++---- 1 file changed, 706 insertions(+), 150 deletions(-) (limited to 'Godeps/_workspace/src/golang.org/x/tools/imports/fix.go') diff --git a/Godeps/_workspace/src/golang.org/x/tools/imports/fix.go b/Godeps/_workspace/src/golang.org/x/tools/imports/fix.go index 4fc50570d..a241c2976 100644 --- a/Godeps/_workspace/src/golang.org/x/tools/imports/fix.go +++ b/Godeps/_workspace/src/golang.org/x/tools/imports/fix.go @@ -5,23 +5,46 @@ package imports import ( + "bufio" + "bytes" "fmt" "go/ast" "go/build" "go/parser" "go/token" + "io/ioutil" + "log" "os" "path" "path/filepath" + "sort" "strings" "sync" "golang.org/x/tools/go/ast/astutil" ) +// Debug controls verbose logging. +var Debug = false + +var ( + inTests = false // set true by fix_test.go; if false, no need to use testMu + testMu sync.RWMutex // guards globals reset by tests; used only if inTests +) + +// If set, LocalPrefix instructs Process to sort import paths with the given +// prefix into another group after 3rd-party packages. +var LocalPrefix string + // importToGroup is a list of functions which map from an import path to // a group number. var importToGroup = []func(importPath string) (num int, ok bool){ + func(importPath string) (num int, ok bool) { + if LocalPrefix != "" && strings.HasPrefix(importPath, LocalPrefix) { + return 3, true + } + return + }, func(importPath string) (num int, ok bool) { if strings.HasPrefix(importPath, "appengine") { return 2, true @@ -45,6 +68,62 @@ func importGroup(importPath string) int { return 0 } +// packageInfo is a summary of features found in a package. +type packageInfo struct { + Globals map[string]bool // symbol => true +} + +// dirPackageInfo gets information from other files in the package. +func dirPackageInfo(srcDir, filename string) (*packageInfo, error) { + considerTests := strings.HasSuffix(filename, "_test.go") + + // Handle file from stdin + if _, err := os.Stat(filename); err != nil { + if os.IsNotExist(err) { + return &packageInfo{}, nil + } + return nil, err + } + + fileBase := filepath.Base(filename) + packageFileInfos, err := ioutil.ReadDir(srcDir) + if err != nil { + return nil, err + } + + info := &packageInfo{Globals: make(map[string]bool)} + for _, fi := range packageFileInfos { + if fi.Name() == fileBase || !strings.HasSuffix(fi.Name(), ".go") { + continue + } + if !considerTests && strings.HasSuffix(fi.Name(), "_test.go") { + continue + } + + fileSet := token.NewFileSet() + root, err := parser.ParseFile(fileSet, filepath.Join(srcDir, fi.Name()), nil, 0) + if err != nil { + continue + } + + for _, decl := range root.Decls { + genDecl, ok := decl.(*ast.GenDecl) + if !ok { + continue + } + + for _, spec := range genDecl.Specs { + valueSpec, ok := spec.(*ast.ValueSpec) + if !ok { + continue + } + info.Globals[valueSpec.Names[0].Name] = true + } + } + } + return info, nil +} + func fixImports(fset *token.FileSet, f *ast.File, filename string) (added []string, err error) { // refs are a set of possible package references currently unsatisfied by imports. // first key: either base package (e.g. "fmt") or renamed package @@ -54,6 +133,18 @@ func fixImports(fset *token.FileSet, f *ast.File, filename string) (added []stri // decls are the current package imports. key is base package or renamed package. decls := make(map[string]*ast.ImportSpec) + abs, err := filepath.Abs(filename) + if err != nil { + return nil, err + } + srcDir := filepath.Dir(abs) + if Debug { + log.Printf("fixImports(filename=%q), abs=%q, srcDir=%q ...", filename, abs, srcDir) + } + + var packageInfo *packageInfo + var loadedPackageInfo bool + // collect potential uses of packages. var visitor visitFn visitor = visitFn(func(node ast.Node) ast.Visitor { @@ -64,10 +155,14 @@ func fixImports(fset *token.FileSet, f *ast.File, filename string) (added []stri case *ast.ImportSpec: if v.Name != nil { decls[v.Name.Name] = v - } else { - local := importPathToName(strings.Trim(v.Path.Value, `\"`)) - decls[local] = v + break } + ipath := strings.Trim(v.Path.Value, `"`) + if ipath == "C" { + break + } + local := importPathToName(ipath, srcDir) + decls[local] = v case *ast.SelectorExpr: xident, ok := v.X.(*ast.Ident) if !ok { @@ -81,7 +176,11 @@ func fixImports(fset *token.FileSet, f *ast.File, filename string) (added []stri if refs[pkgName] == nil { refs[pkgName] = make(map[string]bool) } - if decls[pkgName] == nil { + if !loadedPackageInfo { + loadedPackageInfo = true + packageInfo, _ = dirPackageInfo(srcDir, filename) + } + if decls[pkgName] == nil && (packageInfo == nil || !packageInfo.Globals[pkgName]) { refs[pkgName][v.Sel.Name] = true } } @@ -108,18 +207,22 @@ func fixImports(fset *token.FileSet, f *ast.File, filename string) (added []stri astutil.DeleteNamedImport(fset, f, name, ipath) } + for pkgName, symbols := range refs { + if len(symbols) == 0 { + // skip over packages already imported + delete(refs, pkgName) + } + } + // Search for imports matching potential package references. searches := 0 type result struct { - ipath string - name string + ipath string // import path (if err == nil) + name string // optional name to rename import as err error } results := make(chan result) for pkgName, symbols := range refs { - if len(symbols) == 0 { - continue // skip over packages already imported - } go func(pkgName string, symbols map[string]bool) { ipath, rename, err := findImport(pkgName, symbols, filename) r := result{ipath: ipath, err: err} @@ -149,34 +252,131 @@ func fixImports(fset *token.FileSet, f *ast.File, filename string) (added []stri } // importPathToName returns the package name for the given import path. -var importPathToName = importPathToNameGoPath +var importPathToName func(importPath, srcDir string) (packageName string) = importPathToNameGoPath // importPathToNameBasic assumes the package name is the base of import path. -func importPathToNameBasic(importPath string) (packageName string) { +func importPathToNameBasic(importPath, srcDir string) (packageName string) { return path.Base(importPath) } // importPathToNameGoPath finds out the actual package name, as declared in its .go files. // If there's a problem, it falls back to using importPathToNameBasic. -func importPathToNameGoPath(importPath string) (packageName string) { - if buildPkg, err := build.Import(importPath, "", 0); err == nil { - return buildPkg.Name - } else { - return importPathToNameBasic(importPath) +func importPathToNameGoPath(importPath, srcDir string) (packageName string) { + // Fast path for standard library without going to disk. + if pkg, ok := stdImportPackage[importPath]; ok { + return pkg + } + + pkgName, err := importPathToNameGoPathParse(importPath, srcDir) + if Debug { + log.Printf("importPathToNameGoPathParse(%q, srcDir=%q) = %q, %v", importPath, srcDir, pkgName, err) + } + if err == nil { + return pkgName + } + return importPathToNameBasic(importPath, srcDir) +} + +// importPathToNameGoPathParse is a faster version of build.Import if +// the only thing desired is the package name. It uses build.FindOnly +// to find the directory and then only parses one file in the package, +// trusting that the files in the directory are consistent. +func importPathToNameGoPathParse(importPath, srcDir string) (packageName string, err error) { + buildPkg, err := build.Import(importPath, srcDir, build.FindOnly) + if err != nil { + return "", err + } + d, err := os.Open(buildPkg.Dir) + if err != nil { + return "", err + } + names, err := d.Readdirnames(-1) + d.Close() + if err != nil { + return "", err + } + sort.Strings(names) // to have predictable behavior + var lastErr error + var nfile int + for _, name := range names { + if !strings.HasSuffix(name, ".go") { + continue + } + if strings.HasSuffix(name, "_test.go") { + continue + } + nfile++ + fullFile := filepath.Join(buildPkg.Dir, name) + + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, fullFile, nil, parser.PackageClauseOnly) + if err != nil { + lastErr = err + continue + } + pkgName := f.Name.Name + if pkgName == "documentation" { + // Special case from go/build.ImportDir, not + // handled by ctx.MatchFile. + continue + } + if pkgName == "main" { + // Also skip package main, assuming it's a +build ignore generator or example. + // Since you can't import a package main anyway, there's no harm here. + continue + } + return pkgName, nil + } + if lastErr != nil { + return "", lastErr + } + return "", fmt.Errorf("no importable package found in %d Go files", nfile) +} + +var stdImportPackage = map[string]string{} // "net/http" => "http" + +func init() { + // Nothing in the standard library has a package name not + // matching its import base name. + for _, pkg := range stdlib { + if _, ok := stdImportPackage[pkg]; !ok { + stdImportPackage[pkg] = path.Base(pkg) + } } } +// Directory-scanning state. +var ( + // scanGoRootOnce guards calling scanGoRoot (for $GOROOT) + scanGoRootOnce sync.Once + // scanGoPathOnce guards calling scanGoPath (for $GOPATH) + scanGoPathOnce sync.Once + + // populateIgnoreOnce guards calling populateIgnore + populateIgnoreOnce sync.Once + ignoredDirs []os.FileInfo + + dirScanMu sync.RWMutex + dirScan map[string]*pkg // abs dir path => *pkg +) + type pkg struct { - importpath string // full pkg import path, e.g. "net/http" - dir string // absolute file path to pkg directory e.g. "/usr/lib/go/src/fmt" + dir string // absolute file path to pkg directory ("/usr/lib/go/src/net/http") + importPath string // full pkg import path ("net/http", "foo/bar/vendor/a/b") + importPathShort string // vendorless import path ("net/http", "a/b") } -var pkgIndexOnce sync.Once +// byImportPathShortLength sorts by the short import path length, breaking ties on the +// import string itself. +type byImportPathShortLength []*pkg + +func (s byImportPathShortLength) Len() int { return len(s) } +func (s byImportPathShortLength) Less(i, j int) bool { + vi, vj := s[i].importPathShort, s[j].importPathShort + return len(vi) < len(vj) || (len(vi) == len(vj) && vi < vj) -var pkgIndex struct { - sync.Mutex - m map[string][]pkg // shortname => []pkg, e.g "http" => "net/http" } +func (s byImportPathShortLength) Swap(i, j int) { s[i], s[j] = s[j], s[i] } // gate is a semaphore for limiting concurrency. type gate chan struct{} @@ -184,194 +384,512 @@ type gate chan struct{} func (g gate) enter() { g <- struct{}{} } func (g gate) leave() { <-g } -// fsgate protects the OS & filesystem from too much concurrency. -// Too much disk I/O -> too many threads -> swapping and bad scheduling. -var fsgate = make(gate, 8) - -func loadPkgIndex() { - pkgIndex.Lock() - pkgIndex.m = make(map[string][]pkg) - pkgIndex.Unlock() +var visitedSymlinks struct { + sync.Mutex + m map[string]struct{} +} - var wg sync.WaitGroup - for _, path := range build.Default.SrcDirs() { - fsgate.enter() - f, err := os.Open(path) - if err != nil { - fsgate.leave() - fmt.Fprint(os.Stderr, err) +// guarded by populateIgnoreOnce; populates ignoredDirs. +func populateIgnore() { + for _, srcDir := range build.Default.SrcDirs() { + if srcDir == filepath.Join(build.Default.GOROOT, "src") { continue } - children, err := f.Readdir(-1) - f.Close() - fsgate.leave() + populateIgnoredDirs(srcDir) + } +} + +// populateIgnoredDirs reads an optional config file at /.goimportsignore +// of relative directories to ignore when scanning for go files. +// The provided path is one of the $GOPATH entries with "src" appended. +func populateIgnoredDirs(path string) { + file := filepath.Join(path, ".goimportsignore") + slurp, err := ioutil.ReadFile(file) + if Debug { if err != nil { - fmt.Fprint(os.Stderr, err) + log.Print(err) + } else { + log.Printf("Read %s", file) + } + } + if err != nil { + return + } + bs := bufio.NewScanner(bytes.NewReader(slurp)) + for bs.Scan() { + line := strings.TrimSpace(bs.Text()) + if line == "" || strings.HasPrefix(line, "#") { continue } - for _, child := range children { - if child.IsDir() { - wg.Add(1) - go func(path, name string) { - defer wg.Done() - loadPkg(&wg, path, name) - }(path, child.Name()) + full := filepath.Join(path, line) + if fi, err := os.Stat(full); err == nil { + ignoredDirs = append(ignoredDirs, fi) + if Debug { + log.Printf("Directory added to ignore list: %s", full) } + } else if Debug { + log.Printf("Error statting entry in .goimportsignore: %v", err) } } - wg.Wait() } -func loadPkg(wg *sync.WaitGroup, root, pkgrelpath string) { - importpath := filepath.ToSlash(pkgrelpath) - dir := filepath.Join(root, importpath) +func skipDir(fi os.FileInfo) bool { + for _, ignoredDir := range ignoredDirs { + if os.SameFile(fi, ignoredDir) { + return true + } + } + return false +} - fsgate.enter() - defer fsgate.leave() - pkgDir, err := os.Open(dir) +// shouldTraverse reports whether the symlink fi should, found in dir, +// should be followed. It makes sure symlinks were never visited +// before to avoid symlink loops. +func shouldTraverse(dir string, fi os.FileInfo) bool { + path := filepath.Join(dir, fi.Name()) + target, err := filepath.EvalSymlinks(path) if err != nil { - return + if !os.IsNotExist(err) { + fmt.Fprintln(os.Stderr, err) + } + return false } - children, err := pkgDir.Readdir(-1) - pkgDir.Close() + ts, err := os.Stat(target) if err != nil { - return + fmt.Fprintln(os.Stderr, err) + return false + } + if !ts.IsDir() { + return false + } + + realParent, err := filepath.EvalSymlinks(dir) + if err != nil { + fmt.Fprint(os.Stderr, err) + return false } - // hasGo tracks whether a directory actually appears to be a - // Go source code directory. If $GOPATH == $HOME, and - // $HOME/src has lots of other large non-Go projects in it, - // then the calls to importPathToName below can be expensive. - hasGo := false - for _, child := range children { - // Avoid .foo, _foo, and testdata directory trees. - name := child.Name() - if name == "" || name[0] == '.' || name[0] == '_' || name == "testdata" { + realPath := filepath.Join(realParent, fi.Name()) + visitedSymlinks.Lock() + defer visitedSymlinks.Unlock() + if visitedSymlinks.m == nil { + visitedSymlinks.m = make(map[string]struct{}) + } + if _, ok := visitedSymlinks.m[realPath]; ok { + return false + } + visitedSymlinks.m[realPath] = struct{}{} + return true +} + +var testHookScanDir = func(dir string) {} + +var scanGoRootDone = make(chan struct{}) // closed when scanGoRoot is done + +func scanGoRoot() { + go func() { + scanGoDirs(true) + close(scanGoRootDone) + }() +} + +func scanGoPath() { scanGoDirs(false) } + +func scanGoDirs(goRoot bool) { + if Debug { + which := "$GOROOT" + if !goRoot { + which = "$GOPATH" + } + log.Printf("scanning " + which) + defer log.Printf("scanned " + which) + } + dirScanMu.Lock() + if dirScan == nil { + dirScan = make(map[string]*pkg) + } + dirScanMu.Unlock() + + for _, srcDir := range build.Default.SrcDirs() { + isGoroot := srcDir == filepath.Join(build.Default.GOROOT, "src") + if isGoroot != goRoot { continue } - if strings.HasSuffix(name, ".go") { - hasGo = true + testHookScanDir(srcDir) + walkFn := func(path string, typ os.FileMode) error { + dir := filepath.Dir(path) + if typ.IsRegular() { + if dir == srcDir { + // Doesn't make sense to have regular files + // directly in your $GOPATH/src or $GOROOT/src. + return nil + } + if !strings.HasSuffix(path, ".go") { + return nil + } + dirScanMu.Lock() + if _, dup := dirScan[dir]; !dup { + importpath := filepath.ToSlash(dir[len(srcDir)+len("/"):]) + dirScan[dir] = &pkg{ + importPath: importpath, + importPathShort: vendorlessImportPath(importpath), + dir: dir, + } + } + dirScanMu.Unlock() + return nil + } + if typ == os.ModeDir { + base := filepath.Base(path) + if base == "" || base[0] == '.' || base[0] == '_' || + base == "testdata" || base == "node_modules" { + return filepath.SkipDir + } + fi, err := os.Lstat(path) + if err == nil && skipDir(fi) { + if Debug { + log.Printf("skipping directory %q under %s", fi.Name(), dir) + } + return filepath.SkipDir + } + return nil + } + if typ == os.ModeSymlink { + base := filepath.Base(path) + if strings.HasPrefix(base, ".#") { + // Emacs noise. + return nil + } + fi, err := os.Lstat(path) + if err != nil { + // Just ignore it. + return nil + } + if shouldTraverse(dir, fi) { + return traverseLink + } + } + return nil } - if child.IsDir() { - wg.Add(1) - go func(root, name string) { - defer wg.Done() - loadPkg(wg, root, name) - }(root, filepath.Join(importpath, name)) + if err := fastWalk(srcDir, walkFn); err != nil { + log.Printf("goimports: scanning directory %v: %v", srcDir, err) } } - if hasGo { - shortName := importPathToName(importpath) - pkgIndex.Lock() - pkgIndex.m[shortName] = append(pkgIndex.m[shortName], pkg{ - importpath: importpath, - dir: dir, - }) - pkgIndex.Unlock() - } +} +// vendorlessImportPath returns the devendorized version of the provided import path. +// e.g. "foo/bar/vendor/a/b" => "a/b" +func vendorlessImportPath(ipath string) string { + // Devendorize for use in import statement. + if i := strings.LastIndex(ipath, "/vendor/"); i >= 0 { + return ipath[i+len("/vendor/"):] + } + if strings.HasPrefix(ipath, "vendor/") { + return ipath[len("vendor/"):] + } + return ipath } -// loadExports returns a list exports for a package. -var loadExports = loadExportsGoPath +// loadExports returns the set of exported symbols in the package at dir. +// It returns nil on error or if the package name in dir does not match expectPackage. +var loadExports func(expectPackage, dir string) map[string]bool = loadExportsGoPath -func loadExportsGoPath(dir string) map[string]bool { +func loadExportsGoPath(expectPackage, dir string) map[string]bool { + if Debug { + log.Printf("loading exports in dir %s (seeking package %s)", dir, expectPackage) + } exports := make(map[string]bool) - buildPkg, err := build.ImportDir(dir, 0) - if err != nil { - if strings.Contains(err.Error(), "no buildable Go source files in") { - return nil + + ctx := build.Default + + // ReadDir is like ioutil.ReadDir, but only returns *.go files + // and filters out _test.go files since they're not relevant + // and only slow things down. + ctx.ReadDir = func(dir string) (notTests []os.FileInfo, err error) { + all, err := ioutil.ReadDir(dir) + if err != nil { + return nil, err } - fmt.Fprintf(os.Stderr, "could not import %q: %v\n", dir, err) + notTests = all[:0] + for _, fi := range all { + name := fi.Name() + if strings.HasSuffix(name, ".go") && !strings.HasSuffix(name, "_test.go") { + notTests = append(notTests, fi) + } + } + return notTests, nil + } + + files, err := ctx.ReadDir(dir) + if err != nil { + log.Print(err) return nil } + fset := token.NewFileSet() - for _, files := range [...][]string{buildPkg.GoFiles, buildPkg.CgoFiles} { - for _, file := range files { - f, err := parser.ParseFile(fset, filepath.Join(dir, file), nil, 0) - if err != nil { - fmt.Fprintf(os.Stderr, "could not parse %q: %v\n", file, err) - continue + + for _, fi := range files { + match, err := ctx.MatchFile(dir, fi.Name()) + if err != nil || !match { + continue + } + fullFile := filepath.Join(dir, fi.Name()) + f, err := parser.ParseFile(fset, fullFile, nil, 0) + if err != nil { + if Debug { + log.Printf("Parsing %s: %v", fullFile, err) } - for name := range f.Scope.Objects { - if ast.IsExported(name) { - exports[name] = true - } + return nil + } + pkgName := f.Name.Name + if pkgName == "documentation" { + // Special case from go/build.ImportDir, not + // handled by ctx.MatchFile. + continue + } + if pkgName != expectPackage { + if Debug { + log.Printf("scan of dir %v is not expected package %v (actually %v)", dir, expectPackage, pkgName) + } + return nil + } + for name := range f.Scope.Objects { + if ast.IsExported(name) { + exports[name] = true } } } + + if Debug { + exportList := make([]string, 0, len(exports)) + for k := range exports { + exportList = append(exportList, k) + } + sort.Strings(exportList) + log.Printf("loaded exports in dir %v (package %v): %v", dir, expectPackage, strings.Join(exportList, ", ")) + } return exports } // findImport searches for a package with the given symbols. -// If no package is found, findImport returns "". -// Declared as a variable rather than a function so goimports can be easily -// extended by adding a file with an init function. -var findImport = findImportGoPath +// If no package is found, findImport returns ("", false, nil) +// +// This is declared as a variable rather than a function so goimports +// can be easily extended by adding a file with an init function. +// +// The rename value tells goimports whether to use the package name as +// a local qualifier in an import. For example, if findImports("pkg", +// "X") returns ("foo/bar", rename=true), then goimports adds the +// import line: +// import pkg "foo/bar" +// to satisfy uses of pkg.X in the file. +var findImport func(pkgName string, symbols map[string]bool, filename string) (foundPkg string, rename bool, err error) = findImportGoPath + +// findImportGoPath is the normal implementation of findImport. +// (Some companies have their own internally.) +func findImportGoPath(pkgName string, symbols map[string]bool, filename string) (foundPkg string, rename bool, err error) { + if inTests { + testMu.RLock() + defer testMu.RUnlock() + } -func findImportGoPath(pkgName string, symbols map[string]bool, filename string) (string, bool, error) { // Fast path for the standard library. // In the common case we hopefully never have to scan the GOPATH, which can // be slow with moving disks. if pkg, rename, ok := findImportStdlib(pkgName, symbols); ok { return pkg, rename, nil } + if pkgName == "rand" && symbols["Read"] { + // Special-case rand.Read. + // + // If findImportStdlib didn't find it above, don't go + // searching for it, lest it find and pick math/rand + // in GOROOT (new as of Go 1.6) + // + // crypto/rand is the safer choice. + return "", false, nil + } // TODO(sameer): look at the import lines for other Go files in the // local directory, since the user is likely to import the same packages // in the current Go file. Return rename=true when the other Go files // use a renamed package that's also used in the current file. - pkgIndexOnce.Do(loadPkgIndex) + // Read all the $GOPATH/src/.goimportsignore files before scanning directories. + populateIgnoreOnce.Do(populateIgnore) + + // Start scanning the $GOROOT asynchronously, then run the + // GOPATH scan synchronously if needed, and then wait for the + // $GOROOT to finish. + // + // TODO(bradfitz): run each $GOPATH entry async. But nobody + // really has more than one anyway, so low priority. + scanGoRootOnce.Do(scanGoRoot) // async + if !fileInDir(filename, build.Default.GOROOT) { + scanGoPathOnce.Do(scanGoPath) // blocking + } + <-scanGoRootDone - // Collect exports for packages with matching names. - var ( - wg sync.WaitGroup - mu sync.Mutex - shortest string - ) - pkgIndex.Lock() - for _, pkg := range pkgIndex.m[pkgName] { - if !canUse(filename, pkg.dir) { - continue + // Find candidate packages, looking only at their directory names first. + var candidates []*pkg + for _, pkg := range dirScan { + if pkgIsCandidate(filename, pkgName, pkg) { + candidates = append(candidates, pkg) + } + } + + // Sort the candidates by their import package length, + // assuming that shorter package names are better than long + // ones. Note that this sorts by the de-vendored name, so + // there's no "penalty" for vendoring. + sort.Sort(byImportPathShortLength(candidates)) + if Debug { + for i, pkg := range candidates { + log.Printf("%s candidate %d/%d: %v", pkgName, i+1, len(candidates), pkg.importPathShort) } - wg.Add(1) - go func(importpath, dir string) { - defer wg.Done() - exports := loadExports(dir) - if exports == nil { + } + + // Collect exports for packages with matching names. + + done := make(chan struct{}) // closed when we find the answer + defer close(done) + + rescv := make([]chan *pkg, len(candidates)) + for i := range candidates { + rescv[i] = make(chan *pkg) + } + const maxConcurrentPackageImport = 4 + loadExportsSem := make(chan struct{}, maxConcurrentPackageImport) + + go func() { + for i, pkg := range candidates { + select { + case loadExportsSem <- struct{}{}: + select { + case <-done: + default: + } + case <-done: return } - // If it doesn't have the right symbols, stop. - for symbol := range symbols { - if !exports[symbol] { - return + pkg := pkg + resc := rescv[i] + go func() { + if inTests { + testMu.RLock() + defer testMu.RUnlock() } - } + defer func() { <-loadExportsSem }() + exports := loadExports(pkgName, pkg.dir) + + // If it doesn't have the right + // symbols, send nil to mean no match. + for symbol := range symbols { + if !exports[symbol] { + pkg = nil + break + } + } + select { + case resc <- pkg: + case <-done: + } + }() + } + }() + for _, resc := range rescv { + pkg := <-resc + if pkg == nil { + continue + } + // If the package name in the source doesn't match the import path's base, + // return true so the rewriter adds a name (import foo "github.com/bar/go-foo") + needsRename := path.Base(pkg.importPath) != pkgName + return pkg.importPathShort, needsRename, nil + } + return "", false, nil +} - // Devendorize for use in import statement. - if i := strings.LastIndex(importpath, "/vendor/"); i >= 0 { - importpath = importpath[i+len("/vendor/"):] - } else if strings.HasPrefix(importpath, "vendor/") { - importpath = importpath[len("vendor/"):] - } +// pkgIsCandidate reports whether pkg is a candidate for satisfying the +// finding which package pkgIdent in the file named by filename is trying +// to refer to. +// +// This check is purely lexical and is meant to be as fast as possible +// because it's run over all $GOPATH directories to filter out poor +// candidates in order to limit the CPU and I/O later parsing the +// exports in candidate packages. +// +// filename is the file being formatted. +// pkgIdent is the package being searched for, like "client" (if +// searching for "client.New") +func pkgIsCandidate(filename, pkgIdent string, pkg *pkg) bool { + // Check "internal" and "vendor" visibility: + if !canUse(filename, pkg.dir) { + return false + } - // Save as the answer. - // If there are multiple candidates, the shortest wins, - // to prefer "bytes" over "github.com/foo/bytes". - mu.Lock() - if shortest == "" || len(importpath) < len(shortest) || len(importpath) == len(shortest) && importpath < shortest { - shortest = importpath - } - mu.Unlock() - }(pkg.importpath, pkg.dir) + // Speed optimization to minimize disk I/O: + // the last two components on disk must contain the + // package name somewhere. + // + // This permits mismatch naming like directory + // "go-foo" being package "foo", or "pkg.v3" being "pkg", + // or directory "google.golang.org/api/cloudbilling/v1" + // being package "cloudbilling", but doesn't + // permit a directory "foo" to be package + // "bar", which is strongly discouraged + // anyway. There's no reason goimports needs + // to be slow just to accomodate that. + lastTwo := lastTwoComponents(pkg.importPathShort) + if strings.Contains(lastTwo, pkgIdent) { + return true + } + if hasHyphenOrUpperASCII(lastTwo) && !hasHyphenOrUpperASCII(pkgIdent) { + lastTwo = lowerASCIIAndRemoveHyphen(lastTwo) + if strings.Contains(lastTwo, pkgIdent) { + return true + } } - pkgIndex.Unlock() - wg.Wait() - return shortest, false, nil + return false +} + +func hasHyphenOrUpperASCII(s string) bool { + for i := 0; i < len(s); i++ { + b := s[i] + if b == '-' || ('A' <= b && b <= 'Z') { + return true + } + } + return false } +func lowerASCIIAndRemoveHyphen(s string) (ret string) { + buf := make([]byte, 0, len(s)) + for i := 0; i < len(s); i++ { + b := s[i] + switch { + case b == '-': + continue + case 'A' <= b && b <= 'Z': + buf = append(buf, b+('a'-'A')) + default: + buf = append(buf, b) + } + } + return string(buf) +} + +// canUse reports whether the package in dir is usable from filename, +// respecting the Go "internal" and "vendor" visibility rules. func canUse(filename, dir string) bool { + // Fast path check, before any allocations. If it doesn't contain vendor + // or internal, it's not tricky: + // Note that this can false-negative on directories like "notinternal", + // but we check it correctly below. This is just a fast path. + if !strings.Contains(dir, "vendor") && !strings.Contains(dir, "internal") { + return true + } + dirSlash := filepath.ToSlash(dir) if !strings.Contains(dirSlash, "/vendor/") && !strings.Contains(dirSlash, "/internal/") && !strings.HasSuffix(dirSlash, "/internal") { return true @@ -382,11 +900,15 @@ func canUse(filename, dir string) bool { // or bar/vendor or bar/internal. // After stripping all the leading ../, the only okay place to see vendor or internal // is at the very beginning of the path. - abs, err := filepath.Abs(filename) + absfile, err := filepath.Abs(filename) if err != nil { return false } - rel, err := filepath.Rel(abs, dir) + absdir, err := filepath.Abs(dir) + if err != nil { + return false + } + rel, err := filepath.Rel(absfile, absdir) if err != nil { return false } @@ -397,6 +919,21 @@ func canUse(filename, dir string) bool { return !strings.Contains(relSlash, "/vendor/") && !strings.Contains(relSlash, "/internal/") && !strings.HasSuffix(relSlash, "/internal") } +// lastTwoComponents returns at most the last two path components +// of v, using either / or \ as the path separator. +func lastTwoComponents(v string) string { + nslash := 0 + for i := len(v) - 1; i >= 0; i-- { + if v[i] == '/' || v[i] == '\\' { + nslash++ + if nslash == 2 { + return v[i:] + } + } + } + return v +} + type visitFn func(node ast.Node) ast.Visitor func (fn visitFn) Visit(node ast.Node) ast.Visitor { @@ -405,8 +942,12 @@ func (fn visitFn) Visit(node ast.Node) ast.Visitor { func findImportStdlib(shortPkg string, symbols map[string]bool) (importPath string, rename, ok bool) { for symbol := range symbols { - path := stdlib[shortPkg+"."+symbol] + key := shortPkg + "." + symbol + path := stdlib[key] if path == "" { + if key == "rand.Read" { + continue + } return "", false, false } if importPath != "" && importPath != path { @@ -415,5 +956,20 @@ func findImportStdlib(shortPkg string, symbols map[string]bool) (importPath stri } importPath = path } + if importPath == "" && shortPkg == "rand" && symbols["Read"] { + return "crypto/rand", false, true + } return importPath, false, importPath != "" } + +// fileInDir reports whether the provided file path looks like +// it's in dir. (without hitting the filesystem) +func fileInDir(file, dir string) bool { + rest := strings.TrimPrefix(file, dir) + if len(rest) == len(file) { + // dir is not a prefix of file. + return false + } + // Check for boundary: either nothing (file == dir), or a slash. + return len(rest) == 0 || rest[0] == '/' || rest[0] == '\\' +} -- cgit v1.2.3